diff --git a/.gitattributes b/.gitattributes index 5093538938123574d64a661e635ebf9a3fdc2a76..1bf49805a0be03e59410d82a9cd57b9e03637f21 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,46 +1,7 @@ -*.7z filter=lfs diff=lfs merge=lfs -text -*.arrow filter=lfs diff=lfs merge=lfs -text -*.bin filter=lfs diff=lfs merge=lfs -text -*.bz2 filter=lfs diff=lfs merge=lfs -text -*.ckpt filter=lfs diff=lfs merge=lfs -text -*.ftz filter=lfs diff=lfs merge=lfs -text -*.gz filter=lfs diff=lfs merge=lfs -text -*.h5 filter=lfs diff=lfs merge=lfs -text -*.joblib filter=lfs diff=lfs merge=lfs -text -*.lfs.* filter=lfs diff=lfs merge=lfs -text -*.mlmodel filter=lfs diff=lfs merge=lfs -text -*.model filter=lfs diff=lfs merge=lfs -text -*.msgpack filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.tif filter=lfs diff=lfs merge=lfs -text +*.tiff filter=lfs diff=lfs merge=lfs -text +*.png filter=lfs diff=lfs merge=lfs -text +*.pdf filter=lfs diff=lfs merge=lfs -text *.npy filter=lfs diff=lfs merge=lfs -text *.npz filter=lfs diff=lfs merge=lfs -text -*.onnx filter=lfs diff=lfs merge=lfs -text -*.ot filter=lfs diff=lfs merge=lfs -text -*.parquet filter=lfs diff=lfs merge=lfs -text -*.pb filter=lfs diff=lfs merge=lfs -text -*.pickle filter=lfs diff=lfs merge=lfs -text -*.pkl filter=lfs diff=lfs merge=lfs -text -*.pt filter=lfs diff=lfs merge=lfs -text -*.pth filter=lfs diff=lfs merge=lfs -text -*.rar filter=lfs diff=lfs merge=lfs -text -*.safetensors filter=lfs diff=lfs merge=lfs -text -saved_model/**/* filter=lfs diff=lfs merge=lfs -text -*.tar.* filter=lfs diff=lfs merge=lfs -text -*.tar filter=lfs diff=lfs merge=lfs -text -*.tflite filter=lfs diff=lfs merge=lfs -text -*.tgz filter=lfs diff=lfs merge=lfs -text -*.wasm filter=lfs diff=lfs merge=lfs -text -*.xz filter=lfs diff=lfs merge=lfs -text -*.zip filter=lfs diff=lfs merge=lfs -text -*.zst filter=lfs diff=lfs merge=lfs -text -*tfevents* filter=lfs diff=lfs merge=lfs -text -assets/primary_rank_change_preview.png filter=lfs diff=lfs merge=lfs -text -assets/selection_regret_preview.png filter=lfs diff=lfs merge=lfs -text -assets/task_rank_map_preview.png filter=lfs diff=lfs merge=lfs -text -paper/manuscript_final.pdf filter=lfs diff=lfs merge=lfs -text -paper_outputs/figures/fig_selection_regret_scatter.pdf filter=lfs diff=lfs merge=lfs -text -paper_outputs/figures/overview_wildfire.pdf filter=lfs diff=lfs merge=lfs -text -assets/overview_final.png filter=lfs diff=lfs merge=lfs -text -assets/primary_rank_change_final.png filter=lfs diff=lfs merge=lfs -text -assets/selection_regret_final.png filter=lfs diff=lfs merge=lfs -text -assets/supporting_rank_map_final.png filter=lfs diff=lfs merge=lfs -text -paper/wildfire_fm_evaluation_contracts.pdf filter=lfs diff=lfs merge=lfs -text diff --git a/.gitignore b/.gitignore deleted file mode 100644 index fa8debcaf4c6c24650a584d5abb523c4e1e2be13..0000000000000000000000000000000000000000 --- a/.gitignore +++ /dev/null @@ -1,6 +0,0 @@ -__pycache__/ -*.py[cod] -.DS_Store -.ipynb_checkpoints/ -*.log -tmp/ diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 0000000000000000000000000000000000000000..7798244dbb89dd7c366d3f8a8a788d8730aeb902 --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,10 @@ +cff-version: 1.2.0 +title: "FireWx-FM" +message: "If you use this repository, please cite the FireWx-FM technical report or model release." +type: software +authors: + - family-names: Yang + given-names: X. +date-released: 2026-07-02 +repository-code: "https://github.com/yx21e/Wildfire-fm-evaluation-contracts" +abstract: "A wildfire-specialized gridded reference model for 12-hour active-fire occupancy prediction over the Lower 48 United States." diff --git a/LICENSE b/LICENSE index 9663b6210060f0b72e444d3e11a4460912268e9c..2254be1ae04751bd7e7066e977d5fee8de8bbcfd 100644 --- a/LICENSE +++ b/LICENSE @@ -1,9 +1,25 @@ -This release is provided for scholarly reproducibility of the associated paper. +Code in this repository is released for research use under the MIT License. -Code files are released under the MIT License. -Paper-output tables, figures, and bundled summary artifacts are released for -non-commercial research and review use with attribution. +Copyright (c) 2026 FireWx-FM contributors -Raw data from NOAA, NASA, LANDFIRE, Wildfire Risk to Communities, LandScan, -WFIGS, MTBS, and model providers are not redistributed here. Users must obtain -those data from the original providers and comply with their terms. +Permission is hereby granted, free of charge, to any person obtaining a copy of +the code files in this repository and associated documentation files to deal in +the code without restriction, including without limitation the rights to use, +copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the +code, and to permit persons to whom the code is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the code. + +THE CODE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, +INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE CODE OR THE USE OR OTHER DEALINGS IN THE CODE. + +Raw NOAA, NASA FIRMS, LANDFIRE, Wildfire Risk to Communities, and LandScan +datasets are not redistributed by this +repository. Users must obtain source data from original providers and comply +with the applicable terms. diff --git a/README.md b/README.md index 9093f19284c8e10e2e6e3f58a875314927f64337..a0d83dc8f15844abb1c72deb16dfd3961a49d28c 100644 --- a/README.md +++ b/README.md @@ -5,422 +5,105 @@ tags: - geospatial - weather - earth-observation - - foundation-models - - evaluation - pytorch + - reproducibility pipeline_tag: image-segmentation library_name: pytorch -pretty_name: WildFIRE-FM +pretty_name: FireWx-FM --- -
+# FireWx-FM -

WildFIRE-FM

+FireWx-FM is a wildfire-focused gridded model for 12-hour active-fire occupancy +prediction over the Lower 48 United States. The released code maps a fixed +16-channel tensor on a 5 km EPSG:5070 grid to an active-fire occupancy +probability map. -

A wildfire-specialized reference backbone for fixed-contract Earth-FM transfer evaluation

+This Hugging Face release is the clean ICLR/technical-report companion package: +it includes model code, checkpoints, inference utilities, training/cache +templates, public data-source notes, example outputs, the ICLR manuscript +source, and the technical report source. -

Five seeded California regional PyTorch checkpoints, paper-aligned evaluation artifacts, final figure previews, source-data access notes, and a CONUS retraining recipe for 12-hour wildfire occupancy prediction.

+## Contents -

- Paper PDF - Checkpoints - PyTorch - Data notes - Results - Figures - Grid - Input - Tasks - License -

- -

- Read the paper PDF · - Download checkpoints · - Training/data loader · - Custom spatial serving · - Load the model · - Inspect task-contract results -

- -

5 California Seeded Checkpoints · 16-Channel Gridded Input · 12-Hour Occupancy Lead · CONUS Retraining Recipe

- -

- Why WildFIRE-FM · - Release Navigation · - Visual Tour · - Quick Start · - Training Pipeline · - Spatial Serving · - CONUS Retraining · - Integration Handoff · - Task Snapshot · - Data Sources · - How to Cite -

- -
- ---- - -

- WildFIRE-FM overview -

- -## Release Navigation - - - - - - - - - - - - - - - - - - - - -
-

Paper PDF

-

Read the compiled manuscript that defines the fixed-contract wildfire transfer setting and reports the final results.

-
-

Model Checkpoints

-

Access five seeded WildFIRE-FM weights with manifest metadata and SHA-256 hashes for release auditing.

-
-

Model Code

-

Inspect the compact U-Net implementation used to load the released wildfire occupancy backbone.

-
-

Training/Data Loader

-

Read the original cache builder, tensor assembly, tile sampler, full-map loader, and training loop.

-
-

Custom Spatial Serving

-

Aggregate native grid probabilities to any user-defined polygon granularity.

-
-

Quick Start

-

Clone the repository, instantiate the model, load a seeded checkpoint, and run the artifact check.

-
-

Task Snapshot

-

View the final-paper task-contract summary for occupancy, spread, retrieval, burned area, smoke, and heat.

-
-

Numeric Artifacts

-

Use sanitized CSV and JSON summaries that back the public model-card tables and figure previews.

-
-

Visual Tour

-

Browse final-paper previews for matching-rule sensitivity, selection regret, and task-form ranking changes.

-
-

Data Sources

-

See the public provider links and roles for HRRR, FIRMS, LANDFIRE, WRC, LandScan, WFIGS, and MTBS.

-
- ---- - -## Why WildFIRE-FM - -Wildfire Earth-FM transfer scores depend strongly on the contract used for comparison: task form, metric, matching rule, spatial scope, and head-selection criterion. The paper studies these choices by holding outputs or features fixed and then changing only the evaluation contract. - -WildFIRE-FM is the in-region reference model used in those comparisons. It is trained for 12-hour gridded wildfire occupancy on a California 5 km grid, then evaluated under the same task-specific contracts as the transferred Earth-FM backbones. - -The released weights are California regional checkpoints, not nationwide-trained weights. For nationwide use, the release includes a CONUS cache and training recipe under `training/NATIONWIDE_RETRAINING.md`. - -This Hub repository is model-first. It provides the released weights, model-loading code, compact result summaries, paper previews, and source-data access notes needed to inspect the release boundary before running larger local reproductions. - ---- - -## Key Features - -- **Wildfire-specialized reference model:** Compact U-Net for 12-hour occupancy prediction on a projected California grid. -- **Five seeded checkpoints:** Seeds `1`, `7`, `42`, `99`, and `123` are released with manifest hashes. -- **Fixed-contract artifacts:** Compact summaries cover matching-rule, head-selection, and task-form comparisons from the final paper. -- **Source-data aware release:** Provider links and data roles are documented for each public resource used by the study. -- **Compiled paper PDF:** The readable manuscript is available directly from the Hub release. -- **Paper-aligned previews:** Figure assets summarize selection regret, supporting-task rank changes, and primary-task rank changes. - - - - - - - - - - -
-

Reference model for wildfire transfer

-

WildFIRE-FM combines regional weather, active-fire supervision, and static fuel, canopy, housing, and population context on the same grid used by the occupancy contract.

-
-

Contract-first evaluation artifacts

-

The released summaries keep metric roles separate across decision, overlap, ranking, regression, and supporting-task checks.

-
-

Local-data reproducibility path

-

Full raw-data reruns use provider-hosted sources and local feature caches. The Hub release documents those inputs and provides lightweight artifact checks.

-
-

Final-paper result previews

-

The model card highlights the paper's fixed-output, fixed-feature, and task-form findings without requiring manuscript source files.

-
- ---- - -## Visual Tour - -

- Release contents -

- -**Release contents.** The Hub repository is organized around weights, model code, source-data notes, compact artifacts, and final-paper visual previews. - -

- Head-selection regret -

- -**Head-selection regret.** Choosing a lightweight head by a ranking metric can lose decision performance under the same frozen features. - -

- Supporting task rank map -

- -**Supporting-task rank map.** Backbone ordering changes across burned area, analog retrieval, smoke PM2.5, and extreme heat task contracts. - -

- Primary rank changes -

- -**Primary-task rank changes.** Primary-task rankings change under fixed matching-rule and task-form choices. - ---- - -## At A Glance - -| Signal | Release value | +| Path | Purpose | |---|---| -| Core task | 12-hour gridded wildfire occupancy prediction | -| Released checkpoint region/grid | California regional grid, 5 km, EPSG:5070 | -| Nationwide path | CONUS cache and training templates in `training/NATIONWIDE_RETRAINING.md` | -| Input channels | 16 channels: weather fields, validity masks, static fuel/canopy/exposure layers | -| Architecture | Compact U-Net with occupancy and auxiliary spatial-support heads | -| Training split | June-August 2024 train, September 2024 validation, October 2024 test | -| Released checkpoints | 5 seeded PyTorch checkpoints | -| Data access path | Source links and roles documented in `data_sources/` | -| Training/data loader path | Original cache builder and PyTorch loader in `training/` | -| Spatial serving path | Custom polygon-granularity aggregation in `spatial_serving/` | -| Paper artifact path | Compiled PDF in `paper/`; manuscript source managed in the authors' paper workspace | - -### Input Channel Contract - -The pretrained checkpoints expect a fixed 16-channel tensor in `[channel, y, x]` -order. Channel order matters. -Inputs use native source units with no mean/std normalization, min/max scaling, -or unit conversion. Invalid or missing values are zero-filled, with validity -masks included as channels 10 and 11. - -| Channels | Names | Source | -|---:|---|---| -| 0-9 | `t2m`, `d2m`, `u10`, `v10`, `cape`, `sp`, `blh`, `vis`, `prate`, `tp` | NOAA HRRR | -| 10 | `firewx_valid` | Cache-level dynamic input presence mask; 1.0 everywhere in this release | -| 11 | `static_valid` | Fraction of static layers valid after reprojection | -| 12-15 | `fuel_fbfm40`, `canopy_cover`, `housing_density`, `population` | LANDFIRE, WRC housing density, LandScan | - -FIRMS provides the occupancy target, not an input channel. WFIGS and MTBS are -used for supporting event-level tasks and are not part of the 16-channel -pretrained occupancy input. Full channel-level details are in -`data_sources/DATA_SOURCES.md`; the same contract is available as -`models/wildfire_fm/input_channels.json`. -The `cape` channel is the HRRR surface instantaneous CAPE field selected with -`typeOfLevel=surface` and `stepType=instant`. -For validity channels, `firewx_valid` is 1.0 everywhere in the released -California regional cache and is not a per-variable HRRR missing-data mask; -`static_valid` is the fraction of the four static layers valid after -reprojection at each grid cell. - ---- - -## Hugging Face Release - -This Hub repository is a model release with paper-aligned reproducibility artifacts. - -| Folder | Purpose | Example | -|---|---|---| -| `models/wildfire_fm/` | Model code, manifests, checkpoint metadata | `models/wildfire_fm/modeling_unet.py` | -| `models/wildfire_fm/input_channels.json` | Machine-readable 16-channel input contract | `models/wildfire_fm/input_channels.json` | -| `models/wildfire_fm/checkpoints/` | Five seeded PyTorch checkpoints | `seed_1/best_firms_prauc.pt` | -| `training/` | Original cache builder, data loader, tile sampler, and training loop | `train_cold_tiled_mainline.py` | -| `training/NATIONWIDE_RETRAINING.md` | CONUS retraining recipe and data coverage notes | `stage1_cache_conus_hrrr_us_5km_l12_template.json` | -| `spatial_serving/` | Grid-to-polygon serving adapter for custom spatial granularities | `grid_to_polygons.py` | -| `data_downloader/hrrr_downloader.py` | Direct NOAA HRRR public archive downloader | `hrrr_download_manifest.json` | -| `docs/hugh_handoff_status.md` | Integration answers for channel order, masks, normalization, HRRR, and serving boundary | handoff status | -| `paper/` | Compiled paper PDF | `wildfire_fm_evaluation_contracts.pdf` | -| `assets/` | Hub-page visuals and final-paper previews | `selection_regret_final.png` | -| `paper_outputs/figures/` | Selected final-paper figure PDFs | `fig_rank_heatmap1.pdf` | -| `artifacts/results/` | Sanitized compact numeric summaries | `selection_regret_scope_sweep_20260505.csv` | -| `data_sources/` | Public data-source roles and access notes | `DATA_SOURCES.md` | -| `experiments/` | Sanitized raw-rerun references and Slurm template | `slurm/submit_template.sbatch` | - -Full local reruns require users to prepare the provider-hosted raw data and comparator assets described in `data_sources/`. - ---- +| `firewxfm/` | Model definition and tiled CONUS inference code. | +| `models/checkpoints/` | Released PyTorch checkpoints. | +| `models/metadata/` | Input-channel contract, normalization statistics, and run metadata. | +| `training/` | Cache-building, training, and metric utilities. | +| `training/configs/` | Path-free templates for CONUS and multi-year reruns. | +| `scripts/` | Data preparation, release checks, live-stack builder, seam audit, and smoke tests. | +| `examples/final_prediction/` | Small final probability/preview example for inspection. | +| `data_sources/` | Source-data provenance and access notes. | +| `paper/iclr_source/` | ICLR manuscript LaTeX source. | +| `technical_report/source/` | Technical report LaTeX source. | +| `artifacts/results/` | Compact public result summaries without internal filesystem paths. | ## Quick Start -Clone the Hub repository or download the files you need: - ```bash -git clone https://huggingface.co/RAI-Lab/Wildfire-FM -cd Wildfire-FM +python3 -m venv .venv +source .venv/bin/activate +pip install -r requirements.txt +python3 scripts/check_release_files.py +python3 scripts/smoke_inference.py --device cpu ``` -Load a seeded checkpoint: +The smoke test uses a synthetic tensor. Full scientific reproduction requires +the public source datasets listed in `data_sources/DATA_SOURCES.md`. -```python -import torch -from models.wildfire_fm.modeling_unet import UNetSmallFlex +## Model Input Contract -model = UNetSmallFlex( - in_ch=16, - base=32, - dropout=0.1, - norm_type="group", - norm_groups=8, - use_aux_spatial_head=True, -) -checkpoint = torch.load( - "models/wildfire_fm/checkpoints/seed_1/best_firms_prauc.pt", - map_location="cpu", -) -state = checkpoint.get("model", checkpoint) -model.load_state_dict(state) -model.eval() -``` +The model expects a channel-first tensor with shape `[16,H,W]`. -Verify the public release artifacts: - -```bash -python3 scripts/reproduce_paper_outputs.py -``` - -The checkpoint expects the same fixed-order 16-channel gridded input described -above and in `data_sources/DATA_SOURCES.md`. - ---- - -## Task-Contract Snapshot - -The card below reports the best final-paper mean for each displayed task contract, with the winning backbone named in the final column. - -| Task contract | Best final-paper mean | Winner | -|---|---:|---| -| Occupancy union F1 | `60.1506 ± 7.5865` percent | ClimaX | -| Fire-spread spatial F1 | `80.9700 ± 2.0200` percent | WildFIRE-FM | -| Final burned-area log-RMSE | `1.1657 ± 0.0126`, lower is better | WildFIRE-FM | -| Analog retrieval nDCG@10 | `0.5099 ± 0.0336` | WildFIRE-FM | -| Smoke PM2.5 RMSE | `4.4403 ± 0.0488`, lower is better | AlphaEarth | -| Extreme-heat RMSE-C | `0.2179 ± 0.0043`, lower is better | WildFIRE-FM | - -The release provides sanitized CSV/JSON summaries used to audit displayed values. +| Channels | Source | Role | +|---:|---|---| +| 0-9 | NOAA HRRR | Dynamic weather fields. | +| 10-11 | Input builder | Dynamic and static validity masks. | +| 12 | LANDFIRE | Fire-behavior fuel model. | +| 13 | LANDFIRE | Canopy cover. | +| 14-15 | WRC / LandScan | Housing density and population; zeroed in the no-exposure serving run. | ---- +See `models/metadata/input_channels.json` and +`models/metadata/input_normalization_stats.json` for the exact machine-readable +contract. -## Data Sources +## Inference -The study uses public or provider-hosted resources. Source roles and access paths are documented in `data_sources/DATA_SOURCES.md`. +Run inference from a prebuilt 16-channel stack: -- NOAA HRRR fields for regional weather inputs. -- NASA FIRMS active-fire detections for occupancy supervision. -- LANDFIRE fuel and canopy layers for static landscape context. -- Wildfire Risk to Communities housing density and LandScan population for exposure context. -- WFIGS and MTBS event-level resources for burned-area and analog tasks. -- External Earth-FM and backbone assets for comparator features. +```bash +python3 -m firewxfm.serve_conus \ + --input-npy /path/to/input_stack_16chw.npy \ + --checkpoint models/checkpoints/firewxfm_2024_staticfix_region_balanced_bce_seed42.pt \ + --normalization-stats models/metadata/input_normalization_stats.json \ + --output-npy outputs/firewxfm_probability.npy +``` ---- +For full-map serving, use overlap-window inference with phase averaging. The +default settings are documented in `docs/serving_notes.md`. -## Repository Layout +## Reproducibility -```text -WildFIRE-FM/ - README.md - LICENSE - requirements.txt - models/wildfire_fm/ - modeling_unet.py - checkpoint_manifest.json - checkpoints/seed_*/best_firms_prauc.pt - training/ - build_phase1_cache_regional_hrrr.py - train_cold_tiled_mainline.py - configs/ - spatial_serving/ - grid_to_polygons.py - data_downloader/ - hrrr_downloader.py - paper/ - wildfire_fm_evaluation_contracts.pdf - assets/ - wildfire_fm_model_card.svg - release_contents.svg - selection_regret_final.png - supporting_rank_map_final.png - primary_rank_change_final.png - paper_outputs/figures/ - matching.pdf - fig_task_contract_tiles.pdf - fig_selection_regret_scatter.pdf - fig_rank_heatmap1.pdf - artifacts/ - manifests/ - results/ - data_sources/ - DATA_SOURCES.md - experiments/ - raw_reference/ - slurm/ - scripts/ - audit_release.py - reproduce_paper_outputs.py -``` +Use `REPRODUCIBILITY.md` for the exact verification path: ---- +- artifact presence check, +- checkpoint smoke inference, +- ICLR source compilation, +- technical report source compilation, +- real-data cache rebuild and retraining commands. -## How To Cite +Raw source data are not redistributed. This package documents where each source +comes from and provides path-free templates for local reproduction. -If you use WildFIRE-FM, the released checkpoints, the fixed-contract evaluation artifacts, or the paper-aligned scripts, please cite: +## Citation ```bibtex -@misc{wildfire_fm_evaluation_contracts_2026, - title = {Does Your Wildfire Prediction Model Actually Work, or Just Score Well?}, - author = {Yangshuang Xu and Yuyang Dai and Liling Chang and Qi Wang and Yushun Dong}, +@misc{firewxfm2026, + title = {FireWx-FM: A Wildfire Foundation Model Technical Report}, + author = {Xu, Yangshuang and Dai, Yuyang and Chang, Liling and Wang, Qi and Dong, Yushun}, year = {2026}, - note = {WildFIRE-FM model and fixed-contract wildfire evaluation artifacts} + note = {FireWx-FM code, checkpoints, and reproducibility package} } ``` - -The citation will be updated with arXiv metadata after the preprint is public. - ---- - -## News - -- **2026-05-18** - The model card was reorganized with a larger navigation panel and final-paper visual previews. -- **2026-05-13** - The Hub release added the compiled paper PDF without checklist pages. -- **2026-05-13** - Five seeded WildFIRE-FM checkpoints were uploaded with SHA-256 metadata. - ---- - -## Scope - -WildFIRE-FM is scoped to the paper's California regional grid and fixed-contract wildfire transfer comparisons. Use in a new region or grid should start with local preprocessing, validation, and contract-specific evaluation. The release is designed for research inspection, artifact-level checking, and model-loading experiments rather than operational alerting. - ---- - -## License - -This release is provided under the MIT license. External data and comparator assets remain governed by their original providers and terms of use. - ---- - -WildFIRE-FM is released to make wildfire Earth-FM transfer comparisons easier to inspect, reproduce at the artifact level, and evaluate under explicit contracts. diff --git a/REPRODUCIBILITY.md b/REPRODUCIBILITY.md new file mode 100644 index 0000000000000000000000000000000000000000..7447b40587fc0ff318b9a5104109c258c51f97dc --- /dev/null +++ b/REPRODUCIBILITY.md @@ -0,0 +1,102 @@ +# Reproducibility + +This release is designed to make the FireWx-FM technical report and ICLR +manuscript inspectable and reproducible from documented ingredients. + +## What Is Included + +- Model and inference code in `firewxfm/`. +- Training, cache-building, and metric utilities in `training/`. +- Public-data preparation and audit scripts in `scripts/`. +- Two released PyTorch checkpoints in `models/checkpoints/`. +- Input-channel metadata, normalization statistics, and run summaries in + `models/metadata/`. +- A small final prediction example in `examples/final_prediction/`. +- ICLR manuscript source in `paper/iclr_source/`. +- Technical report source in `technical_report/source/`. +- Source-data notes in `data_sources/DATA_SOURCES.md`. + +## What Is Not Included + +Raw HRRR, FIRMS, LANDFIRE, Wildfire Risk to Communities, LandScan, WFIGS, and +MTBS data are not redistributed. Users must download those data from the +original providers and follow their licenses. + +## Quick Verification + +Install dependencies: + +```bash +python3 -m venv .venv +source .venv/bin/activate +pip install -r requirements.txt +``` + +Check that required artifacts are present: + +```bash +python3 scripts/check_release_files.py +``` + +Verify checkpoint/model compatibility with a synthetic tensor: + +```bash +python3 scripts/smoke_inference.py \ + --checkpoint models/checkpoints/firewxfm_2024_staticfix_region_balanced_bce_seed42.pt \ + --device cpu +``` + +Compile the technical report source: + +```bash +cd technical_report/source +tectonic main.tex +``` + +Compile the ICLR manuscript source: + +```bash +cd paper/iclr_source +tectonic main.tex +``` + +## Inference From A Real 16-Channel Stack + +After constructing a `[16,H,W]` stack that follows +`models/metadata/input_channels.json`, run: + +```bash +python3 -m firewxfm.serve_conus \ + --input-npy /path/to/input_stack_16chw.npy \ + --checkpoint models/checkpoints/firewxfm_2024_staticfix_region_balanced_bce_seed42.pt \ + --normalization-stats models/metadata/input_normalization_stats.json \ + --output-npy outputs/firewxfm_probability.npy +``` + +The output is an `[H,W]` probability map for 12-hour active-fire occupancy. + +## Rebuilding Training Caches + +Use the template configs under `training/configs/` after replacing placeholder +paths with local source-data paths: + +```bash +python3 scripts/prepare_firms_target_hour_csvs.py \ + --daily-dir /path/to/firms_daily_csv \ + --output-dir /path/to/firms_target_hour_csv \ + --config training/configs/stage1_cache_conus_multiyear_template.json + +python3 scripts/audit_firewxfm_phase0.py \ + --config training/configs/stage1_cache_conus_multiyear_template.json \ + --output artifacts/manifests/phase0_data_split_audit.json + +python3 training/build_phase1_cache_regional_hrrr.py \ + --config training/configs/stage1_cache_conus_multiyear_template.json + +python3 training/train_cold_tiled_mainline.py \ + --config training/configs/train_firewxfm_conus_multiyear_template.json \ + --run-name firewxfm_conus_multiyear_noexposure_seed42 +``` + +Large cache building and training should be run on compute nodes, not login +nodes. diff --git a/artifacts/manifests/paper_outputs.sha256 b/artifacts/manifests/paper_outputs.sha256 deleted file mode 100644 index 8c0cf0ab9a60e74dc19ec2d35d0e65741f65cc1e..0000000000000000000000000000000000000000 --- a/artifacts/manifests/paper_outputs.sha256 +++ /dev/null @@ -1,15 +0,0 @@ -6db4f4aff90da8709edff97e4782aee9b4c5e8feefec7c7431a4ec8787cfe57c assets/overview_final.png -ca11c75c03078a9be26421b527ab5a49f5fc43ce8e5edd7da14af120a247b67c assets/primary_rank_change_final.png -1f992d7245b55b8674c00753efbd0e25f4e3be658e736b353772624dea2b7a8f assets/release_contents.svg -5552fb6cca6a0a683592e724b4bd562f923cf99c04e2abdb846546b1d67aecc4 assets/selection_regret_final.png -34807e65ca71365a26a3b74cae70e6b40ae6f2151110e12c53e0efa9f8b726aa assets/supporting_rank_map_final.png -7adb6a949bce3203b6cb10a458e115c60ba0b3efdb10f2ca1a1fd508924bc2ac assets/wildfire_fm_model_card.svg -95596f05829639ecb3041721fe3c871caacc49f30c831d35f2c503b1073b498d paper/wildfire_fm_evaluation_contracts.pdf -b369d13e0419fa8272ccdc994b6642f3b141248a879c030218e387c583537eb2 paper_outputs/figures/fig_fireprone_contract_progression_compact.pdf -e3110c70c3cf8ecb8671163a401a155920266e3f907f9c6baf08e27ec6e6c410 paper_outputs/figures/fig_primary_rank_change_map.pdf -4e5b791ba4d136f722bd75a61097203836819ce9411def1caac4cc1e6d881275 paper_outputs/figures/fig_rank_heatmap1.pdf -fabb8b55aac901199cc03773741a26685becffd074f52568c93bee517c2c42c0 paper_outputs/figures/fig_selection_regret_scatter.pdf -bc4d35ad9cb4c1f9ba8f31c7c340d9684c9dd2d55f5a2e60604a2b58b90cbe40 paper_outputs/figures/fig_task_contract_tiles.pdf -c382f5d69f25cc2f5db174601a33d0fd0928b44910a2a4b1c131954bd42113d9 paper_outputs/figures/fig_task_rank_map.pdf -015ab951b0af5c130e4894092a5dd0bb0fd62e710467163a9df8246d8cf369f4 paper_outputs/figures/matching.pdf -7dca6ae4a9b179693802f47d24dd66734c0f332b372a7976832a0d429333b755 paper_outputs/figures/overview_wildfire.pdf diff --git a/artifacts/manifests/paper_outputs.yml b/artifacts/manifests/paper_outputs.yml deleted file mode 100644 index 1a7596293669fdaff2dff4b6b02597e49e3b8cd7..0000000000000000000000000000000000000000 --- a/artifacts/manifests/paper_outputs.yml +++ /dev/null @@ -1,32 +0,0 @@ -# Auto-generated public-output manifest for the Hugging Face release. -outputs: - - path: assets/overview_final.png - sha256: 6db4f4aff90da8709edff97e4782aee9b4c5e8feefec7c7431a4ec8787cfe57c - - path: assets/primary_rank_change_final.png - sha256: ca11c75c03078a9be26421b527ab5a49f5fc43ce8e5edd7da14af120a247b67c - - path: assets/release_contents.svg - sha256: 1f992d7245b55b8674c00753efbd0e25f4e3be658e736b353772624dea2b7a8f - - path: assets/selection_regret_final.png - sha256: 5552fb6cca6a0a683592e724b4bd562f923cf99c04e2abdb846546b1d67aecc4 - - path: assets/supporting_rank_map_final.png - sha256: 34807e65ca71365a26a3b74cae70e6b40ae6f2151110e12c53e0efa9f8b726aa - - path: assets/wildfire_fm_model_card.svg - sha256: 7adb6a949bce3203b6cb10a458e115c60ba0b3efdb10f2ca1a1fd508924bc2ac - - path: paper/wildfire_fm_evaluation_contracts.pdf - sha256: 95596f05829639ecb3041721fe3c871caacc49f30c831d35f2c503b1073b498d - - path: paper_outputs/figures/fig_fireprone_contract_progression_compact.pdf - sha256: b369d13e0419fa8272ccdc994b6642f3b141248a879c030218e387c583537eb2 - - path: paper_outputs/figures/fig_primary_rank_change_map.pdf - sha256: e3110c70c3cf8ecb8671163a401a155920266e3f907f9c6baf08e27ec6e6c410 - - path: paper_outputs/figures/fig_rank_heatmap1.pdf - sha256: 4e5b791ba4d136f722bd75a61097203836819ce9411def1caac4cc1e6d881275 - - path: paper_outputs/figures/fig_selection_regret_scatter.pdf - sha256: fabb8b55aac901199cc03773741a26685becffd074f52568c93bee517c2c42c0 - - path: paper_outputs/figures/fig_task_contract_tiles.pdf - sha256: bc4d35ad9cb4c1f9ba8f31c7c340d9684c9dd2d55f5a2e60604a2b58b90cbe40 - - path: paper_outputs/figures/fig_task_rank_map.pdf - sha256: c382f5d69f25cc2f5db174601a33d0fd0928b44910a2a4b1c131954bd42113d9 - - path: paper_outputs/figures/matching.pdf - sha256: 015ab951b0af5c130e4894092a5dd0bb0fd62e710467163a9df8246d8cf369f4 - - path: paper_outputs/figures/overview_wildfire.pdf - sha256: 7dca6ae4a9b179693802f47d24dd66734c0f332b372a7976832a0d429333b755 diff --git a/artifacts/results/downstream_summary.md b/artifacts/results/downstream_summary.md new file mode 100644 index 0000000000000000000000000000000000000000..05b2aabf1f2570b80176c1feb65975c63fccd7bd --- /dev/null +++ b/artifacts/results/downstream_summary.md @@ -0,0 +1,26 @@ +# Forced Mean/Std Gap-Fill Summary + +## table2_headcontrol + + +## table3_occupancy + + +## table3_spread + +- Prithvi-WxC: strict_f1 20.72 +/- 3.80 (n=5); spatial_f1 57.32 +/- 2.14 (n=5); ap 5.04 +/- 0.40 (n=5) +- Reference: strict_f1 20.32 +/- 1.15 (n=5); spatial_f1 68.58 +/- 1.98 (n=5); ap 30.44 +/- 1.81 (n=5) + +## table4_final_area + +- Reference: log_rmse 1.17 +/- 0.01 (n=5); log_mae 1.05 +/- 0.01 (n=5); log_spearman 0.59 +/- 0.03 (n=5) + +## table4_analog + +- reference_strict: ndcg_at_10 0.52 +/- 0.00 (n=5); log_rmse 1.18 +/- 0.00 (n=5); log_mae 0.97 +/- 0.00 (n=5) + +## table4_smoke + + +## table4_heat + diff --git a/artifacts/results/fireprone_contract_progression_summary.json b/artifacts/results/fireprone_contract_progression_summary.json deleted file mode 100644 index 2099b6eadc0cc63bf2ddbd793ffc683434dc98e8..0000000000000000000000000000000000000000 --- a/artifacts/results/fireprone_contract_progression_summary.json +++ /dev/null @@ -1,16543 +0,0 @@ -{ - "run_roots": [ - "", - "" - ], - "repair_run_roots": [], - "repair_sources": [ - "", - "", - "" - ], - "repair_scopes": [ - "train_fire_top02pct" - ], - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "threshold_selector": "strict_f1", - "threshold_min_ppr": 1e-06, - "threshold_max_ppr": 0.999999, - "repair_threshold_selector": "strict_f1_nondegenerate", - "repair_threshold_min_ppr": 1e-06, - "repair_threshold_max_ppr": 0.999999, - "min_runs": 5, - "missing_or_incomplete": [], - "seed_variation_issues": [], - "nondegenerate_prediction_issues": [], - "rows": [ - { - "model_tag": "alphaearth", - "label": "AlphaEarth", - "seed": 1, - "scope": "full_domain", - "threshold_key": "0.0050", - "threshold_selector": "strict_f1", - "threshold": 0.005, - "strict_f1": 0.01841620626151013, - "tolerance_f1": 0.3233671389763082, - "union_f1": 0.4075802286978974, - "difference": 0.38916402243638726, - "predicted_positive_rate": 0.0006305504019789734, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "alphaearth", - "label": "AlphaEarth", - "seed": 1, - "scope": "train_fire_top05pct", - "threshold_key": "0.0050", - "threshold_selector": "strict_f1", - "threshold": 0.005, - "strict_f1": 0.05801376597836775, - "tolerance_f1": 0.45453465315678554, - "union_f1": 0.5487719544302176, - "difference": 0.4907581884518499, - "predicted_positive_rate": 0.0024215889977243494, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "alphaearth", - "label": "AlphaEarth", - "seed": 1, - "scope": "train_fire_top10pct", - "threshold_key": "0.0050", - "threshold_selector": "strict_f1", - "threshold": 0.005, - "strict_f1": 0.05730937348227295, - "tolerance_f1": 0.4503813713224908, - "union_f1": 0.5437059200683907, - "difference": 0.48639654658611775, - "predicted_positive_rate": 0.0012293459978232908, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "alphaearth", - "label": "AlphaEarth", - "seed": 1, - "scope": "train_fire_top20pct", - "threshold_key": "0.0050", - "threshold_selector": "strict_f1", - "threshold": 0.005, - "strict_f1": 0.0544280442804428, - "tolerance_f1": 0.42554776574916503, - "union_f1": 0.5120841886230071, - "difference": 0.4576561443425643, - "predicted_positive_rate": 0.0006233766233766234, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "alphaearth", - "label": "AlphaEarth", - "seed": 123, - "scope": "full_domain", - "threshold_key": "0.0050", - "threshold_selector": "strict_f1", - "threshold": 0.005, - "strict_f1": 0.015582982831138383, - "tolerance_f1": 0.3412019506417814, - "union_f1": 0.4879949976282558, - "difference": 0.4724120147971174, - "predicted_positive_rate": 0.0015512677798392085, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "alphaearth", - "label": "AlphaEarth", - "seed": 123, - "scope": "train_fire_top05pct", - "threshold_key": "0.0050", - "threshold_selector": "strict_f1", - "threshold": 0.005, - "strict_f1": 0.06425702811244981, - "tolerance_f1": 0.44689699140777084, - "union_f1": 0.6362946372146648, - "difference": 0.572037609102215, - "predicted_positive_rate": 0.005397249431087365, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "alphaearth", - "label": "AlphaEarth", - "seed": 123, - "scope": "train_fire_top10pct", - "threshold_key": "0.0050", - "threshold_selector": "strict_f1", - "threshold": 0.005, - "strict_f1": 0.06358911647814125, - "tolerance_f1": 0.4435187312985661, - "union_f1": 0.6310773342718137, - "difference": 0.5674882177936724, - "predicted_positive_rate": 0.002728307113881468, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "alphaearth", - "label": "AlphaEarth", - "seed": 123, - "scope": "train_fire_top20pct", - "threshold_key": "0.0050", - "threshold_selector": "strict_f1", - "threshold": 0.005, - "strict_f1": 0.06170275882527441, - "tolerance_f1": 0.4259185655133553, - "union_f1": 0.5992872540844005, - "difference": 0.5375844952591261, - "predicted_positive_rate": 0.0013673469387755102, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "alphaearth", - "label": "AlphaEarth", - "seed": 42, - "scope": "full_domain", - "threshold_key": "0.0050", - "threshold_selector": "strict_f1", - "threshold": 0.005, - "strict_f1": 0.024598770061496925, - "tolerance_f1": 0.3224566312114399, - "union_f1": 0.3927541342199395, - "difference": 0.3681553641584426, - "predicted_positive_rate": 0.0005820655534941249, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "alphaearth", - "label": "AlphaEarth", - "seed": 42, - "scope": "train_fire_top05pct", - "threshold_key": "0.0050", - "threshold_selector": "strict_f1", - "threshold": 0.005, - "strict_f1": 0.07774936061381074, - "tolerance_f1": 0.46103410735073025, - "union_f1": 0.5064003599986154, - "difference": 0.42865099938480467, - "predicted_positive_rate": 0.002226179875333927, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "alphaearth", - "label": "AlphaEarth", - "seed": 42, - "scope": "train_fire_top10pct", - "threshold_key": "0.0050", - "threshold_selector": "strict_f1", - "threshold": 0.005, - "strict_f1": 0.07711821410451548, - "tolerance_f1": 0.4570756272994282, - "union_f1": 0.5020582486954701, - "difference": 0.42494003459095464, - "predicted_positive_rate": 0.00112051053725141, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "alphaearth", - "label": "AlphaEarth", - "seed": 42, - "scope": "train_fire_top20pct", - "threshold_key": "0.0050", - "threshold_selector": "strict_f1", - "threshold": 0.005, - "strict_f1": 0.07357212003872217, - "tolerance_f1": 0.42986611450839224, - "union_f1": 0.4723374235406742, - "difference": 0.39876530350195205, - "predicted_positive_rate": 0.0005602968460111318, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "alphaearth", - "label": "AlphaEarth", - "seed": 7, - "scope": "full_domain", - "threshold_key": "0.0200", - "threshold_selector": "strict_f1", - "threshold": 0.02, - "strict_f1": 0.018581463856786766, - "tolerance_f1": 0.19137568834619276, - "union_f1": 0.21572734820321318, - "difference": 0.1971458843464264, - "predicted_positive_rate": 0.00030327767470624613, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "alphaearth", - "label": "AlphaEarth", - "seed": 7, - "scope": "train_fire_top05pct", - "threshold_key": "0.0100", - "threshold_selector": "strict_f1", - "threshold": 0.01, - "strict_f1": 0.06853879105188007, - "tolerance_f1": 0.348489928787613, - "union_f1": 0.39499846060642174, - "difference": 0.32645966955454164, - "predicted_positive_rate": 0.0025873157217769865, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "alphaearth", - "label": "AlphaEarth", - "seed": 7, - "scope": "train_fire_top10pct", - "threshold_key": "0.0100", - "threshold_selector": "strict_f1", - "threshold": 0.01, - "strict_f1": 0.05725646123260437, - "tolerance_f1": 0.31417824074074074, - "union_f1": 0.3547275267815556, - "difference": 0.29747106554895125, - "predicted_positive_rate": 0.001793311566241219, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "alphaearth", - "label": "AlphaEarth", - "seed": 7, - "scope": "train_fire_top20pct", - "threshold_key": "0.0200", - "threshold_selector": "strict_f1", - "threshold": 0.02, - "strict_f1": 0.047398843930635835, - "tolerance_f1": 0.25536195152859176, - "union_f1": 0.2676781973617166, - "difference": 0.22027935343108077, - "predicted_positive_rate": 0.00035250463821892396, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "alphaearth", - "label": "AlphaEarth", - "seed": 99, - "scope": "full_domain", - "threshold_key": "0.0100", - "threshold_selector": "strict_f1", - "threshold": 0.01, - "strict_f1": 0.02585193889541716, - "tolerance_f1": 0.29397699288019324, - "union_f1": 0.36737185673853523, - "difference": 0.3415199178431181, - "predicted_positive_rate": 0.00038899196042053183, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "alphaearth", - "label": "AlphaEarth", - "seed": 99, - "scope": "train_fire_top05pct", - "threshold_key": "0.0100", - "threshold_selector": "strict_f1", - "threshold": 0.01, - "strict_f1": 0.07710843373493977, - "tolerance_f1": 0.4329934518241347, - "union_f1": 0.5007776336496106, - "difference": 0.42366919991467084, - "predicted_positive_rate": 0.001496487582863362, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "alphaearth", - "label": "AlphaEarth", - "seed": 99, - "scope": "train_fire_top10pct", - "threshold_key": "0.0100", - "threshold_selector": "strict_f1", - "threshold": 0.01, - "strict_f1": 0.07655502392344497, - "tolerance_f1": 0.4297532170909699, - "union_f1": 0.4969907507243763, - "difference": 0.42043572680093133, - "predicted_positive_rate": 0.0007507173246264965, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "alphaearth", - "label": "AlphaEarth", - "seed": 99, - "scope": "train_fire_top20pct", - "threshold_key": "0.0100", - "threshold_selector": "strict_f1", - "threshold": 0.01, - "strict_f1": 0.0724391624221845, - "tolerance_f1": 0.40493051697817745, - "union_f1": 0.4677768948219149, - "difference": 0.3953377323997304, - "predicted_positive_rate": 0.0003753865182436611, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "aurora", - "label": "Aurora", - "seed": 1, - "scope": "full_domain", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.0004925501476206198, - "tolerance_f1": 0.07032155974511028, - "union_f1": 0.19601398468283832, - "difference": 0.1955214345352177, - "predicted_positive_rate": 0.9846233766233766, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "aurora", - "label": "Aurora", - "seed": 1, - "scope": "train_fire_top05pct", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.005636060100166944, - "tolerance_f1": 0.15977956000729543, - "union_f1": 0.39407737525514647, - "difference": 0.38844131515497954, - "predicted_positive_rate": 0.9234194122885129, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "aurora", - "label": "Aurora", - "seed": 1, - "scope": "train_fire_top10pct", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.002747305898968793, - "tolerance_f1": 0.08713204456539597, - "union_f1": 0.2309636203280157, - "difference": 0.2282163144290469, - "predicted_positive_rate": 0.9575541703769664, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "aurora", - "label": "Aurora", - "seed": 1, - "scope": "train_fire_top20pct", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.001471035812113219, - "tolerance_f1": 0.06246138809231819, - "union_f1": 0.17013954201238674, - "difference": 0.16866850620027352, - "predicted_positive_rate": 0.9746196660482375, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "aurora", - "label": "Aurora", - "seed": 123, - "scope": "full_domain", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.0004925501476206198, - "tolerance_f1": 0.07032155974511028, - "union_f1": 0.19601398468283832, - "difference": 0.1955214345352177, - "predicted_positive_rate": 0.9846233766233766, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "aurora", - "label": "Aurora", - "seed": 123, - "scope": "train_fire_top05pct", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.005636060100166944, - "tolerance_f1": 0.15977956000729543, - "union_f1": 0.39407737525514647, - "difference": 0.38844131515497954, - "predicted_positive_rate": 0.9234194122885129, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "aurora", - "label": "Aurora", - "seed": 123, - "scope": "train_fire_top10pct", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.002747305898968793, - "tolerance_f1": 0.08713204456539597, - "union_f1": 0.2309636203280157, - "difference": 0.2282163144290469, - "predicted_positive_rate": 0.9575541703769664, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "aurora", - "label": "Aurora", - "seed": 123, - "scope": "train_fire_top20pct", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.001471035812113219, - "tolerance_f1": 0.06246138809231819, - "union_f1": 0.17013954201238674, - "difference": 0.16866850620027352, - "predicted_positive_rate": 0.9746196660482375, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "aurora", - "label": "Aurora", - "seed": 42, - "scope": "full_domain", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.0004887421693148924, - "tolerance_f1": 0.06979673814819128, - "union_f1": 0.19464572145581938, - "difference": 0.1941569792865045, - "predicted_positive_rate": 0.9922968460111318, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "aurora", - "label": "Aurora", - "seed": 42, - "scope": "train_fire_top05pct", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.00562524160545995, - "tolerance_f1": 0.15949652359184557, - "union_f1": 0.3934680413788068, - "difference": 0.38784279977334685, - "predicted_positive_rate": 0.92520035618878, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "aurora", - "label": "Aurora", - "seed": 42, - "scope": "train_fire_top10pct", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.0027388277045923585, - "tolerance_f1": 0.0868744819295806, - "union_f1": 0.23033208616831716, - "difference": 0.2275932584637248, - "predicted_positive_rate": 0.960522410210745, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "aurora", - "label": "Aurora", - "seed": 42, - "scope": "train_fire_top20pct", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.0014656828060244616, - "tolerance_f1": 0.06224100702898884, - "union_f1": 0.1695724929003409, - "difference": 0.16810681009431644, - "predicted_positive_rate": 0.9781818181818182, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "aurora", - "label": "Aurora", - "seed": 7, - "scope": "full_domain", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.0004887421693148924, - "tolerance_f1": 0.06979673814819128, - "union_f1": 0.19464572145581938, - "difference": 0.1941569792865045, - "predicted_positive_rate": 0.9922968460111318, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "aurora", - "label": "Aurora", - "seed": 7, - "scope": "train_fire_top05pct", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.00562524160545995, - "tolerance_f1": 0.15949652359184557, - "union_f1": 0.3934680413788068, - "difference": 0.38784279977334685, - "predicted_positive_rate": 0.92520035618878, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "aurora", - "label": "Aurora", - "seed": 7, - "scope": "train_fire_top10pct", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.0027388277045923585, - "tolerance_f1": 0.0868744819295806, - "union_f1": 0.23033208616831716, - "difference": 0.2275932584637248, - "predicted_positive_rate": 0.960522410210745, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "aurora", - "label": "Aurora", - "seed": 7, - "scope": "train_fire_top20pct", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.0014656828060244616, - "tolerance_f1": 0.06224100702898884, - "union_f1": 0.1695724929003409, - "difference": 0.16810681009431644, - "predicted_positive_rate": 0.9781818181818182, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "aurora", - "label": "Aurora", - "seed": 99, - "scope": "full_domain", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.0004925501476206198, - "tolerance_f1": 0.07032155974511028, - "union_f1": 0.19601398468283832, - "difference": 0.1955214345352177, - "predicted_positive_rate": 0.9846233766233766, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "aurora", - "label": "Aurora", - "seed": 99, - "scope": "train_fire_top05pct", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.005636060100166944, - "tolerance_f1": 0.15977956000729543, - "union_f1": 0.39407737525514647, - "difference": 0.38844131515497954, - "predicted_positive_rate": 0.9234194122885129, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "aurora", - "label": "Aurora", - "seed": 99, - "scope": "train_fire_top10pct", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.002747305898968793, - "tolerance_f1": 0.08713204456539597, - "union_f1": 0.2309636203280157, - "difference": 0.2282163144290469, - "predicted_positive_rate": 0.9575541703769664, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "aurora", - "label": "Aurora", - "seed": 99, - "scope": "train_fire_top20pct", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.001471035812113219, - "tolerance_f1": 0.06246138809231819, - "union_f1": 0.17013954201238674, - "difference": 0.16866850620027352, - "predicted_positive_rate": 0.9746196660482375, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "climax", - "label": "ClimaX", - "seed": 1, - "scope": "full_domain", - "threshold_key": "0.0000", - "threshold_selector": "strict_f1", - "threshold": 2e-05, - "strict_f1": 0.00048497822606044473, - "tolerance_f1": 0.06928162590128548, - "union_f1": 0.19330497146159284, - "difference": 0.1928199932355324, - "predicted_positive_rate": 1.0, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "climax", - "label": "ClimaX", - "seed": 1, - "scope": "train_fire_top05pct", - "threshold_key": "0.0000", - "threshold_selector": "strict_f1", - "threshold": 2e-05, - "strict_f1": 0.005205570700778368, - "tolerance_f1": 0.14852385215852132, - "union_f1": 0.36969737295773625, - "difference": 0.3644918022569579, - "predicted_positive_rate": 1.0, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "climax", - "label": "ClimaX", - "seed": 1, - "scope": "train_fire_top10pct", - "threshold_key": "0.0000", - "threshold_selector": "strict_f1", - "threshold": 2e-05, - "strict_f1": 0.0026308476146364055, - "tolerance_f1": 0.08362683149966108, - "union_f1": 0.22238246829765904, - "difference": 0.21975162068302262, - "predicted_positive_rate": 1.0, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "climax", - "label": "ClimaX", - "seed": 1, - "scope": "train_fire_top20pct", - "threshold_key": "0.0000", - "threshold_selector": "strict_f1", - "threshold": 2e-05, - "strict_f1": 0.0014337271963217482, - "tolerance_f1": 0.06094415351278495, - "union_f1": 0.16625085053300068, - "difference": 0.16481712333667892, - "predicted_positive_rate": 1.0, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "climax", - "label": "ClimaX", - "seed": 123, - "scope": "full_domain", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.0004909510781395833, - "tolerance_f1": 0.07010120797866884, - "union_f1": 0.19543966728693182, - "difference": 0.19494871620879223, - "predicted_positive_rate": 0.9878311688311688, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "climax", - "label": "ClimaX", - "seed": 123, - "scope": "train_fire_top05pct", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.0056274920321647175, - "tolerance_f1": 0.15955540678900526, - "union_f1": 0.39359483049372823, - "difference": 0.3879673384615635, - "predicted_positive_rate": 0.9248293262095577, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "climax", - "label": "ClimaX", - "seed": 123, - "scope": "train_fire_top10pct", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.002740589677112216, - "tolerance_f1": 0.08692801516378341, - "union_f1": 0.23046337081520796, - "difference": 0.22772278113809574, - "predicted_positive_rate": 0.9599040269120411, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "climax", - "label": "ClimaX", - "seed": 123, - "scope": "train_fire_top20pct", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.001466794800465328, - "tolerance_f1": 0.06228679141188911, - "union_f1": 0.1696903162151458, - "difference": 0.16822352141468047, - "predicted_positive_rate": 0.9774397031539889, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "climax", - "label": "ClimaX", - "seed": 42, - "scope": "full_domain", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.0004887472853776956, - "tolerance_f1": 0.06980139080211295, - "union_f1": 0.19466137394685679, - "difference": 0.19417262666147908, - "predicted_positive_rate": 0.9922864564007421, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "climax", - "label": "ClimaX", - "seed": 42, - "scope": "train_fire_top05pct", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.0052190130376387305, - "tolerance_f1": 0.14887976367271333, - "union_f1": 0.3704772353928872, - "difference": 0.36525822235524846, - "predicted_positive_rate": 0.9974176313446127, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "climax", - "label": "ClimaX", - "seed": 42, - "scope": "train_fire_top10pct", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.0026397290373912042, - "tolerance_f1": 0.08389766026250713, - "union_f1": 0.22305036009296705, - "difference": 0.22041063105557585, - "predicted_positive_rate": 0.9966310477886613, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "climax", - "label": "ClimaX", - "seed": 42, - "scope": "train_fire_top20pct", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.0014391381051513698, - "tolerance_f1": 0.061167283891925196, - "union_f1": 0.16682636175147197, - "difference": 0.1653872236463206, - "predicted_positive_rate": 0.9962374768089054, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "climax", - "label": "ClimaX", - "seed": 7, - "scope": "full_domain", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.00048500701355756607, - "tolerance_f1": 0.06928559683931526, - "union_f1": 0.19331533916065632, - "difference": 0.19283033214709877, - "predicted_positive_rate": 0.9999406307977736, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "climax", - "label": "ClimaX", - "seed": 7, - "scope": "train_fire_top05pct", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.005208654759994569, - "tolerance_f1": 0.14860552018347153, - "union_f1": 0.36987636090753395, - "difference": 0.3646677061475394, - "predicted_positive_rate": 0.9994063520332442, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "climax", - "label": "ClimaX", - "seed": 7, - "scope": "train_fire_top10pct", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.0026316277173409446, - "tolerance_f1": 0.08365062280406448, - "union_f1": 0.22244115266880124, - "difference": 0.2198095249514603, - "predicted_positive_rate": 0.9997031760166222, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "climax", - "label": "ClimaX", - "seed": 7, - "scope": "train_fire_top20pct", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.0014339398734177216, - "tolerance_f1": 0.06095292465378015, - "union_f1": 0.1662734779058162, - "difference": 0.16483953803239848, - "predicted_positive_rate": 0.9998515769944342, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "climax", - "label": "ClimaX", - "seed": 99, - "scope": "full_domain", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.0004925501476206198, - "tolerance_f1": 0.07032155974511028, - "union_f1": 0.19601398468283832, - "difference": 0.1955214345352177, - "predicted_positive_rate": 0.9846233766233766, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "climax", - "label": "ClimaX", - "seed": 99, - "scope": "train_fire_top05pct", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.005636060100166944, - "tolerance_f1": 0.15977956000729543, - "union_f1": 0.39407737525514647, - "difference": 0.38844131515497954, - "predicted_positive_rate": 0.9234194122885129, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "climax", - "label": "ClimaX", - "seed": 99, - "scope": "train_fire_top10pct", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.002747305898968793, - "tolerance_f1": 0.08713204456539597, - "union_f1": 0.2309636203280157, - "difference": 0.2282163144290469, - "predicted_positive_rate": 0.9575541703769664, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "climax", - "label": "ClimaX", - "seed": 99, - "scope": "train_fire_top20pct", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.001471035812113219, - "tolerance_f1": 0.06246138809231819, - "union_f1": 0.17013954201238674, - "difference": 0.16866850620027352, - "predicted_positive_rate": 0.9746196660482375, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "pangu_weather", - "label": "Pangu-Weather", - "seed": 1, - "scope": "full_domain", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.0004867477888159953, - "tolerance_f1": 0.06952568867327341, - "union_f1": 0.19394205171734238, - "difference": 0.1934553039285264, - "predicted_positive_rate": 0.9963636363636363, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "pangu_weather", - "label": "Pangu-Weather", - "seed": 1, - "scope": "train_fire_top05pct", - "threshold_key": "0.0000", - "threshold_selector": "strict_f1", - "threshold": 2e-05, - "strict_f1": 0.005205570700778368, - "tolerance_f1": 0.14852385215852132, - "union_f1": 0.36969737295773625, - "difference": 0.3644918022569579, - "predicted_positive_rate": 1.0, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "pangu_weather", - "label": "Pangu-Weather", - "seed": 1, - "scope": "train_fire_top10pct", - "threshold_key": "0.0000", - "threshold_selector": "strict_f1", - "threshold": 2e-05, - "strict_f1": 0.0026308476146364055, - "tolerance_f1": 0.08362683149966108, - "union_f1": 0.22238246829765904, - "difference": 0.21975162068302262, - "predicted_positive_rate": 1.0, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "pangu_weather", - "label": "Pangu-Weather", - "seed": 1, - "scope": "train_fire_top20pct", - "threshold_key": "0.0000", - "threshold_selector": "strict_f1", - "threshold": 2e-05, - "strict_f1": 0.0014337271963217482, - "tolerance_f1": 0.06094415351278495, - "union_f1": 0.16625085053300068, - "difference": 0.16481712333667892, - "predicted_positive_rate": 1.0, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "pangu_weather", - "label": "Pangu-Weather", - "seed": 123, - "scope": "full_domain", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.0004925501476206198, - "tolerance_f1": 0.07032155974511028, - "union_f1": 0.19601398468283832, - "difference": 0.1955214345352177, - "predicted_positive_rate": 0.9846233766233766, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "pangu_weather", - "label": "Pangu-Weather", - "seed": 123, - "scope": "train_fire_top05pct", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.005636060100166944, - "tolerance_f1": 0.15977956000729543, - "union_f1": 0.39407737525514647, - "difference": 0.38844131515497954, - "predicted_positive_rate": 0.9234194122885129, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "pangu_weather", - "label": "Pangu-Weather", - "seed": 123, - "scope": "train_fire_top10pct", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.002747305898968793, - "tolerance_f1": 0.08713204456539597, - "union_f1": 0.2309636203280157, - "difference": 0.2282163144290469, - "predicted_positive_rate": 0.9575541703769664, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "pangu_weather", - "label": "Pangu-Weather", - "seed": 123, - "scope": "train_fire_top20pct", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.001471035812113219, - "tolerance_f1": 0.06246138809231819, - "union_f1": 0.17013954201238674, - "difference": 0.16866850620027352, - "predicted_positive_rate": 0.9746196660482375, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "pangu_weather", - "label": "Pangu-Weather", - "seed": 42, - "scope": "full_domain", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.0004925501476206198, - "tolerance_f1": 0.07032155974511028, - "union_f1": 0.19601398468283832, - "difference": 0.1955214345352177, - "predicted_positive_rate": 0.9846233766233766, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "pangu_weather", - "label": "Pangu-Weather", - "seed": 42, - "scope": "train_fire_top05pct", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.005636060100166944, - "tolerance_f1": 0.15977956000729543, - "union_f1": 0.39407737525514647, - "difference": 0.38844131515497954, - "predicted_positive_rate": 0.9234194122885129, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "pangu_weather", - "label": "Pangu-Weather", - "seed": 42, - "scope": "train_fire_top10pct", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.002747305898968793, - "tolerance_f1": 0.08713204456539597, - "union_f1": 0.2309636203280157, - "difference": 0.2282163144290469, - "predicted_positive_rate": 0.9575541703769664, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "pangu_weather", - "label": "Pangu-Weather", - "seed": 42, - "scope": "train_fire_top20pct", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.001471035812113219, - "tolerance_f1": 0.06246138809231819, - "union_f1": 0.17013954201238674, - "difference": 0.16866850620027352, - "predicted_positive_rate": 0.9746196660482375, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "pangu_weather", - "label": "Pangu-Weather", - "seed": 7, - "scope": "full_domain", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.000486755037982903, - "tolerance_f1": 0.06952668837151453, - "union_f1": 0.19394466064997973, - "difference": 0.19345790561199683, - "predicted_positive_rate": 0.9963487940630797, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "pangu_weather", - "label": "Pangu-Weather", - "seed": 7, - "scope": "train_fire_top05pct", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.005207112273731229, - "tolerance_f1": 0.1485646749474909, - "union_f1": 0.3697868452737493, - "difference": 0.36457973300001806, - "predicted_positive_rate": 0.9997031760166222, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "pangu_weather", - "label": "Pangu-Weather", - "seed": 7, - "scope": "train_fire_top10pct", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.0026312376081679546, - "tolerance_f1": 0.08363872545998453, - "union_f1": 0.22241180661219537, - "difference": 0.21978056900402743, - "predicted_positive_rate": 0.9998515880083111, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "pangu_weather", - "label": "Pangu-Weather", - "seed": 7, - "scope": "train_fire_top20pct", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.0014338335269832637, - "tolerance_f1": 0.06094853876771749, - "union_f1": 0.16626216344954295, - "difference": 0.1648283299225597, - "predicted_positive_rate": 0.999925788497217, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "pangu_weather", - "label": "Pangu-Weather", - "seed": 99, - "scope": "full_domain", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.0004925501476206198, - "tolerance_f1": 0.07032155974511028, - "union_f1": 0.19601398468283832, - "difference": 0.1955214345352177, - "predicted_positive_rate": 0.9846233766233766, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "pangu_weather", - "label": "Pangu-Weather", - "seed": 99, - "scope": "train_fire_top05pct", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.005636060100166944, - "tolerance_f1": 0.15977956000729543, - "union_f1": 0.39407737525514647, - "difference": 0.38844131515497954, - "predicted_positive_rate": 0.9234194122885129, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "pangu_weather", - "label": "Pangu-Weather", - "seed": 99, - "scope": "train_fire_top10pct", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.002747305898968793, - "tolerance_f1": 0.08713204456539597, - "union_f1": 0.2309636203280157, - "difference": 0.2282163144290469, - "predicted_positive_rate": 0.9575541703769664, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "pangu_weather", - "label": "Pangu-Weather", - "seed": 99, - "scope": "train_fire_top20pct", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.001471035812113219, - "tolerance_f1": 0.06246138809231819, - "union_f1": 0.17013954201238674, - "difference": 0.16866850620027352, - "predicted_positive_rate": 0.9746196660482375, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "prithvi_wxc", - "label": "Prithvi-WxC", - "seed": 1, - "scope": "full_domain", - "threshold_key": "0.0000", - "threshold_selector": "strict_f1", - "threshold": 2e-05, - "strict_f1": 0.00048497822606044473, - "tolerance_f1": 0.06928162590128548, - "union_f1": 0.19330497146159284, - "difference": 0.1928199932355324, - "predicted_positive_rate": 1.0, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "prithvi_wxc", - "label": "Prithvi-WxC", - "seed": 1, - "scope": "train_fire_top05pct", - "threshold_key": "0.0000", - "threshold_selector": "strict_f1", - "threshold": 2e-05, - "strict_f1": 0.005205570700778368, - "tolerance_f1": 0.14852385215852132, - "union_f1": 0.36969737295773625, - "difference": 0.3644918022569579, - "predicted_positive_rate": 1.0, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "prithvi_wxc", - "label": "Prithvi-WxC", - "seed": 1, - "scope": "train_fire_top10pct", - "threshold_key": "0.0000", - "threshold_selector": "strict_f1", - "threshold": 2e-05, - "strict_f1": 0.0026308476146364055, - "tolerance_f1": 0.08362683149966108, - "union_f1": 0.22238246829765904, - "difference": 0.21975162068302262, - "predicted_positive_rate": 1.0, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "prithvi_wxc", - "label": "Prithvi-WxC", - "seed": 1, - "scope": "train_fire_top20pct", - "threshold_key": "0.0000", - "threshold_selector": "strict_f1", - "threshold": 2e-05, - "strict_f1": 0.0014337271963217482, - "tolerance_f1": 0.06094415351278495, - "union_f1": 0.16625085053300068, - "difference": 0.16481712333667892, - "predicted_positive_rate": 1.0, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "prithvi_wxc", - "label": "Prithvi-WxC", - "seed": 123, - "scope": "full_domain", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.0004925501476206198, - "tolerance_f1": 0.07032155974511028, - "union_f1": 0.19601398468283832, - "difference": 0.1955214345352177, - "predicted_positive_rate": 0.9846233766233766, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "prithvi_wxc", - "label": "Prithvi-WxC", - "seed": 123, - "scope": "train_fire_top05pct", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.005636060100166944, - "tolerance_f1": 0.15977956000729543, - "union_f1": 0.39407737525514647, - "difference": 0.38844131515497954, - "predicted_positive_rate": 0.9234194122885129, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "prithvi_wxc", - "label": "Prithvi-WxC", - "seed": 123, - "scope": "train_fire_top10pct", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.002747305898968793, - "tolerance_f1": 0.08713204456539597, - "union_f1": 0.2309636203280157, - "difference": 0.2282163144290469, - "predicted_positive_rate": 0.9575541703769664, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "prithvi_wxc", - "label": "Prithvi-WxC", - "seed": 123, - "scope": "train_fire_top20pct", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.001471035812113219, - "tolerance_f1": 0.06246138809231819, - "union_f1": 0.17013954201238674, - "difference": 0.16866850620027352, - "predicted_positive_rate": 0.9746196660482375, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "prithvi_wxc", - "label": "Prithvi-WxC", - "seed": 42, - "scope": "full_domain", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.0004887275525827606, - "tolerance_f1": 0.06979472310089276, - "union_f1": 0.19464046545789707, - "difference": 0.19415173790531431, - "predicted_positive_rate": 0.9923265306122449, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "prithvi_wxc", - "label": "Prithvi-WxC", - "seed": 42, - "scope": "train_fire_top05pct", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.005621644664100071, - "tolerance_f1": 0.15940240082223983, - "union_f1": 0.39326534861308166, - "difference": 0.38764370394898157, - "predicted_positive_rate": 0.9257940041555358, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "prithvi_wxc", - "label": "Prithvi-WxC", - "seed": 42, - "scope": "train_fire_top10pct", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.002737982762277539, - "tolerance_f1": 0.08684880939041824, - "union_f1": 0.23026912265195387, - "difference": 0.22753113988967633, - "predicted_positive_rate": 0.9608192341941229, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "prithvi_wxc", - "label": "Prithvi-WxC", - "seed": 42, - "scope": "train_fire_top20pct", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.0014654606094294809, - "tolerance_f1": 0.062231858228220235, - "union_f1": 0.16954894786900954, - "difference": 0.16808348725958006, - "predicted_positive_rate": 0.9783302411873841, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "prithvi_wxc", - "label": "Prithvi-WxC", - "seed": 7, - "scope": "full_domain", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.000490517270597817, - "tolerance_f1": 0.07004142068407385, - "union_f1": 0.19528379925803643, - "difference": 0.19479328198743862, - "predicted_positive_rate": 0.9887050092764379, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "prithvi_wxc", - "label": "Prithvi-WxC", - "seed": 7, - "scope": "train_fire_top05pct", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.005621644664100071, - "tolerance_f1": 0.15940240082223983, - "union_f1": 0.39326534861308166, - "difference": 0.38764370394898157, - "predicted_positive_rate": 0.9257940041555358, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "prithvi_wxc", - "label": "Prithvi-WxC", - "seed": 7, - "scope": "train_fire_top10pct", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.002737982762277539, - "tolerance_f1": 0.08684880939041824, - "union_f1": 0.23026912265195387, - "difference": 0.22753113988967633, - "predicted_positive_rate": 0.9608192341941229, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "prithvi_wxc", - "label": "Prithvi-WxC", - "seed": 7, - "scope": "train_fire_top20pct", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.0014654606094294809, - "tolerance_f1": 0.062231858228220235, - "union_f1": 0.16954894786900954, - "difference": 0.16808348725958006, - "predicted_positive_rate": 0.9783302411873841, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "prithvi_wxc", - "label": "Prithvi-WxC", - "seed": 99, - "scope": "full_domain", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.0004925501476206198, - "tolerance_f1": 0.07032155974511028, - "union_f1": 0.19601398468283832, - "difference": 0.1955214345352177, - "predicted_positive_rate": 0.9846233766233766, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "prithvi_wxc", - "label": "Prithvi-WxC", - "seed": 99, - "scope": "train_fire_top05pct", - "threshold_key": "0.0001", - "threshold_selector": "strict_f1", - "threshold": 0.0001, - "strict_f1": 0.005636060100166944, - "tolerance_f1": 0.15977956000729543, - "union_f1": 0.39407737525514647, - "difference": 0.38844131515497954, - "predicted_positive_rate": 0.9234194122885129, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "prithvi_wxc", - "label": "Prithvi-WxC", - "seed": 99, - "scope": "train_fire_top10pct", - "threshold_key": "0.0001", - "threshold_selector": "strict_f1", - "threshold": 0.0001, - "strict_f1": 0.002747305898968793, - "tolerance_f1": 0.08713204456539597, - "union_f1": 0.2309636203280157, - "difference": 0.2282163144290469, - "predicted_positive_rate": 0.9575541703769664, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "prithvi_wxc", - "label": "Prithvi-WxC", - "seed": 99, - "scope": "train_fire_top20pct", - "threshold_key": "0.0001", - "threshold_selector": "strict_f1", - "threshold": 0.0001, - "strict_f1": 0.001471035812113219, - "tolerance_f1": 0.06246138809231819, - "union_f1": 0.17013954201238674, - "difference": 0.16866850620027352, - "predicted_positive_rate": 0.9746196660482375, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "reference", - "label": "Reference", - "seed": 0, - "scope": "full_domain", - "threshold_key": "0.9500", - "threshold_selector": "strict_f1", - "threshold": 0.95, - "strict_f1": 0.006568199246490847, - "tolerance_f1": 0.29937890424270136, - "union_f1": 0.6079171724719479, - "difference": 0.601348973225457, - "predicted_positive_rate": 0.014596289424860853, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "reference", - "label": "Reference", - "seed": 0, - "scope": "train_fire_top05pct", - "threshold_key": "0.9500", - "threshold_selector": "strict_f1", - "threshold": 0.95, - "strict_f1": 0.04873596787441818, - "tolerance_f1": 0.40145293653054454, - "union_f1": 0.7623590438272105, - "difference": 0.7136230759527924, - "predicted_positive_rate": 0.02449292569506283, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "reference", - "label": "Reference", - "seed": 0, - "scope": "train_fire_top10pct", - "threshold_key": "0.9500", - "threshold_selector": "strict_f1", - "threshold": 0.95, - "strict_f1": 0.0486915291328531, - "tolerance_f1": 0.4004617740449798, - "union_f1": 0.7590968995963846, - "difference": 0.7104053704635315, - "predicted_positive_rate": 0.012246462847531414, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "reference", - "label": "Reference", - "seed": 0, - "scope": "train_fire_top20pct", - "threshold_key": "0.9500", - "threshold_selector": "strict_f1", - "threshold": 0.95, - "strict_f1": 0.04827336828783222, - "tolerance_f1": 0.39128423640663973, - "union_f1": 0.7294445251003491, - "difference": 0.6811711568125168, - "predicted_positive_rate": 0.006123685837971553, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "stormcast", - "label": "StormCast", - "seed": 1, - "scope": "full_domain", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.0004887494780088282, - "tolerance_f1": 0.06980169309182878, - "union_f1": 0.19466216243584913, - "difference": 0.1941734129578403, - "predicted_positive_rate": 0.9922820037105752, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "stormcast", - "label": "StormCast", - "seed": 1, - "scope": "train_fire_top05pct", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.005219477804850899, - "tolerance_f1": 0.14889206690983883, - "union_f1": 0.3705041858886147, - "difference": 0.36528470808376384, - "predicted_positive_rate": 0.9973285841495992, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "stormcast", - "label": "StormCast", - "seed": 1, - "scope": "train_fire_top10pct", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.0026398468145228757, - "tolerance_f1": 0.08390125124244428, - "union_f1": 0.22305921371803578, - "difference": 0.2204193669035129, - "predicted_positive_rate": 0.9965865241911547, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "stormcast", - "label": "StormCast", - "seed": 1, - "scope": "train_fire_top20pct", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.001439170243914543, - "tolerance_f1": 0.06116860905264603, - "union_f1": 0.1668297790017525, - "difference": 0.16539060875783795, - "predicted_positive_rate": 0.9962152133580705, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "stormcast", - "label": "StormCast", - "seed": 123, - "scope": "full_domain", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.0004905319945803844, - "tolerance_f1": 0.07004345000159352, - "union_f1": 0.1952890900585387, - "difference": 0.19479855806395832, - "predicted_positive_rate": 0.9886753246753247, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "stormcast", - "label": "StormCast", - "seed": 123, - "scope": "train_fire_top05pct", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.005623442559599164, - "tolerance_f1": 0.15944944831691155, - "union_f1": 0.39336666888522, - "difference": 0.3877432263256208, - "predicted_positive_rate": 0.9254971801721579, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "stormcast", - "label": "StormCast", - "seed": 123, - "scope": "train_fire_top10pct", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.002738405168257641, - "tolerance_f1": 0.08686164376307684, - "union_f1": 0.23030060010662487, - "difference": 0.22756219493836724, - "predicted_positive_rate": 0.9606708222024339, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "stormcast", - "label": "StormCast", - "seed": 123, - "scope": "train_fire_top20pct", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.0014655716993051168, - "tolerance_f1": 0.06223643229238444, - "union_f1": 0.16956071956731536, - "difference": 0.16809514786801025, - "predicted_positive_rate": 0.9782560296846011, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "stormcast", - "label": "StormCast", - "seed": 42, - "scope": "full_domain", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.0004925501476206198, - "tolerance_f1": 0.07032155974511028, - "union_f1": 0.19601398468283832, - "difference": 0.1955214345352177, - "predicted_positive_rate": 0.9846233766233766, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "stormcast", - "label": "StormCast", - "seed": 42, - "scope": "train_fire_top05pct", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.005636060100166944, - "tolerance_f1": 0.15977956000729543, - "union_f1": 0.39407737525514647, - "difference": 0.38844131515497954, - "predicted_positive_rate": 0.9234194122885129, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "stormcast", - "label": "StormCast", - "seed": 42, - "scope": "train_fire_top10pct", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.002747305898968793, - "tolerance_f1": 0.08713204456539597, - "union_f1": 0.2309636203280157, - "difference": 0.2282163144290469, - "predicted_positive_rate": 0.9575541703769664, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "stormcast", - "label": "StormCast", - "seed": 42, - "scope": "train_fire_top20pct", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.001471035812113219, - "tolerance_f1": 0.06246138809231819, - "union_f1": 0.17013954201238674, - "difference": 0.16866850620027352, - "predicted_positive_rate": 0.9746196660482375, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "stormcast", - "label": "StormCast", - "seed": 7, - "scope": "full_domain", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.0004905319945803844, - "tolerance_f1": 0.07004345000159352, - "union_f1": 0.1952890900585387, - "difference": 0.19479855806395832, - "predicted_positive_rate": 0.9886753246753247, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "stormcast", - "label": "StormCast", - "seed": 7, - "scope": "train_fire_top05pct", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.00562524160545995, - "tolerance_f1": 0.15949652359184557, - "union_f1": 0.3934680413788068, - "difference": 0.38784279977334685, - "predicted_positive_rate": 0.92520035618878, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "stormcast", - "label": "StormCast", - "seed": 7, - "scope": "train_fire_top10pct", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.0027388277045923585, - "tolerance_f1": 0.0868744819295806, - "union_f1": 0.23033208616831716, - "difference": 0.2275932584637248, - "predicted_positive_rate": 0.960522410210745, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "stormcast", - "label": "StormCast", - "seed": 7, - "scope": "train_fire_top20pct", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.0014656828060244616, - "tolerance_f1": 0.06224100702898884, - "union_f1": 0.1695724929003409, - "difference": 0.16810681009431644, - "predicted_positive_rate": 0.9781818181818182, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "stormcast", - "label": "StormCast", - "seed": 99, - "scope": "full_domain", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.0004887494780088282, - "tolerance_f1": 0.06979774571547358, - "union_f1": 0.1946483495612302, - "difference": 0.19415960008322136, - "predicted_positive_rate": 0.9922820037105752, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "stormcast", - "label": "StormCast", - "seed": 99, - "scope": "train_fire_top05pct", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.005623442559599164, - "tolerance_f1": 0.15944944831691155, - "union_f1": 0.39336666888522, - "difference": 0.3877432263256208, - "predicted_positive_rate": 0.9254971801721579, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "stormcast", - "label": "StormCast", - "seed": 99, - "scope": "train_fire_top10pct", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.002738405168257641, - "tolerance_f1": 0.08686164376307684, - "union_f1": 0.23030060010662487, - "difference": 0.22756219493836724, - "predicted_positive_rate": 0.9606708222024339, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "stormcast", - "label": "StormCast", - "seed": 99, - "scope": "train_fire_top20pct", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1", - "threshold": 0.0002, - "strict_f1": 0.0014655716993051168, - "tolerance_f1": 0.06223643229238444, - "union_f1": 0.16956071956731536, - "difference": 0.16809514786801025, - "predicted_positive_rate": 0.9782560296846011, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "dlwp", - "label": "DLWP", - "seed": 17, - "scope": "full_domain", - "threshold_key": "0.0800", - "threshold_selector": "strict_f1", - "threshold": 0.08, - "strict_f1": 0.001786130143937306, - "tolerance_f1": 0.16383974771641738, - "union_f1": 0.3062219870023089, - "difference": 0.3044358568583716, - "predicted_positive_rate": 0.057648732220160794, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "dlwp", - "label": "DLWP", - "seed": 17, - "scope": "train_fire_top05pct", - "threshold_key": "0.0800", - "threshold_selector": "strict_f1", - "threshold": 0.08, - "strict_f1": 0.01972516815912186, - "tolerance_f1": 0.3500819147736832, - "union_f1": 0.5865273622259427, - "difference": 0.5668021940668209, - "predicted_positive_rate": 0.05733155238943306, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "dlwp", - "label": "DLWP", - "seed": 17, - "scope": "train_fire_top10pct", - "threshold_key": "0.0800", - "threshold_selector": "strict_f1", - "threshold": 0.08, - "strict_f1": 0.018998410174880763, - "tolerance_f1": 0.3409571337826908, - "union_f1": 0.57270947387435, - "difference": 0.5537110636994692, - "predicted_positive_rate": 0.02979989116453943, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "dlwp", - "label": "DLWP", - "seed": 17, - "scope": "train_fire_top20pct", - "threshold_key": "0.1000", - "threshold_selector": "strict_f1", - "threshold": 0.1, - "strict_f1": 0.01931818181818182, - "tolerance_f1": 0.28953047944835925, - "union_f1": 0.4666657840911348, - "difference": 0.44734760227295295, - "predicted_positive_rate": 0.010166975881261595, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "dlwp", - "label": "DLWP", - "seed": 37, - "scope": "full_domain", - "threshold_key": "0.1000", - "threshold_selector": "strict_f1", - "threshold": 0.1, - "strict_f1": 0.0019600476944939, - "tolerance_f1": 0.17299721297318826, - "union_f1": 0.2944465174913956, - "difference": 0.2924864697969017, - "predicted_positive_rate": 0.04519196042053185, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "dlwp", - "label": "DLWP", - "seed": 37, - "scope": "train_fire_top05pct", - "threshold_key": "0.1000", - "threshold_selector": "strict_f1", - "threshold": 0.1, - "strict_f1": 0.024960845732184808, - "tolerance_f1": 0.35323298220195054, - "union_f1": 0.5303411060257172, - "difference": 0.5053802602935323, - "predicted_positive_rate": 0.04792965271593945, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "dlwp", - "label": "DLWP", - "seed": 37, - "scope": "train_fire_top10pct", - "threshold_key": "0.1000", - "threshold_selector": "strict_f1", - "threshold": 0.1, - "strict_f1": 0.023191305534082125, - "tolerance_f1": 0.33689092573275314, - "union_f1": 0.5094071833388841, - "difference": 0.486215877804802, - "predicted_positive_rate": 0.025880577817354308, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "dlwp", - "label": "DLWP", - "seed": 37, - "scope": "train_fire_top20pct", - "threshold_key": "0.2000", - "threshold_selector": "strict_f1", - "threshold": 0.2, - "strict_f1": 0.028949691085613415, - "tolerance_f1": 0.20397124965652888, - "union_f1": 0.29215296893898335, - "difference": 0.26320327785336994, - "predicted_positive_rate": 0.0027860235003092145, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "dlwp", - "label": "DLWP", - "seed": 57, - "scope": "full_domain", - "threshold_key": "0.1000", - "threshold_selector": "strict_f1", - "threshold": 0.1, - "strict_f1": 0.001008153911497155, - "tolerance_f1": 0.09460750931776095, - "union_f1": 0.17657638185878205, - "difference": 0.17556822794728488, - "predicted_positive_rate": 0.02625751391465677, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "dlwp", - "label": "DLWP", - "seed": 57, - "scope": "train_fire_top05pct", - "threshold_key": "0.0500", - "threshold_selector": "strict_f1", - "threshold": 0.05, - "strict_f1": 0.017035016422646553, - "tolerance_f1": 0.3101459291213085, - "union_f1": 0.5806373053637345, - "difference": 0.5636022889410879, - "predicted_positive_rate": 0.1306866528148808, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "dlwp", - "label": "DLWP", - "seed": 57, - "scope": "train_fire_top10pct", - "threshold_key": "0.0800", - "threshold_selector": "strict_f1", - "threshold": 0.08, - "strict_f1": 0.018477139910622206, - "tolerance_f1": 0.2606560686639405, - "union_f1": 0.41307788619921965, - "difference": 0.3946007462885974, - "predicted_positive_rate": 0.02746487582863362, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "dlwp", - "label": "DLWP", - "seed": 57, - "scope": "train_fire_top20pct", - "threshold_key": "0.0800", - "threshold_selector": "strict_f1", - "threshold": 0.08, - "strict_f1": 0.01226752201682564, - "tolerance_f1": 0.19661692001462533, - "union_f1": 0.3215961142158263, - "difference": 0.30932859219900066, - "predicted_positive_rate": 0.021262213976499692, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "dlwp", - "label": "DLWP", - "seed": 7, - "scope": "full_domain", - "threshold_key": "0.0800", - "threshold_selector": "strict_f1", - "threshold": 0.08, - "strict_f1": 0.0016304484151205278, - "tolerance_f1": 0.14442504878529014, - "union_f1": 0.265022372628049, - "difference": 0.26339192421292845, - "predicted_positive_rate": 0.0589282622139765, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "dlwp", - "label": "DLWP", - "seed": 7, - "scope": "train_fire_top05pct", - "threshold_key": "0.0800", - "threshold_selector": "strict_f1", - "threshold": 0.08, - "strict_f1": 0.016864097224592905, - "tolerance_f1": 0.28520211660517514, - "union_f1": 0.47313209204823786, - "difference": 0.45626799482364494, - "predicted_positive_rate": 0.06984763035519936, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "dlwp", - "label": "DLWP", - "seed": 7, - "scope": "train_fire_top10pct", - "threshold_key": "0.0800", - "threshold_selector": "strict_f1", - "threshold": 0.08, - "strict_f1": 0.011312372621310312, - "tolerance_f1": 0.2160234001333052, - "union_f1": 0.3704561794666731, - "difference": 0.3591438068453628, - "predicted_positive_rate": 0.052691204115959234, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "dlwp", - "label": "DLWP", - "seed": 7, - "scope": "train_fire_top20pct", - "threshold_key": "0.0800", - "threshold_selector": "strict_f1", - "threshold": 0.08, - "strict_f1": 0.007379212786746584, - "tolerance_f1": 0.1600767886426284, - "union_f1": 0.2769459546244767, - "difference": 0.2695667418377301, - "predicted_positive_rate": 0.04168893011750155, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "dlwp", - "label": "DLWP", - "seed": 97, - "scope": "full_domain", - "threshold_key": "0.1000", - "threshold_selector": "strict_f1", - "threshold": 0.1, - "strict_f1": 0.002078902199880332, - "tolerance_f1": 0.16987085515333694, - "union_f1": 0.36723610814010926, - "difference": 0.36515720594022893, - "predicted_positive_rate": 0.028077427334570193, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "dlwp", - "label": "DLWP", - "seed": 97, - "scope": "train_fire_top05pct", - "threshold_key": "0.0500", - "threshold_selector": "strict_f1", - "threshold": 0.05, - "strict_f1": 0.011684287006420426, - "tolerance_f1": 0.2874938355126855, - "union_f1": 0.6023409019864735, - "difference": 0.5906566149800531, - "predicted_positive_rate": 0.16420797467102008, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "dlwp", - "label": "DLWP", - "seed": 97, - "scope": "train_fire_top10pct", - "threshold_key": "0.0500", - "threshold_selector": "strict_f1", - "threshold": 0.05, - "strict_f1": 0.008571862589787548, - "tolerance_f1": 0.22837595071441363, - "union_f1": 0.4906920775169778, - "difference": 0.48212021492719026, - "predicted_positive_rate": 0.11266572672405263, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "dlwp", - "label": "DLWP", - "seed": 97, - "scope": "train_fire_top20pct", - "threshold_key": "0.1000", - "threshold_selector": "strict_f1", - "threshold": 0.1, - "strict_f1": 0.008323671928416423, - "tolerance_f1": 0.19681923783547378, - "union_f1": 0.3891450726598544, - "difference": 0.38082140073143794, - "predicted_positive_rate": 0.02008596165739023, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "fcn", - "label": "FCN", - "seed": 17, - "scope": "full_domain", - "threshold_key": "0.1500", - "threshold_selector": "strict_f1", - "threshold": 0.15, - "strict_f1": 0.002068681772914111, - "tolerance_f1": 0.19094441931639336, - "union_f1": 0.38616767546901964, - "difference": 0.3840989936961055, - "predicted_positive_rate": 0.03192430426716141, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "fcn", - "label": "FCN", - "seed": 17, - "scope": "train_fire_top05pct", - "threshold_key": "0.1500", - "threshold_selector": "strict_f1", - "threshold": 0.15, - "strict_f1": 0.012095158293562608, - "tolerance_f1": 0.25572856051368525, - "union_f1": 0.47261322844070225, - "difference": 0.46051807014713964, - "predicted_positive_rate": 0.06528643514395963, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "fcn", - "label": "FCN", - "seed": 17, - "scope": "train_fire_top10pct", - "threshold_key": "0.2000", - "threshold_selector": "strict_f1", - "threshold": 0.2, - "strict_f1": 0.006403642961329112, - "tolerance_f1": 0.16176542623012913, - "union_f1": 0.3108270369881256, - "difference": 0.30442339402679647, - "predicted_positive_rate": 0.03344711586029484, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "fcn", - "label": "FCN", - "seed": 17, - "scope": "train_fire_top20pct", - "threshold_key": "0.3000", - "threshold_selector": "strict_f1", - "threshold": 0.3, - "strict_f1": 0.004578192172408018, - "tolerance_f1": 0.10554731176319206, - "union_f1": 0.2167503101126067, - "difference": 0.2121721179401987, - "predicted_positive_rate": 0.01035930735930736, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "fcn", - "label": "FCN", - "seed": 37, - "scope": "full_domain", - "threshold_key": "0.2000", - "threshold_selector": "strict_f1", - "threshold": 0.2, - "strict_f1": 0.00208290997764682, - "tolerance_f1": 0.16485748333864078, - "union_f1": 0.33983598065101456, - "difference": 0.33775307067336774, - "predicted_positive_rate": 0.009495980210265924, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "fcn", - "label": "FCN", - "seed": 37, - "scope": "train_fire_top05pct", - "threshold_key": "0.2000", - "threshold_selector": "strict_f1", - "threshold": 0.2, - "strict_f1": 0.024537870112874206, - "tolerance_f1": 0.27999632336313063, - "union_f1": 0.5116723674766515, - "difference": 0.48713449736377723, - "predicted_positive_rate": 0.01251113089937667, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "fcn", - "label": "FCN", - "seed": 37, - "scope": "train_fire_top10pct", - "threshold_key": "0.2000", - "threshold_selector": "strict_f1", - "threshold": 0.2, - "strict_f1": 0.019849146486701073, - "tolerance_f1": 0.23420382391023928, - "union_f1": 0.4437163794060617, - "difference": 0.4238672329193606, - "predicted_positive_rate": 0.00802908875037103, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "fcn", - "label": "FCN", - "seed": 37, - "scope": "train_fire_top20pct", - "threshold_key": "0.2000", - "threshold_selector": "strict_f1", - "threshold": 0.2, - "strict_f1": 0.016197783461210574, - "tolerance_f1": 0.19512358967789506, - "union_f1": 0.3809676016171751, - "difference": 0.3647698181559645, - "predicted_positive_rate": 0.005085961657390229, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "fcn", - "label": "FCN", - "seed": 57, - "scope": "full_domain", - "threshold_key": "0.5000", - "threshold_selector": "strict_f1", - "threshold": 0.5, - "strict_f1": 0.0033447053314602988, - "tolerance_f1": 0.16083569292964747, - "union_f1": 0.2863487755227932, - "difference": 0.2830040701913329, - "predicted_positive_rate": 0.0016064316635745208, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "fcn", - "label": "FCN", - "seed": 57, - "scope": "train_fire_top05pct", - "threshold_key": "0.3000", - "threshold_selector": "strict_f1", - "threshold": 0.3, - "strict_f1": 0.01569234169521494, - "tolerance_f1": 0.300679830289768, - "union_f1": 0.49777625508437506, - "difference": 0.4820839133891601, - "predicted_positive_rate": 0.0197734243593549, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "fcn", - "label": "FCN", - "seed": 57, - "scope": "train_fire_top10pct", - "threshold_key": "0.3000", - "threshold_selector": "strict_f1", - "threshold": 0.3, - "strict_f1": 0.010726695875509895, - "tolerance_f1": 0.2256264421482596, - "union_f1": 0.3983538151409869, - "difference": 0.387627119265477, - "predicted_positive_rate": 0.015055159790244386, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "fcn", - "label": "FCN", - "seed": 57, - "scope": "train_fire_top20pct", - "threshold_key": "0.3000", - "threshold_selector": "strict_f1", - "threshold": 0.3, - "strict_f1": 0.00859106529209622, - "tolerance_f1": 0.18647162974553239, - "union_f1": 0.33360989768899596, - "difference": 0.3250188323968997, - "predicted_positive_rate": 0.010080395794681509, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "fcn", - "label": "FCN", - "seed": 7, - "scope": "full_domain", - "threshold_key": "0.1500", - "threshold_selector": "strict_f1", - "threshold": 0.15, - "strict_f1": 0.0026467855605157606, - "tolerance_f1": 0.23211789308126832, - "union_f1": 0.48493981229339306, - "difference": 0.4822930267328773, - "predicted_positive_rate": 0.027048113790970933, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "fcn", - "label": "FCN", - "seed": 7, - "scope": "train_fire_top05pct", - "threshold_key": "0.1500", - "threshold_selector": "strict_f1", - "threshold": 0.15, - "strict_f1": 0.01222529471692621, - "tolerance_f1": 0.3028035236186743, - "union_f1": 0.5768233236256973, - "difference": 0.5645980289087711, - "predicted_positive_rate": 0.06537300880577818, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "fcn", - "label": "FCN", - "seed": 7, - "scope": "train_fire_top10pct", - "threshold_key": "0.2000", - "threshold_selector": "strict_f1", - "threshold": 0.2, - "strict_f1": 0.008918001736248123, - "tolerance_f1": 0.2270292295163525, - "union_f1": 0.4541508313100257, - "difference": 0.4452328295737776, - "predicted_positive_rate": 0.030024982685267636, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "fcn", - "label": "FCN", - "seed": 7, - "scope": "train_fire_top20pct", - "threshold_key": "0.3000", - "threshold_selector": "strict_f1", - "threshold": 0.3, - "strict_f1": 0.008704984497972811, - "tolerance_f1": 0.1596269732904185, - "union_f1": 0.3313747927063083, - "difference": 0.3226698082083355, - "predicted_positive_rate": 0.009654916512059368, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "fcn", - "label": "FCN", - "seed": 97, - "scope": "full_domain", - "threshold_key": "0.3000", - "threshold_selector": "strict_f1", - "threshold": 0.3, - "strict_f1": 0.00400312243549969, - "tolerance_f1": 0.22655113161542761, - "union_f1": 0.5057290416588791, - "difference": 0.5017259192233794, - "predicted_positive_rate": 0.005936920222634508, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "fcn", - "label": "FCN", - "seed": 97, - "scope": "train_fire_top05pct", - "threshold_key": "0.1500", - "threshold_selector": "strict_f1", - "threshold": 0.15, - "strict_f1": 0.016606090988845718, - "tolerance_f1": 0.3296389749733903, - "union_f1": 0.6562774730922674, - "difference": 0.6396713821034217, - "predicted_positive_rate": 0.07633570792520035, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "fcn", - "label": "FCN", - "seed": 97, - "scope": "train_fire_top10pct", - "threshold_key": "0.1500", - "threshold_selector": "strict_f1", - "threshold": 0.15, - "strict_f1": 0.012986069049061175, - "tolerance_f1": 0.2724598040570218, - "union_f1": 0.5655038878451817, - "difference": 0.5525178187961205, - "predicted_positive_rate": 0.049730384881765115, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "fcn", - "label": "FCN", - "seed": 97, - "scope": "train_fire_top20pct", - "threshold_key": "0.3000", - "threshold_selector": "strict_f1", - "threshold": 0.3, - "strict_f1": 0.011738073279114613, - "tolerance_f1": 0.20219147661108805, - "union_f1": 0.4415948106689657, - "difference": 0.4298567373898511, - "predicted_positive_rate": 0.006658627087198516, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "fengwu", - "label": "FengWu", - "seed": 17, - "scope": "full_domain", - "threshold_key": "0.3000", - "threshold_selector": "strict_f1", - "threshold": 0.3, - "strict_f1": 0.003438698584843275, - "tolerance_f1": 0.1569315132056949, - "union_f1": 0.283249876555753, - "difference": 0.2798111779709097, - "predicted_positive_rate": 0.004433395176252319, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "fengwu", - "label": "FengWu", - "seed": 17, - "scope": "train_fire_top05pct", - "threshold_key": "0.3000", - "threshold_selector": "strict_f1", - "threshold": 0.3, - "strict_f1": 0.01848001848001848, - "tolerance_f1": 0.21207047330588913, - "union_f1": 0.3374691422247865, - "difference": 0.31898912374476807, - "predicted_positive_rate": 0.008098347679825863, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "fengwu", - "label": "FengWu", - "seed": 17, - "scope": "train_fire_top10pct", - "threshold_key": "0.3000", - "threshold_selector": "strict_f1", - "threshold": 0.3, - "strict_f1": 0.014059753954305799, - "tolerance_f1": 0.17969557482138315, - "union_f1": 0.29555470142676327, - "difference": 0.28149494747245746, - "predicted_positive_rate": 0.005720045513010785, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "fengwu", - "label": "FengWu", - "seed": 17, - "scope": "train_fire_top20pct", - "threshold_key": "0.3000", - "threshold_selector": "strict_f1", - "threshold": 0.3, - "strict_f1": 0.011695906432748539, - "tolerance_f1": 0.1579992352547558, - "union_f1": 0.26532858639687246, - "difference": 0.2536326799641239, - "predicted_positive_rate": 0.0035126777983920843, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "fengwu", - "label": "FengWu", - "seed": 37, - "scope": "full_domain", - "threshold_key": "0.2000", - "threshold_selector": "strict_f1", - "threshold": 0.2, - "strict_f1": 0.0018414421399496025, - "tolerance_f1": 0.10327893388535589, - "union_f1": 0.2254839395366447, - "difference": 0.2236424973966951, - "predicted_positive_rate": 0.0023098330241187385, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "fengwu", - "label": "FengWu", - "seed": 37, - "scope": "train_fire_top05pct", - "threshold_key": "0.2000", - "threshold_selector": "strict_f1", - "threshold": 0.2, - "strict_f1": 0.01267427122940431, - "tolerance_f1": 0.13288915578187116, - "union_f1": 0.27966639148093675, - "difference": 0.26699212025153246, - "predicted_positive_rate": 0.0032452755515979025, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "fengwu", - "label": "FengWu", - "seed": 37, - "scope": "train_fire_top10pct", - "threshold_key": "0.2000", - "threshold_selector": "strict_f1", - "threshold": 0.2, - "strict_f1": 0.011273957158962795, - "tolerance_f1": 0.1191832198916516, - "union_f1": 0.259409805794969, - "difference": 0.24813584863600618, - "predicted_positive_rate": 0.0019738794894627485, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "fengwu", - "label": "FengWu", - "seed": 37, - "scope": "train_fire_top20pct", - "threshold_key": "0.2000", - "threshold_selector": "strict_f1", - "threshold": 0.2, - "strict_f1": 0.010316368638239341, - "tolerance_f1": 0.11041562343515275, - "union_f1": 0.24476649923702953, - "difference": 0.2344501305987902, - "predicted_positive_rate": 0.0010810142238713667, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "fengwu", - "label": "FengWu", - "seed": 57, - "scope": "full_domain", - "threshold_key": "0.5000", - "threshold_selector": "strict_f1", - "threshold": 0.5, - "strict_f1": 0.001842299189388357, - "tolerance_f1": 0.042906126190853267, - "union_f1": 0.06919785857013828, - "difference": 0.06735555938074993, - "predicted_positive_rate": 0.00042881880024737165, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "fengwu", - "label": "FengWu", - "seed": 57, - "scope": "train_fire_top05pct", - "threshold_key": "0.3000", - "threshold_selector": "strict_f1", - "threshold": 0.3, - "strict_f1": 0.0205686630369026, - "tolerance_f1": 0.1866312847963942, - "union_f1": 0.3374418848724159, - "difference": 0.3168732218355133, - "predicted_positive_rate": 0.005567923221529633, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "fengwu", - "label": "FengWu", - "seed": 57, - "scope": "train_fire_top10pct", - "threshold_key": "0.3000", - "threshold_selector": "strict_f1", - "threshold": 0.3, - "strict_f1": 0.02015411973918198, - "tolerance_f1": 0.1840127014362662, - "union_f1": 0.33432003556596124, - "difference": 0.3141659158267793, - "predicted_positive_rate": 0.0028556940734144652, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "fengwu", - "label": "FengWu", - "seed": 57, - "scope": "train_fire_top20pct", - "threshold_key": "0.3000", - "threshold_selector": "strict_f1", - "threshold": 0.3, - "strict_f1": 0.018831348656881748, - "tolerance_f1": 0.17388386719173746, - "union_f1": 0.31802311631118385, - "difference": 0.2991917676543021, - "predicted_positive_rate": 0.0015157699443413728, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "fengwu", - "label": "FengWu", - "seed": 7, - "scope": "full_domain", - "threshold_key": "0.3000", - "threshold_selector": "strict_f1", - "threshold": 0.3, - "strict_f1": 0.00268370607028754, - "tolerance_f1": 0.09710793321083327, - "union_f1": 0.186659362535767, - "difference": 0.18397565646547948, - "predicted_positive_rate": 0.0016931354359925788, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "fengwu", - "label": "FengWu", - "seed": 7, - "scope": "train_fire_top05pct", - "threshold_key": "0.3000", - "threshold_selector": "strict_f1", - "threshold": 0.3, - "strict_f1": 0.013054830287206266, - "tolerance_f1": 0.12305221580797777, - "union_f1": 0.22237658124637444, - "difference": 0.20932175095916816, - "predicted_positive_rate": 0.0030746017611556348, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "fengwu", - "label": "FengWu", - "seed": 7, - "scope": "train_fire_top10pct", - "threshold_key": "0.3000", - "threshold_selector": "strict_f1", - "threshold": 0.3, - "strict_f1": 0.011185682326621925, - "tolerance_f1": 0.11778287307219655, - "union_f1": 0.21014056663719285, - "difference": 0.19895488431057093, - "predicted_positive_rate": 0.0019998515880083113, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "fengwu", - "label": "FengWu", - "seed": 7, - "scope": "train_fire_top20pct", - "threshold_key": "0.3000", - "threshold_selector": "strict_f1", - "threshold": 0.3, - "strict_f1": 0.010309278350515464, - "tolerance_f1": 0.11130870584431331, - "union_f1": 0.20443519333920424, - "difference": 0.19412591498868878, - "predicted_positive_rate": 0.0010822510822510823, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "fengwu", - "label": "FengWu", - "seed": 97, - "scope": "full_domain", - "threshold_key": "0.2000", - "threshold_selector": "strict_f1", - "threshold": 0.2, - "strict_f1": 0.0032567291196169464, - "tolerance_f1": 0.20002472134774485, - "union_f1": 0.4405198598043062, - "difference": 0.43726313068468925, - "predicted_positive_rate": 0.012518243661100804, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "fengwu", - "label": "FengWu", - "seed": 97, - "scope": "train_fire_top05pct", - "threshold_key": "0.3000", - "threshold_selector": "strict_f1", - "threshold": 0.3, - "strict_f1": 0.013695949911954608, - "tolerance_f1": 0.15917085028122405, - "union_f1": 0.3283194730321549, - "difference": 0.3146235231202003, - "predicted_positive_rate": 0.010032650638171565, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "fengwu", - "label": "FengWu", - "seed": 97, - "scope": "train_fire_top10pct", - "threshold_key": "0.5000", - "threshold_selector": "strict_f1", - "threshold": 0.5, - "strict_f1": 0.005460750853242321, - "tolerance_f1": 0.04684227229450968, - "union_f1": 0.10984387215996587, - "difference": 0.10438312130672354, - "predicted_positive_rate": 0.0004947066389630949, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "fengwu", - "label": "FengWu", - "seed": 97, - "scope": "train_fire_top20pct", - "threshold_key": "0.5000", - "threshold_selector": "strict_f1", - "threshold": 0.5, - "strict_f1": 0.004807692307692307, - "tolerance_f1": 0.04393081279657125, - "union_f1": 0.1067457375833951, - "difference": 0.1019380452757028, - "predicted_positive_rate": 0.0003116883116883117, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "fuxi", - "label": "FuXi", - "seed": 17, - "scope": "full_domain", - "threshold_key": "0.3000", - "threshold_selector": "strict_f1", - "threshold": 0.3, - "strict_f1": 0.003097173828881146, - "tolerance_f1": 0.20441214417386547, - "union_f1": 0.37735961055895995, - "difference": 0.3742624367300788, - "predicted_positive_rate": 0.004230179344465059, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "fuxi", - "label": "FuXi", - "seed": 17, - "scope": "train_fire_top05pct", - "threshold_key": "0.2000", - "threshold_selector": "strict_f1", - "threshold": 0.2, - "strict_f1": 0.02138840232322301, - "tolerance_f1": 0.30180702036802187, - "union_f1": 0.551963463682504, - "difference": 0.530575061359281, - "predicted_positive_rate": 0.024220837043633127, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "fuxi", - "label": "FuXi", - "seed": 17, - "scope": "train_fire_top10pct", - "threshold_key": "0.3000", - "threshold_selector": "strict_f1", - "threshold": 0.3, - "strict_f1": 0.019476002782286114, - "tolerance_f1": 0.26166119072060445, - "union_f1": 0.45937946725189766, - "difference": 0.43990346446961154, - "predicted_positive_rate": 0.0040170179083803305, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "fuxi", - "label": "FuXi", - "seed": 17, - "scope": "train_fire_top20pct", - "threshold_key": "0.3000", - "threshold_selector": "strict_f1", - "threshold": 0.3, - "strict_f1": 0.013674670058832883, - "tolerance_f1": 0.25519274478475146, - "union_f1": 0.4153742248616508, - "difference": 0.40169955480281794, - "predicted_positive_rate": 0.0031719233147804574, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "fuxi", - "label": "FuXi", - "seed": 37, - "scope": "full_domain", - "threshold_key": "0.2000", - "threshold_selector": "strict_f1", - "threshold": 0.2, - "strict_f1": 0.0029424127784783522, - "tolerance_f1": 0.13539446199130012, - "union_f1": 0.22039566333970034, - "difference": 0.217453250561222, - "predicted_positive_rate": 0.001522943722943723, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "fuxi", - "label": "FuXi", - "seed": 37, - "scope": "train_fire_top05pct", - "threshold_key": "0.1500", - "threshold_selector": "strict_f1", - "threshold": 0.15, - "strict_f1": 0.02109053497942387, - "tolerance_f1": 0.26086649919553234, - "union_f1": 0.4073509199876191, - "difference": 0.3862603850081952, - "predicted_positive_rate": 0.007007519540912239, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "fuxi", - "label": "FuXi", - "seed": 37, - "scope": "train_fire_top10pct", - "threshold_key": "0.2000", - "threshold_selector": "strict_f1", - "threshold": 0.2, - "strict_f1": 0.016701461377870565, - "tolerance_f1": 0.1656142112861716, - "union_f1": 0.24700654033204764, - "difference": 0.23030507895417707, - "predicted_positive_rate": 0.0016448995745522905, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "fuxi", - "label": "FuXi", - "seed": 37, - "scope": "train_fire_top20pct", - "threshold_key": "0.2000", - "threshold_selector": "strict_f1", - "threshold": 0.2, - "strict_f1": 0.015267175572519083, - "tolerance_f1": 0.15340459969134643, - "union_f1": 0.23133715755118187, - "difference": 0.21606998197866278, - "predicted_positive_rate": 0.0009029066171923315, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "fuxi", - "label": "FuXi", - "seed": 57, - "scope": "full_domain", - "threshold_key": "0.3000", - "threshold_selector": "strict_f1", - "threshold": 0.3, - "strict_f1": 0.005897939994871356, - "tolerance_f1": 0.22566599998043468, - "union_f1": 0.3638836107393784, - "difference": 0.35798567074450705, - "predicted_positive_rate": 0.0026514533085961658, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "fuxi", - "label": "FuXi", - "seed": 57, - "scope": "train_fire_top05pct", - "threshold_key": "0.3000", - "threshold_selector": "strict_f1", - "threshold": 0.3, - "strict_f1": 0.030370370370370364, - "tolerance_f1": 0.3499638832422191, - "union_f1": 0.5231807249206608, - "difference": 0.49281035455029043, - "predicted_positive_rate": 0.004068962105471455, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "fuxi", - "label": "FuXi", - "seed": 57, - "scope": "train_fire_top10pct", - "threshold_key": "0.3000", - "threshold_selector": "strict_f1", - "threshold": 0.3, - "strict_f1": 0.026973684210526316, - "tolerance_f1": 0.3231296638237823, - "union_f1": 0.48397755354947186, - "difference": 0.45700386933894555, - "predicted_positive_rate": 0.002442614029880281, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "fuxi", - "label": "FuXi", - "seed": 57, - "scope": "train_fire_top20pct", - "threshold_key": "0.3000", - "threshold_selector": "strict_f1", - "threshold": 0.3, - "strict_f1": 0.0239695995323005, - "tolerance_f1": 0.30046102485294895, - "union_f1": 0.4554233007850691, - "difference": 0.4314537012527686, - "predicted_positive_rate": 0.0013982683982683983, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "fuxi", - "label": "FuXi", - "seed": 7, - "scope": "full_domain", - "threshold_key": "0.2000", - "threshold_selector": "strict_f1", - "threshold": 0.2, - "strict_f1": 0.0033573873422136956, - "tolerance_f1": 0.2682493391647262, - "union_f1": 0.44868694500233336, - "difference": 0.44532955766011967, - "predicted_positive_rate": 0.005430797773654916, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "fuxi", - "label": "FuXi", - "seed": 7, - "scope": "train_fire_top05pct", - "threshold_key": "0.1000", - "threshold_selector": "strict_f1", - "threshold": 0.1, - "strict_f1": 0.016660707901322847, - "tolerance_f1": 0.38200179460101663, - "union_f1": 0.6304826850687859, - "difference": 0.6138219771674631, - "predicted_positive_rate": 0.0665751459384585, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "fuxi", - "label": "FuXi", - "seed": 7, - "scope": "train_fire_top10pct", - "threshold_key": "0.3000", - "threshold_selector": "strict_f1", - "threshold": 0.3, - "strict_f1": 0.011389521640091117, - "tolerance_f1": 0.2251054852320675, - "union_f1": 0.35879236229131267, - "difference": 0.34740284065122157, - "predicted_positive_rate": 0.0019404867913327397, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "fuxi", - "label": "FuXi", - "seed": 7, - "scope": "train_fire_top20pct", - "threshold_key": "0.3000", - "threshold_selector": "strict_f1", - "threshold": 0.3, - "strict_f1": 0.008802816901408451, - "tolerance_f1": 0.18265891116039673, - "union_f1": 0.29083501784981214, - "difference": 0.2820322009484037, - "predicted_positive_rate": 0.0013902288188002473, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "fuxi", - "label": "FuXi", - "seed": 97, - "scope": "full_domain", - "threshold_key": "0.2000", - "threshold_selector": "strict_f1", - "threshold": 0.2, - "strict_f1": 0.003573506191307238, - "tolerance_f1": 0.21789178988640112, - "union_f1": 0.4541140407970205, - "difference": 0.45054053460571325, - "predicted_positive_rate": 0.008687322201607915, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "fuxi", - "label": "FuXi", - "seed": 97, - "scope": "train_fire_top05pct", - "threshold_key": "0.1500", - "threshold_selector": "strict_f1", - "threshold": 0.15, - "strict_f1": 0.012024257632789628, - "tolerance_f1": 0.30008183753413636, - "union_f1": 0.5835624273550017, - "difference": 0.5715381697222122, - "predicted_positive_rate": 0.04470416542990007, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "fuxi", - "label": "FuXi", - "seed": 97, - "scope": "train_fire_top10pct", - "threshold_key": "0.2000", - "threshold_selector": "strict_f1", - "threshold": 0.2, - "strict_f1": 0.008170871559633027, - "tolerance_f1": 0.22512986809908905, - "union_f1": 0.4615416812544075, - "difference": 0.45337080969477445, - "predicted_positive_rate": 0.01593821114079351, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "fuxi", - "label": "FuXi", - "seed": 97, - "scope": "train_fire_top20pct", - "threshold_key": "0.2000", - "threshold_selector": "strict_f1", - "threshold": 0.2, - "strict_f1": 0.006516342573438146, - "tolerance_f1": 0.20602494796886814, - "union_f1": 0.4436019253840096, - "difference": 0.43708558281057147, - "predicted_positive_rate": 0.011240568954854669, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "pangu_weather_h24", - "label": "Pangu-Weather", - "seed": 17, - "scope": "full_domain", - "threshold_key": "0.3000", - "threshold_selector": "strict_f1", - "threshold": 0.3, - "strict_f1": 0.0018999796430752525, - "tolerance_f1": 0.11199020227536667, - "union_f1": 0.27178749222596443, - "difference": 0.2698875125828892, - "predicted_positive_rate": 0.0034029684601113172, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "pangu_weather_h24", - "label": "Pangu-Weather", - "seed": 17, - "scope": "train_fire_top05pct", - "threshold_key": "0.3000", - "threshold_selector": "strict_f1", - "threshold": 0.3, - "strict_f1": 0.006323656223052602, - "tolerance_f1": 0.1236361475439318, - "union_f1": 0.26905489411020966, - "difference": 0.2627312378871571, - "predicted_positive_rate": 0.00599584446423271, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "pangu_weather_h24", - "label": "Pangu-Weather", - "seed": 17, - "scope": "train_fire_top10pct", - "threshold_key": "0.3000", - "threshold_selector": "strict_f1", - "threshold": 0.3, - "strict_f1": 0.004407051282051282, - "tolerance_f1": 0.09804613935969869, - "union_f1": 0.2156791196445671, - "difference": 0.2112720683625158, - "predicted_positive_rate": 0.004856782428020184, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "pangu_weather_h24", - "label": "Pangu-Weather", - "seed": 17, - "scope": "train_fire_top20pct", - "threshold_key": "0.3000", - "threshold_selector": "strict_f1", - "threshold": 0.3, - "strict_f1": 0.003081232492997199, - "tolerance_f1": 0.0918614020950846, - "union_f1": 0.19598081171127166, - "difference": 0.19289957921827447, - "predicted_positive_rate": 0.0036982065553494126, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "pangu_weather_h24", - "label": "Pangu-Weather", - "seed": 37, - "scope": "full_domain", - "threshold_key": "0.1000", - "threshold_selector": "strict_f1", - "threshold": 0.1, - "strict_f1": 0.002025689424980434, - "tolerance_f1": 0.17658327178527805, - "union_f1": 0.4448076750202692, - "difference": 0.44278198559528875, - "predicted_positive_rate": 0.015876932591218307, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "pangu_weather_h24", - "label": "Pangu-Weather", - "seed": 37, - "scope": "train_fire_top05pct", - "threshold_key": "0.1000", - "threshold_selector": "strict_f1", - "threshold": 0.1, - "strict_f1": 0.014334527877323108, - "tolerance_f1": 0.23786880092608761, - "union_f1": 0.5563138131518957, - "difference": 0.5419792852745725, - "predicted_positive_rate": 0.02707034728406055, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "pangu_weather_h24", - "label": "Pangu-Weather", - "seed": 37, - "scope": "train_fire_top10pct", - "threshold_key": "0.1000", - "threshold_selector": "strict_f1", - "threshold": 0.1, - "strict_f1": 0.010952623535404991, - "tolerance_f1": 0.19299696406724362, - "union_f1": 0.478233072358786, - "difference": 0.467280448823381, - "predicted_positive_rate": 0.018105026219451866, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "pangu_weather_h24", - "label": "Pangu-Weather", - "seed": 37, - "scope": "train_fire_top20pct", - "threshold_key": "0.1500", - "threshold_selector": "strict_f1", - "threshold": 0.15, - "strict_f1": 0.003306175463454953, - "tolerance_f1": 0.10046960132658891, - "union_f1": 0.3168721772329861, - "difference": 0.31356600176953114, - "predicted_positive_rate": 0.004520098948670378, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "pangu_weather_h24", - "label": "Pangu-Weather", - "seed": 57, - "scope": "full_domain", - "threshold_key": "0.3000", - "threshold_selector": "strict_f1", - "threshold": 0.3, - "strict_f1": 0.00458918723078781, - "tolerance_f1": 0.14940756501428915, - "union_f1": 0.31114689741621243, - "difference": 0.3065577101854246, - "predicted_positive_rate": 0.0028299319727891157, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "pangu_weather_h24", - "label": "Pangu-Weather", - "seed": 57, - "scope": "train_fire_top05pct", - "threshold_key": "0.3000", - "threshold_selector": "strict_f1", - "threshold": 0.3, - "strict_f1": 0.012796587576646227, - "tolerance_f1": 0.15839502058327895, - "union_f1": 0.3235832074044849, - "difference": 0.3107866198278387, - "predicted_positive_rate": 0.006668645493222519, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "pangu_weather_h24", - "label": "Pangu-Weather", - "seed": 57, - "scope": "train_fire_top10pct", - "threshold_key": "0.3000", - "threshold_selector": "strict_f1", - "threshold": 0.3, - "strict_f1": 0.010430247718383311, - "tolerance_f1": 0.14381976498868418, - "union_f1": 0.2960690978058883, - "difference": 0.285638850087505, - "predicted_positive_rate": 0.004374443455031167, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "pangu_weather_h24", - "label": "Pangu-Weather", - "seed": 57, - "scope": "train_fire_top20pct", - "threshold_key": "0.3000", - "threshold_selector": "strict_f1", - "threshold": 0.3, - "strict_f1": 0.0094525403702245, - "tolerance_f1": 0.1330139941503748, - "union_f1": 0.2753987342044462, - "difference": 0.2659461938342217, - "predicted_positive_rate": 0.002423005565862709, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "pangu_weather_h24", - "label": "Pangu-Weather", - "seed": 7, - "scope": "full_domain", - "threshold_key": "0.5000", - "threshold_selector": "strict_f1", - "threshold": 0.5, - "strict_f1": 0.00412434125105018, - "tolerance_f1": 0.08503531719948608, - "union_f1": 0.1652119748219299, - "difference": 0.1610876335708797, - "predicted_positive_rate": 0.0013768707482993197, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "pangu_weather_h24", - "label": "Pangu-Weather", - "seed": 7, - "scope": "train_fire_top05pct", - "threshold_key": "0.3000", - "threshold_selector": "strict_f1", - "threshold": 0.3, - "strict_f1": 0.00961323496534764, - "tolerance_f1": 0.13752896018113672, - "union_f1": 0.2838373343689194, - "difference": 0.27422409940357173, - "predicted_positive_rate": 0.019518650440288908, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "pangu_weather_h24", - "label": "Pangu-Weather", - "seed": 7, - "scope": "train_fire_top10pct", - "threshold_key": "0.5000", - "threshold_selector": "strict_f1", - "threshold": 0.5, - "strict_f1": 0.005579507988840983, - "tolerance_f1": 0.07891517659613233, - "union_f1": 0.14561269032771065, - "difference": 0.14003318233886966, - "predicted_positive_rate": 0.0035594142673394677, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "pangu_weather_h24", - "label": "Pangu-Weather", - "seed": 7, - "scope": "train_fire_top20pct", - "threshold_key": "0.5000", - "threshold_selector": "strict_f1", - "threshold": 0.5, - "strict_f1": 0.0051329911339244045, - "tolerance_f1": 0.07335569213391166, - "union_f1": 0.13544861662147875, - "difference": 0.13031562548755435, - "predicted_positive_rate": 0.0019332096474953618, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "pangu_weather_h24", - "label": "Pangu-Weather", - "seed": 97, - "scope": "full_domain", - "threshold_key": "0.5000", - "threshold_selector": "strict_f1", - "threshold": 0.5, - "strict_f1": 0.0015570260801868433, - "tolerance_f1": 0.04422040571422221, - "union_f1": 0.1360311898534274, - "difference": 0.13447416377324056, - "predicted_positive_rate": 0.0013461966604823748, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "pangu_weather_h24", - "label": "Pangu-Weather", - "seed": 97, - "scope": "train_fire_top05pct", - "threshold_key": "0.3000", - "threshold_selector": "strict_f1", - "threshold": 0.3, - "strict_f1": 0.010571134928791661, - "tolerance_f1": 0.16460271440438795, - "union_f1": 0.3673387319643668, - "difference": 0.35676759703557515, - "predicted_positive_rate": 0.014237657069357871, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "pangu_weather_h24", - "label": "Pangu-Weather", - "seed": 97, - "scope": "train_fire_top10pct", - "threshold_key": "0.5000", - "threshold_selector": "strict_f1", - "threshold": 0.5, - "strict_f1": 0.002669336002669336, - "tolerance_f1": 0.03347640061525648, - "union_f1": 0.12011082544991639, - "difference": 0.11744148944724705, - "predicted_positive_rate": 0.002389433066191748, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "pangu_weather_h24", - "label": "Pangu-Weather", - "seed": 97, - "scope": "train_fire_top20pct", - "threshold_key": "0.5000", - "threshold_selector": "strict_f1", - "threshold": 0.5, - "strict_f1": 0.002313475997686524, - "tolerance_f1": 0.036969834429575875, - "union_f1": 0.12157774813170726, - "difference": 0.11926427213402073, - "predicted_positive_rate": 0.0014211502782931354, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "pangu6", - "label": "Pangu-Weather", - "seed": 17, - "scope": "full_domain", - "threshold_key": "0.2000", - "threshold_selector": "strict_f1", - "threshold": 0.2, - "strict_f1": 0.0019783244452090142, - "tolerance_f1": 0.1809729686535962, - "union_f1": 0.40012531103066123, - "difference": 0.3981469865854522, - "predicted_positive_rate": 0.01413716759431045, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "pangu6", - "label": "Pangu-Weather", - "seed": 17, - "scope": "train_fire_top05pct", - "threshold_key": "0.2000", - "threshold_selector": "strict_f1", - "threshold": 0.2, - "strict_f1": 0.011848926191063935, - "tolerance_f1": 0.23514322359982898, - "union_f1": 0.46975335070929664, - "difference": 0.4579044245182327, - "predicted_positive_rate": 0.027451271396062134, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "pangu6", - "label": "Pangu-Weather", - "seed": 17, - "scope": "train_fire_top10pct", - "threshold_key": "0.2000", - "threshold_selector": "strict_f1", - "threshold": 0.2, - "strict_f1": 0.008011413520632135, - "tolerance_f1": 0.18336249012390426, - "union_f1": 0.3796403112945744, - "difference": 0.37162889777394226, - "predicted_positive_rate": 0.021221678044919364, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "pangu6", - "label": "Pangu-Weather", - "seed": 17, - "scope": "train_fire_top20pct", - "threshold_key": "0.2000", - "threshold_selector": "strict_f1", - "threshold": 0.2, - "strict_f1": 0.005117388748660143, - "tolerance_f1": 0.14938342628742876, - "union_f1": 0.323212329597846, - "difference": 0.3180949408491859, - "predicted_positive_rate": 0.017168212739641312, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "pangu6", - "label": "Pangu-Weather", - "seed": 37, - "scope": "full_domain", - "threshold_key": "0.1500", - "threshold_selector": "strict_f1", - "threshold": 0.15, - "strict_f1": 0.0014740203906154037, - "tolerance_f1": 0.14188194761227685, - "union_f1": 0.3364875741725597, - "difference": 0.3350135537819443, - "predicted_positive_rate": 0.00479208410636982, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "pangu6", - "label": "Pangu-Weather", - "seed": 37, - "scope": "train_fire_top05pct", - "threshold_key": "0.1500", - "threshold_selector": "strict_f1", - "threshold": 0.15, - "strict_f1": 0.012218828103304637, - "tolerance_f1": 0.1893747086942956, - "union_f1": 0.4388313446697724, - "difference": 0.42661251656646776, - "predicted_positive_rate": 0.006297615514000198, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "pangu6", - "label": "Pangu-Weather", - "seed": 37, - "scope": "train_fire_top10pct", - "threshold_key": "0.1500", - "threshold_selector": "strict_f1", - "threshold": 0.15, - "strict_f1": 0.010149942329873126, - "tolerance_f1": 0.1623425297604306, - "union_f1": 0.41150510005450436, - "difference": 0.4013551577246312, - "predicted_positive_rate": 0.004044226773523301, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "pangu6", - "label": "Pangu-Weather", - "seed": 37, - "scope": "train_fire_top20pct", - "threshold_key": "0.1500", - "threshold_selector": "strict_f1", - "threshold": 0.15, - "strict_f1": 0.008453410182516812, - "tolerance_f1": 0.15039592221466835, - "union_f1": 0.3734736878120269, - "difference": 0.36502027762951006, - "predicted_positive_rate": 0.0025015460729746444, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "pangu6", - "label": "Pangu-Weather", - "seed": 57, - "scope": "full_domain", - "threshold_key": "0.3000", - "threshold_selector": "strict_f1", - "threshold": 0.3, - "strict_f1": 0.003978746443051044, - "tolerance_f1": 0.18301623358471802, - "union_f1": 0.3546823845445782, - "difference": 0.35070363810152716, - "predicted_positive_rate": 0.004669140383426098, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "pangu6", - "label": "Pangu-Weather", - "seed": 57, - "scope": "train_fire_top05pct", - "threshold_key": "0.3000", - "threshold_selector": "strict_f1", - "threshold": 0.3, - "strict_f1": 0.017845192951148784, - "tolerance_f1": 0.2501388218123639, - "union_f1": 0.44006255856354737, - "difference": 0.4222173656123986, - "predicted_positive_rate": 0.008479271791827446, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "pangu6", - "label": "Pangu-Weather", - "seed": 57, - "scope": "train_fire_top10pct", - "threshold_key": "0.3000", - "threshold_selector": "strict_f1", - "threshold": 0.3, - "strict_f1": 0.016403526758253024, - "tolerance_f1": 0.23804674197160172, - "union_f1": 0.4214127938301568, - "difference": 0.40500926707190377, - "predicted_positive_rate": 0.004714554269318294, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "pangu6", - "label": "Pangu-Weather", - "seed": 57, - "scope": "train_fire_top20pct", - "threshold_key": "0.3000", - "threshold_selector": "strict_f1", - "threshold": 0.3, - "strict_f1": 0.01471129091577786, - "tolerance_f1": 0.2249788440916092, - "union_f1": 0.3957167586610078, - "difference": 0.3810054677452299, - "predicted_positive_rate": 0.0026456400742115027, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "pangu6", - "label": "Pangu-Weather", - "seed": 7, - "scope": "full_domain", - "threshold_key": "0.3000", - "threshold_selector": "strict_f1", - "threshold": 0.3, - "strict_f1": 0.0037437942540896885, - "tolerance_f1": 0.12218430629860097, - "union_f1": 0.22268222965661022, - "difference": 0.21893843540252053, - "predicted_positive_rate": 0.0012771799628942485, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "pangu6", - "label": "Pangu-Weather", - "seed": 7, - "scope": "train_fire_top05pct", - "threshold_key": "0.3000", - "threshold_selector": "strict_f1", - "threshold": 0.3, - "strict_f1": 0.010515247108307044, - "tolerance_f1": 0.12664654352782623, - "union_f1": 0.22723717713750308, - "difference": 0.21672193002919604, - "predicted_positive_rate": 0.004447412684278223, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "pangu6", - "label": "Pangu-Weather", - "seed": 7, - "scope": "train_fire_top10pct", - "threshold_key": "0.3000", - "threshold_selector": "strict_f1", - "threshold": 0.3, - "strict_f1": 0.00788436268068331, - "tolerance_f1": 0.1079162350962175, - "union_f1": 0.19681189159430446, - "difference": 0.18892752891362113, - "predicted_positive_rate": 0.0033887404768972, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "pangu6", - "label": "Pangu-Weather", - "seed": 7, - "scope": "train_fire_top20pct", - "threshold_key": "0.3000", - "threshold_selector": "strict_f1", - "threshold": 0.3, - "strict_f1": 0.0070011668611435224, - "tolerance_f1": 0.09937369519832985, - "union_f1": 0.18187865016757276, - "difference": 0.17487748330642924, - "predicted_positive_rate": 0.001932591218305504, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "pangu6", - "label": "Pangu-Weather", - "seed": 97, - "scope": "full_domain", - "threshold_key": "0.1500", - "threshold_selector": "strict_f1", - "threshold": 0.15, - "strict_f1": 0.0026024230327071516, - "tolerance_f1": 0.2264875300093167, - "union_f1": 0.46795172614222125, - "difference": 0.4653493031095141, - "predicted_positive_rate": 0.029129251700680273, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "pangu6", - "label": "Pangu-Weather", - "seed": 97, - "scope": "train_fire_top05pct", - "threshold_key": "0.1500", - "threshold_selector": "strict_f1", - "threshold": 0.15, - "strict_f1": 0.015850265576258328, - "tolerance_f1": 0.3098064830268362, - "union_f1": 0.5952835470121373, - "difference": 0.5794332814358789, - "predicted_positive_rate": 0.056067576926882357, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "pangu6", - "label": "Pangu-Weather", - "seed": 97, - "scope": "train_fire_top10pct", - "threshold_key": "0.1500", - "threshold_selector": "strict_f1", - "threshold": 0.15, - "strict_f1": 0.012206999545484059, - "tolerance_f1": 0.25501453744070673, - "union_f1": 0.5172553944576963, - "difference": 0.5050483949122122, - "predicted_positive_rate": 0.03677772830711388, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "pangu6", - "label": "Pangu-Weather", - "seed": 97, - "scope": "train_fire_top20pct", - "threshold_key": "0.1500", - "threshold_selector": "strict_f1", - "threshold": 0.15, - "strict_f1": 0.008937574479787332, - "tolerance_f1": 0.226729727407505, - "union_f1": 0.45415946247619876, - "difference": 0.44522188799641144, - "predicted_positive_rate": 0.026268398268398267, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 0 - }, - { - "model_tag": "reference", - "label": "Reference", - "seed": 1, - "scope": "full_domain", - "threshold_key": "0.95", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.95, - "strict_f1": 0.004136112962850736, - "tolerance_f1": 0.2985122125101588, - "union_f1": 0.5521558725617307, - "difference": 0.5480197595988799, - "predicted_positive_rate": 0.016084972170686457, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 1 - }, - { - "model_tag": "reference", - "label": "Reference", - "seed": 1, - "scope": "train_fire_top05pct", - "threshold_key": "0.95", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.95, - "strict_f1": 0.03321760345312939, - "tolerance_f1": 0.39555976109165375, - "union_f1": 0.7028883266246737, - "difference": 0.6696707231715443, - "predicted_positive_rate": 0.023750865736618186, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 1 - }, - { - "model_tag": "reference", - "label": "Reference", - "seed": 1, - "scope": "train_fire_top10pct", - "threshold_key": "0.95", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.95, - "strict_f1": 0.033186462923033656, - "tolerance_f1": 0.3945515652074689, - "union_f1": 0.699798626339235, - "difference": 0.6666121634162013, - "predicted_positive_rate": 0.011875432868309093, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 1 - }, - { - "model_tag": "reference", - "label": "Reference", - "seed": 1, - "scope": "train_fire_top20pct", - "threshold_key": "0.95", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.95, - "strict_f1": 0.032893514216688347, - "tolerance_f1": 0.3852239693382854, - "union_f1": 0.6717469182906158, - "difference": 0.6388534040739275, - "predicted_positive_rate": 0.0059381570810142234, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 1 - }, - { - "model_tag": "reference", - "label": "Reference", - "seed": 123, - "scope": "full_domain", - "threshold_key": "0.95", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.95, - "strict_f1": 0.0026426577586601383, - "tolerance_f1": 0.2756667293290642, - "union_f1": 0.5889022335770174, - "difference": 0.5862595758183573, - "predicted_positive_rate": 0.021380705009276437, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 1 - }, - { - "model_tag": "reference", - "label": "Reference", - "seed": 123, - "scope": "train_fire_top05pct", - "threshold_key": "0.9", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.9, - "strict_f1": 0.02462935239291061, - "tolerance_f1": 0.3702273155255325, - "union_f1": 0.7206983694651612, - "difference": 0.6960690170722506, - "predicted_positive_rate": 0.04860987434451371, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 1 - }, - { - "model_tag": "reference", - "label": "Reference", - "seed": 123, - "scope": "train_fire_top10pct", - "threshold_key": "0.9", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.9, - "strict_f1": 0.02461746391852102, - "tolerance_f1": 0.369479820868587, - "union_f1": 0.7178986487638336, - "difference": 0.6932811848453125, - "predicted_positive_rate": 0.024304937172256853, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 1 - }, - { - "model_tag": "reference", - "label": "Reference", - "seed": 123, - "scope": "train_fire_top20pct", - "threshold_key": "0.9", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.9, - "strict_f1": 0.024505093215452623, - "tolerance_f1": 0.36252633147981417, - "union_f1": 0.6923475866906666, - "difference": 0.6678424934752141, - "predicted_positive_rate": 0.012153370439084725, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 1 - }, - { - "model_tag": "reference", - "label": "Reference", - "seed": 42, - "scope": "full_domain", - "threshold_key": "0.95", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.95, - "strict_f1": 0.0045296167247386755, - "tolerance_f1": 0.3056545098251043, - "union_f1": 0.6239373622157508, - "difference": 0.6194077454910121, - "predicted_positive_rate": 0.019636239950525665, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 1 - }, - { - "model_tag": "reference", - "label": "Reference", - "seed": 42, - "scope": "train_fire_top05pct", - "threshold_key": "0.95", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.95, - "strict_f1": 0.038147512864494, - "tolerance_f1": 0.4063154181985604, - "union_f1": 0.7476654358764496, - "difference": 0.7095179230119556, - "predicted_positive_rate": 0.033442168793905214, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 1 - }, - { - "model_tag": "reference", - "label": "Reference", - "seed": 42, - "scope": "train_fire_top10pct", - "threshold_key": "0.95", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.95, - "strict_f1": 0.0381213575591361, - "tolerance_f1": 0.4052951105632299, - "union_f1": 0.7444554956076256, - "difference": 0.7063341380484894, - "predicted_positive_rate": 0.016721084396952607, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 1 - }, - { - "model_tag": "reference", - "label": "Reference", - "seed": 42, - "scope": "train_fire_top20pct", - "threshold_key": "0.95", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.95, - "strict_f1": 0.03787465940054495, - "tolerance_f1": 0.39585180444145995, - "union_f1": 0.7152818658418959, - "difference": 0.677407206441351, - "predicted_positive_rate": 0.008361162646876932, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 1 - }, - { - "model_tag": "reference", - "label": "Reference", - "seed": 7, - "scope": "full_domain", - "threshold_key": "0.95", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.95, - "strict_f1": 0.006568199246490847, - "tolerance_f1": 0.29937890424270136, - "union_f1": 0.6079171724719479, - "difference": 0.601348973225457, - "predicted_positive_rate": 0.014596289424860853, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 1 - }, - { - "model_tag": "reference", - "label": "Reference", - "seed": 7, - "scope": "train_fire_top05pct", - "threshold_key": "0.95", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.95, - "strict_f1": 0.04873596787441818, - "tolerance_f1": 0.40145293653054454, - "union_f1": 0.7623590438272105, - "difference": 0.7136230759527924, - "predicted_positive_rate": 0.02449292569506283, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 1 - }, - { - "model_tag": "reference", - "label": "Reference", - "seed": 7, - "scope": "train_fire_top10pct", - "threshold_key": "0.95", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.95, - "strict_f1": 0.0486915291328531, - "tolerance_f1": 0.4004617740449798, - "union_f1": 0.7590968995963846, - "difference": 0.7104053704635315, - "predicted_positive_rate": 0.012246462847531414, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 1 - }, - { - "model_tag": "reference", - "label": "Reference", - "seed": 7, - "scope": "train_fire_top20pct", - "threshold_key": "0.95", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.95, - "strict_f1": 0.04827336828783222, - "tolerance_f1": 0.39128423640663973, - "union_f1": 0.7294445251003491, - "difference": 0.6811711568125168, - "predicted_positive_rate": 0.006123685837971553, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 1 - }, - { - "model_tag": "reference", - "label": "Reference", - "seed": 99, - "scope": "full_domain", - "threshold_key": "0.95", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.95, - "strict_f1": 0.004855561780634761, - "tolerance_f1": 0.30820726968158313, - "union_f1": 0.5803661485742072, - "difference": 0.5755105867935725, - "predicted_positive_rate": 0.012341125541125542, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 1 - }, - { - "model_tag": "reference", - "label": "Reference", - "seed": 99, - "scope": "train_fire_top05pct", - "threshold_key": "0.95", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.95, - "strict_f1": 0.0332911515623479, - "tolerance_f1": 0.3895272012200133, - "union_f1": 0.707788551758277, - "difference": 0.6744974001959292, - "predicted_positive_rate": 0.022801028989809045, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 1 - }, - { - "model_tag": "reference", - "label": "Reference", - "seed": 99, - "scope": "train_fire_top10pct", - "threshold_key": "0.95", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.95, - "strict_f1": 0.033258776621608484, - "tolerance_f1": 0.38853542457185924, - "union_f1": 0.7047690151929771, - "difference": 0.6715102385713686, - "predicted_positive_rate": 0.011400514494904522, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 1 - }, - { - "model_tag": "reference", - "label": "Reference", - "seed": 99, - "scope": "train_fire_top20pct", - "threshold_key": "0.95", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.95, - "strict_f1": 0.03295432645981885, - "tolerance_f1": 0.37935948303093225, - "union_f1": 0.6773183297441491, - "difference": 0.6443640032843302, - "predicted_positive_rate": 0.005700680272108844, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 1 - }, - { - "model_tag": "aurora", - "label": "Aurora", - "seed": 1, - "scope": "full_domain", - "threshold_key": "0.00028485153", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.0002848515287041664, - "strict_f1": 0.0004938156417811271, - "tolerance_f1": 0.05024333401294363, - "union_f1": 0.14329601271947937, - "difference": 0.14280219707769823, - "predicted_positive_rate": 0.21716512059369203, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 2 - }, - { - "model_tag": "aurora", - "label": "Aurora", - "seed": 1, - "scope": "train_fire_top05pct", - "threshold_key": "0.00028478092", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.0002847809228114784, - "strict_f1": 0.026484309350641822, - "tolerance_f1": 0.24351172066402033, - "union_f1": 0.4964540293986394, - "difference": 0.4699697200479976, - "predicted_positive_rate": 0.08275452656574651, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 2 - }, - { - "model_tag": "aurora", - "label": "Aurora", - "seed": 1, - "scope": "train_fire_top10pct", - "threshold_key": "0.00028478092", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.0002847809228114784, - "strict_f1": 0.026476637409113293, - "tolerance_f1": 0.24290193233931348, - "union_f1": 0.49392607486162804, - "difference": 0.46744943745251477, - "predicted_positive_rate": 0.04137726328287326, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 2 - }, - { - "model_tag": "aurora", - "label": "Aurora", - "seed": 123, - "scope": "full_domain", - "threshold_key": "0.00028207037", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.00028207036666572094, - "strict_f1": 0.0006488102455003172, - "tolerance_f1": 0.09040502340162183, - "union_f1": 0.24482830102631173, - "difference": 0.24417949078081141, - "predicted_positive_rate": 0.7199758812615955, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 2 - }, - { - "model_tag": "aurora", - "label": "Aurora", - "seed": 123, - "scope": "train_fire_top05pct", - "threshold_key": "0.00028194374", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.0002819437358994037, - "strict_f1": 0.005618158058227469, - "tolerance_f1": 0.16005632828519903, - "union_f1": 0.39469838382183675, - "difference": 0.3890802257636093, - "predicted_positive_rate": 0.921086870485802, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 2 - }, - { - "model_tag": "aurora", - "label": "Aurora", - "seed": 123, - "scope": "train_fire_top10pct", - "threshold_key": "0.00028190663", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.00028190662851557136, - "strict_f1": 0.00275028568108307, - "tolerance_f1": 0.08722255267001126, - "union_f1": 0.23118547623512836, - "difference": 0.2284351905540453, - "predicted_positive_rate": 0.9565152864351439, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 2 - }, - { - "model_tag": "aurora", - "label": "Aurora", - "seed": 42, - "scope": "full_domain", - "threshold_key": "0.00028167077", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.00028167077107355, - "strict_f1": 0.0007076278562951189, - "tolerance_f1": 0.09343380358557288, - "union_f1": 0.2527418148996877, - "difference": 0.2520341870433926, - "predicted_positive_rate": 0.5915943104514533, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 2 - }, - { - "model_tag": "aurora", - "label": "Aurora", - "seed": 42, - "scope": "train_fire_top05pct", - "threshold_key": "0.00028234132", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.00028234132332727313, - "strict_f1": 0.00618071474679909, - "tolerance_f1": 0.10264712599745741, - "union_f1": 0.26261754818982164, - "difference": 0.25643683344302254, - "predicted_positive_rate": 0.48403581676066093, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 2 - }, - { - "model_tag": "aurora", - "label": "Aurora", - "seed": 42, - "scope": "train_fire_top10pct", - "threshold_key": "0.00028129236", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.00028129236306995153, - "strict_f1": 0.0035136704264394467, - "tolerance_f1": 0.11052705132032803, - "union_f1": 0.2868180777521314, - "difference": 0.28330440732569195, - "predicted_positive_rate": 0.735743791431681, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 2 - }, - { - "model_tag": "aurora", - "label": "Aurora", - "seed": 7, - "scope": "full_domain", - "threshold_key": "0.00027011667", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.0002701166733984173, - "strict_f1": 0.0007124228455695601, - "tolerance_f1": 0.09742777619989017, - "union_f1": 0.2618373273867167, - "difference": 0.2611249045411472, - "predicted_positive_rate": 0.6671256648113791, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 2 - }, - { - "model_tag": "aurora", - "label": "Aurora", - "seed": 7, - "scope": "train_fire_top05pct", - "threshold_key": "0.00026978867", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.00026978866662830114, - "strict_f1": 0.00565521517042468, - "tolerance_f1": 0.16052688500868942, - "union_f1": 0.39515679465182413, - "difference": 0.38950157948139946, - "predicted_positive_rate": 0.9150341347580885, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 2 - }, - { - "model_tag": "aurora", - "label": "Aurora", - "seed": 7, - "scope": "train_fire_top10pct", - "threshold_key": "0.00026914896", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.00026914896443486214, - "strict_f1": 0.00275028568108307, - "tolerance_f1": 0.08722255267001126, - "union_f1": 0.23118547623512836, - "difference": 0.2284351905540453, - "predicted_positive_rate": 0.9565152864351439, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 2 - }, - { - "model_tag": "aurora", - "label": "Aurora", - "seed": 99, - "scope": "full_domain", - "threshold_key": "0.00028320565", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.0002832056488841772, - "strict_f1": 0.0007150400212604667, - "tolerance_f1": 0.09353511619384626, - "union_f1": 0.25248287808338293, - "difference": 0.25176783806212244, - "predicted_positive_rate": 0.5833835497835498, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 2 - }, - { - "model_tag": "aurora", - "label": "Aurora", - "seed": 99, - "scope": "train_fire_top05pct", - "threshold_key": "0.00028413086", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.00028413085965439677, - "strict_f1": 0.0053561461508737284, - "tolerance_f1": 0.08994268846280176, - "union_f1": 0.22524158122442267, - "difference": 0.21988543507354893, - "predicted_positive_rate": 0.4582789156030474, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 2 - }, - { - "model_tag": "aurora", - "label": "Aurora", - "seed": 99, - "scope": "train_fire_top10pct", - "threshold_key": "0.00028264782", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.000282647815765813, - "strict_f1": 0.0034589138681033788, - "tolerance_f1": 0.10903157063807417, - "union_f1": 0.28340954762838766, - "difference": 0.2799506337602843, - "predicted_positive_rate": 0.749557237558128, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 2 - }, - { - "model_tag": "climax", - "label": "ClimaX", - "seed": 1, - "scope": "full_domain", - "threshold_key": "0.0014425347", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.001442534703828549, - "strict_f1": 0.003394935139973715, - "tolerance_f1": 0.3112148878371163, - "union_f1": 0.6396719244631184, - "difference": 0.6362769893231447, - "predicted_positive_rate": 0.041071861471861475, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 2 - }, - { - "model_tag": "climax", - "label": "ClimaX", - "seed": 1, - "scope": "train_fire_top05pct", - "threshold_key": "0.0014882656", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.0014882655838293205, - "strict_f1": 0.011492789729228419, - "tolerance_f1": 0.3186326966868806, - "union_f1": 0.6456200196251293, - "difference": 0.6341272298959009, - "predicted_positive_rate": 0.13341248639556744, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 2 - }, - { - "model_tag": "climax", - "label": "ClimaX", - "seed": 1, - "scope": "train_fire_top10pct", - "threshold_key": "0.0014882656", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.0014882655838293205, - "strict_f1": 0.011490700169087834, - "tolerance_f1": 0.3179193199949169, - "union_f1": 0.6427222935095626, - "difference": 0.6312315933404747, - "predicted_positive_rate": 0.06670624319778372, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 2 - }, - { - "model_tag": "climax", - "label": "ClimaX", - "seed": 123, - "scope": "full_domain", - "threshold_key": "0.00053939684", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.0005393968409013734, - "strict_f1": 0.002897888198757764, - "tolerance_f1": 0.2855276491036767, - "union_f1": 0.6076291192986658, - "difference": 0.604731231099908, - "predicted_positive_rate": 0.12421632653061225, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 2 - }, - { - "model_tag": "climax", - "label": "ClimaX", - "seed": 123, - "scope": "train_fire_top05pct", - "threshold_key": "0.00054362687", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.0005436268660297483, - "strict_f1": 0.012252516711603196, - "tolerance_f1": 0.3566174282966785, - "union_f1": 0.7174878093274878, - "difference": 0.7052352926158846, - "predicted_positive_rate": 0.3074774908479272, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 2 - }, - { - "model_tag": "climax", - "label": "ClimaX", - "seed": 123, - "scope": "train_fire_top10pct", - "threshold_key": "0.00067760051", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.0006776005131691771, - "strict_f1": 0.010293697838827224, - "tolerance_f1": 0.34872837593596695, - "union_f1": 0.6989921615406495, - "difference": 0.6886984637018222, - "predicted_positive_rate": 0.14598421885821708, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 2 - }, - { - "model_tag": "climax", - "label": "ClimaX", - "seed": 42, - "scope": "full_domain", - "threshold_key": "0.0014632087", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.0014632086704014654, - "strict_f1": 0.004637761701086003, - "tolerance_f1": 0.31289270592627744, - "union_f1": 0.620505744456497, - "difference": 0.615867982755411, - "predicted_positive_rate": 0.037147804576376, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 2 - }, - { - "model_tag": "climax", - "label": "ClimaX", - "seed": 42, - "scope": "train_fire_top05pct", - "threshold_key": "0.0014632083", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.0014632082671483923, - "strict_f1": 0.014246556800108287, - "tolerance_f1": 0.32161191611096424, - "union_f1": 0.6170788209868917, - "difference": 0.6028322641867834, - "predicted_positive_rate": 0.14358118135945386, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 2 - }, - { - "model_tag": "climax", - "label": "ClimaX", - "seed": 42, - "scope": "train_fire_top10pct", - "threshold_key": "0.0014632083", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.0014632082671483923, - "strict_f1": 0.014244146704560834, - "tolerance_f1": 0.3208182317552862, - "union_f1": 0.6141812807681712, - "difference": 0.5999371340636104, - "predicted_positive_rate": 0.07179059067972693, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 2 - }, - { - "model_tag": "climax", - "label": "ClimaX", - "seed": 7, - "scope": "full_domain", - "threshold_key": "0.0013572093", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.0013572092986540852, - "strict_f1": 0.0027501821392557145, - "tolerance_f1": 0.24162386260287075, - "union_f1": 0.4719733074426152, - "difference": 0.46922312530335947, - "predicted_positive_rate": 0.025392455163883734, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 2 - }, - { - "model_tag": "climax", - "label": "ClimaX", - "seed": 7, - "scope": "train_fire_top05pct", - "threshold_key": "0.00074653866", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.0007465386633598005, - "strict_f1": 0.013106997858124742, - "tolerance_f1": 0.3655334236435691, - "union_f1": 0.7398705155472896, - "difference": 0.7267635176891648, - "predicted_positive_rate": 0.3068887899475611, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 2 - }, - { - "model_tag": "climax", - "label": "ClimaX", - "seed": 7, - "scope": "train_fire_top10pct", - "threshold_key": "0.00075197712", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.0007519771209341887, - "strict_f1": 0.01308555880921415, - "tolerance_f1": 0.36399388919405107, - "union_f1": 0.7361209272894985, - "difference": 0.7230353684802844, - "predicted_positive_rate": 0.15368556446027506, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 2 - }, - { - "model_tag": "climax", - "label": "ClimaX", - "seed": 99, - "scope": "full_domain", - "threshold_key": "0.0010212299", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.001021229882830197, - "strict_f1": 0.003717439410136951, - "tolerance_f1": 0.33641526956922785, - "union_f1": 0.6677488562714049, - "difference": 0.664031416861268, - "predicted_positive_rate": 0.055986765615337046, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 2 - }, - { - "model_tag": "climax", - "label": "ClimaX", - "seed": 99, - "scope": "train_fire_top05pct", - "threshold_key": "0.00061661656", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.0006166165592580956, - "strict_f1": 0.013585576434738445, - "tolerance_f1": 0.3665570668664193, - "union_f1": 0.7408743604065289, - "difference": 0.7272887839717904, - "predicted_positive_rate": 0.3090951815573365, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 2 - }, - { - "model_tag": "climax", - "label": "ClimaX", - "seed": 99, - "scope": "train_fire_top10pct", - "threshold_key": "0.00062947387", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.0006294738670532621, - "strict_f1": 0.013497596536465794, - "tolerance_f1": 0.36524448333846904, - "union_f1": 0.7365490884872157, - "difference": 0.7230514919507499, - "predicted_positive_rate": 0.15408504007123774, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 2 - }, - { - "model_tag": "pangu_weather", - "label": "Pangu-Weather", - "seed": 1, - "scope": "full_domain", - "threshold_key": "0.00027441306", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.00027441306156106293, - "strict_f1": 0.0005518596739603732, - "tolerance_f1": 0.06046181002685303, - "union_f1": 0.16301936607754167, - "difference": 0.1624675064035813, - "predicted_positive_rate": 0.21222844774273344, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 2 - }, - { - "model_tag": "pangu_weather", - "label": "Pangu-Weather", - "seed": 1, - "scope": "train_fire_top05pct", - "threshold_key": "0.0002743864", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.00027438640245236456, - "strict_f1": 0.026612927125626056, - "tolerance_f1": 0.24179585238367046, - "union_f1": 0.4864922341150057, - "difference": 0.4598793069893796, - "predicted_positive_rate": 0.08085485307212822, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 2 - }, - { - "model_tag": "pangu_weather", - "label": "Pangu-Weather", - "seed": 1, - "scope": "train_fire_top10pct", - "threshold_key": "0.0002743864", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.00027438640245236456, - "strict_f1": 0.02660504251473943, - "tolerance_f1": 0.2411933776075557, - "union_f1": 0.4840644650115095, - "difference": 0.4574594224967701, - "predicted_positive_rate": 0.04042742653606411, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 2 - }, - { - "model_tag": "pangu_weather", - "label": "Pangu-Weather", - "seed": 123, - "scope": "full_domain", - "threshold_key": "0.0002703598", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.0002703598001971841, - "strict_f1": 0.0006508233558970649, - "tolerance_f1": 0.09112517773927671, - "union_f1": 0.2457493414602635, - "difference": 0.24509851810436645, - "predicted_positive_rate": 0.7109064935064935, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 2 - }, - { - "model_tag": "pangu_weather", - "label": "Pangu-Weather", - "seed": 123, - "scope": "train_fire_top05pct", - "threshold_key": "0.00026974134", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.0002697413437999785, - "strict_f1": 0.005639067545640032, - "tolerance_f1": 0.16012924166858108, - "union_f1": 0.3947594149924353, - "difference": 0.38912034744679525, - "predicted_positive_rate": 0.9211709706144257, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 2 - }, - { - "model_tag": "pangu_weather", - "label": "Pangu-Weather", - "seed": 123, - "scope": "train_fire_top10pct", - "threshold_key": "0.00026974134", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.0002697413437999785, - "strict_f1": 0.0027453654202759313, - "tolerance_f1": 0.08722526453259725, - "union_f1": 0.23115112381331424, - "difference": 0.2284057583930383, - "predicted_positive_rate": 0.9564299495399228, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 2 - }, - { - "model_tag": "pangu_weather", - "label": "Pangu-Weather", - "seed": 42, - "scope": "full_domain", - "threshold_key": "0.00024381682", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.00024381681578233838, - "strict_f1": 0.0005395660023552084, - "tolerance_f1": 0.07610963330033767, - "union_f1": 0.20220259920022512, - "difference": 0.2016630331978699, - "predicted_positive_rate": 0.8827594310451453, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 2 - }, - { - "model_tag": "pangu_weather", - "label": "Pangu-Weather", - "seed": 42, - "scope": "train_fire_top05pct", - "threshold_key": "0.00024314615", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.0002431461471132934, - "strict_f1": 0.00568010509540423, - "tolerance_f1": 0.16063421440737302, - "union_f1": 0.3925140767903833, - "difference": 0.3868339716949791, - "predicted_positive_rate": 0.9162387454239636, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 2 - }, - { - "model_tag": "pangu_weather", - "label": "Pangu-Weather", - "seed": 42, - "scope": "train_fire_top10pct", - "threshold_key": "0.00024276716", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.00024276715703308582, - "strict_f1": 0.002750953470140531, - "tolerance_f1": 0.08724283500251707, - "union_f1": 0.23040938884616793, - "difference": 0.2276584353760274, - "predicted_positive_rate": 0.9562827743148313, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 2 - }, - { - "model_tag": "pangu_weather", - "label": "Pangu-Weather", - "seed": 7, - "scope": "full_domain", - "threshold_key": "0.00027848675", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.0002784867538139224, - "strict_f1": 0.0006577494583008349, - "tolerance_f1": 0.09170559714407109, - "union_f1": 0.2480828253168327, - "difference": 0.24742507585853188, - "predicted_positive_rate": 0.7071789734075449, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 2 - }, - { - "model_tag": "pangu_weather", - "label": "Pangu-Weather", - "seed": 7, - "scope": "train_fire_top05pct", - "threshold_key": "0.00027803043", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.0002780304348561913, - "strict_f1": 0.005648734389013078, - "tolerance_f1": 0.1601110414083686, - "union_f1": 0.39479065437879374, - "difference": 0.38914191998978065, - "predicted_positive_rate": 0.9213416444048679, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 2 - }, - { - "model_tag": "pangu_weather", - "label": "Pangu-Weather", - "seed": 7, - "scope": "train_fire_top10pct", - "threshold_key": "0.00027803043", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.0002780304348561913, - "strict_f1": 0.00275028568108307, - "tolerance_f1": 0.08722255267001126, - "union_f1": 0.23118547623512836, - "difference": 0.2284351905540453, - "predicted_positive_rate": 0.9565152864351439, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 2 - }, - { - "model_tag": "pangu_weather", - "label": "Pangu-Weather", - "seed": 99, - "scope": "full_domain", - "threshold_key": "0.00027882602", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.00027882601716555655, - "strict_f1": 0.000687991383709005, - "tolerance_f1": 0.08961425169301492, - "union_f1": 0.24375811032725037, - "difference": 0.24307011894354136, - "predicted_positive_rate": 0.655947804576376, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 2 - }, - { - "model_tag": "pangu_weather", - "label": "Pangu-Weather", - "seed": 99, - "scope": "train_fire_top05pct", - "threshold_key": "0.0002815545", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.00028155450127087533, - "strict_f1": 0.009651904247534756, - "tolerance_f1": 0.1110438397786047, - "union_f1": 0.2883417650922405, - "difference": 0.27868986084470576, - "predicted_positive_rate": 0.28134213911150685, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 2 - }, - { - "model_tag": "pangu_weather", - "label": "Pangu-Weather", - "seed": 99, - "scope": "train_fire_top10pct", - "threshold_key": "0.00028572054", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.000285720539977774, - "strict_f1": 0.0011827321111768183, - "tolerance_f1": 0.019439944457301548, - "union_f1": 0.029367206619275145, - "difference": 0.02818447450809833, - "predicted_positive_rate": 0.0007742158899772435, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 2 - }, - { - "model_tag": "prithvi_wxc", - "label": "Prithvi-WxC", - "seed": 1, - "scope": "full_domain", - "threshold_key": "0.00028278783", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.00028278783429414034, - "strict_f1": 0.0006040339888888368, - "tolerance_f1": 0.07892160427759505, - "union_f1": 0.22218157532367563, - "difference": 0.2215775413347868, - "predicted_positive_rate": 0.5006159554730983, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 2 - }, - { - "model_tag": "prithvi_wxc", - "label": "Prithvi-WxC", - "seed": 1, - "scope": "train_fire_top05pct", - "threshold_key": "0.0001852147", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.00018521470337873325, - "strict_f1": 0.005636060100166944, - "tolerance_f1": 0.15977956000729543, - "union_f1": 0.39407737525514647, - "difference": 0.38844131515497954, - "predicted_positive_rate": 0.9234194122885129, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 2 - }, - { - "model_tag": "prithvi_wxc", - "label": "Prithvi-WxC", - "seed": 1, - "scope": "train_fire_top10pct", - "threshold_key": "0.0001852147", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.00018521470337873325, - "strict_f1": 0.002747305898968793, - "tolerance_f1": 0.08713204456539597, - "union_f1": 0.2309636203280157, - "difference": 0.2282163144290469, - "predicted_positive_rate": 0.9575541703769664, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 2 - }, - { - "model_tag": "prithvi_wxc", - "label": "Prithvi-WxC", - "seed": 123, - "scope": "full_domain", - "threshold_key": "0.00028123806", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.00028123805532231927, - "strict_f1": 0.0005057825179019543, - "tolerance_f1": 0.07212988048730175, - "union_f1": 0.20070659371418467, - "difference": 0.20020081119628272, - "predicted_positive_rate": 0.9588571428571429, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 2 - }, - { - "model_tag": "prithvi_wxc", - "label": "Prithvi-WxC", - "seed": 123, - "scope": "train_fire_top05pct", - "threshold_key": "0.0002438416", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.0002438416049699299, - "strict_f1": 0.005636060100166944, - "tolerance_f1": 0.15977956000729543, - "union_f1": 0.39407737525514647, - "difference": 0.38844131515497954, - "predicted_positive_rate": 0.9234194122885129, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 2 - }, - { - "model_tag": "prithvi_wxc", - "label": "Prithvi-WxC", - "seed": 123, - "scope": "train_fire_top10pct", - "threshold_key": "0.0002438416", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.0002438416049699299, - "strict_f1": 0.002747305898968793, - "tolerance_f1": 0.08713204456539597, - "union_f1": 0.2309636203280157, - "difference": 0.2282163144290469, - "predicted_positive_rate": 0.9575541703769664, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 2 - }, - { - "model_tag": "prithvi_wxc", - "label": "Prithvi-WxC", - "seed": 42, - "scope": "full_domain", - "threshold_key": "0.00028831177", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.00028831177041865885, - "strict_f1": 0.0005777939978365554, - "tolerance_f1": 0.06182533987250242, - "union_f1": 0.17269452770359542, - "difference": 0.17211673370575886, - "predicted_positive_rate": 0.22581076066790354, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 2 - }, - { - "model_tag": "prithvi_wxc", - "label": "Prithvi-WxC", - "seed": 42, - "scope": "train_fire_top05pct", - "threshold_key": "0.00028831148", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.0002883114793803543, - "strict_f1": 0.02590687977762335, - "tolerance_f1": 0.23967146506700618, - "union_f1": 0.4925384357395131, - "difference": 0.46663155596188977, - "predicted_positive_rate": 0.08637577916295637, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 2 - }, - { - "model_tag": "prithvi_wxc", - "label": "Prithvi-WxC", - "seed": 42, - "scope": "train_fire_top10pct", - "threshold_key": "0.00028831148", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.0002883114793803543, - "strict_f1": 0.02589968042239822, - "tolerance_f1": 0.23908194736539978, - "union_f1": 0.4900551948025372, - "difference": 0.46415551438013897, - "predicted_positive_rate": 0.043187889581478185, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 2 - }, - { - "model_tag": "prithvi_wxc", - "label": "Prithvi-WxC", - "seed": 7, - "scope": "full_domain", - "threshold_key": "0.0002881609", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.00028816089616157115, - "strict_f1": 0.000532941187181528, - "tolerance_f1": 0.07586161173658511, - "union_f1": 0.21037180175292197, - "difference": 0.20983886056574044, - "predicted_positive_rate": 0.9099814471243043, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 2 - }, - { - "model_tag": "prithvi_wxc", - "label": "Prithvi-WxC", - "seed": 7, - "scope": "train_fire_top05pct", - "threshold_key": "0.00026064563", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.0002606456328066997, - "strict_f1": 0.005636060100166944, - "tolerance_f1": 0.15977956000729543, - "union_f1": 0.39407737525514647, - "difference": 0.38844131515497954, - "predicted_positive_rate": 0.9234194122885129, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 2 - }, - { - "model_tag": "prithvi_wxc", - "label": "Prithvi-WxC", - "seed": 7, - "scope": "train_fire_top10pct", - "threshold_key": "0.00026064563", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.0002606456328066997, - "strict_f1": 0.002747305898968793, - "tolerance_f1": 0.08713204456539597, - "union_f1": 0.2309636203280157, - "difference": 0.2282163144290469, - "predicted_positive_rate": 0.9575541703769664, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 2 - }, - { - "model_tag": "prithvi_wxc", - "label": "Prithvi-WxC", - "seed": 99, - "scope": "full_domain", - "threshold_key": "0.00026065187", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.0002606518683023751, - "strict_f1": 0.0005418761442097979, - "tolerance_f1": 0.06950848282026648, - "union_f1": 0.2033106164921145, - "difference": 0.20276874034790468, - "predicted_positive_rate": 0.5032875695732839, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 2 - }, - { - "model_tag": "prithvi_wxc", - "label": "Prithvi-WxC", - "seed": 99, - "scope": "train_fire_top05pct", - "threshold_key": "0.00026065261", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.0002606526104500517, - "strict_f1": 0.027781926811053025, - "tolerance_f1": 0.24416957191197788, - "union_f1": 0.4541949852031399, - "difference": 0.4264130583920869, - "predicted_positive_rate": 0.013950727218759276, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 2 - }, - { - "model_tag": "prithvi_wxc", - "label": "Prithvi-WxC", - "seed": 99, - "scope": "train_fire_top10pct", - "threshold_key": "0.00026065261", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.0002606526104500517, - "strict_f1": 0.02774049217002237, - "tolerance_f1": 0.24342288270264262, - "union_f1": 0.451618067346158, - "difference": 0.4238775751761356, - "predicted_positive_rate": 0.006975363609379638, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 2 - }, - { - "model_tag": "stormcast", - "label": "StormCast", - "seed": 1, - "scope": "full_domain", - "threshold_key": "0.00030108562", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.000301085616229102, - "strict_f1": 0.00043296285460247506, - "tolerance_f1": 0.04396525865017935, - "union_f1": 0.12984783035809752, - "difference": 0.12941486750349504, - "predicted_positive_rate": 0.3248534322820037, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 2 - }, - { - "model_tag": "stormcast", - "label": "StormCast", - "seed": 1, - "scope": "train_fire_top05pct", - "threshold_key": "0.00030108562", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.000301085616229102, - "strict_f1": 0.023876731839958566, - "tolerance_f1": 0.23852766585861737, - "union_f1": 0.49125319073865004, - "difference": 0.46737645889869145, - "predicted_positive_rate": 0.09290590679726922, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 2 - }, - { - "model_tag": "stormcast", - "label": "StormCast", - "seed": 1, - "scope": "train_fire_top10pct", - "threshold_key": "0.00030108562", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.000301085616229102, - "strict_f1": 0.023870550161812297, - "tolerance_f1": 0.23793892088542526, - "union_f1": 0.4887624633565217, - "difference": 0.4648919131947094, - "predicted_positive_rate": 0.04645295339863461, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 2 - }, - { - "model_tag": "stormcast", - "label": "StormCast", - "seed": 123, - "scope": "full_domain", - "threshold_key": "0.00027319475", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.0002731947461143136, - "strict_f1": 0.0005917233172122231, - "tolerance_f1": 0.08255863031622387, - "union_f1": 0.22452793532172965, - "difference": 0.22393621200451744, - "predicted_positive_rate": 0.7873692022263451, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 2 - }, - { - "model_tag": "stormcast", - "label": "StormCast", - "seed": 123, - "scope": "train_fire_top05pct", - "threshold_key": "0.00027169287", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.0002716928720474243, - "strict_f1": 0.005636060100166944, - "tolerance_f1": 0.15977956000729543, - "union_f1": 0.39407737525514647, - "difference": 0.38844131515497954, - "predicted_positive_rate": 0.9234194122885129, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 2 - }, - { - "model_tag": "stormcast", - "label": "StormCast", - "seed": 123, - "scope": "train_fire_top10pct", - "threshold_key": "0.00027169287", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.0002716928720474243, - "strict_f1": 0.002747305898968793, - "tolerance_f1": 0.08713204456539597, - "union_f1": 0.2309636203280157, - "difference": 0.2282163144290469, - "predicted_positive_rate": 0.9575541703769664, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 2 - }, - { - "model_tag": "stormcast", - "label": "StormCast", - "seed": 42, - "scope": "full_domain", - "threshold_key": "0.00027820465", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.0002782046503853053, - "strict_f1": 0.0007179477702826634, - "tolerance_f1": 0.09668169596405224, - "union_f1": 0.2609912375807996, - "difference": 0.26027328981051695, - "predicted_positive_rate": 0.62891280148423, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 2 - }, - { - "model_tag": "stormcast", - "label": "StormCast", - "seed": 42, - "scope": "train_fire_top05pct", - "threshold_key": "0.00027877578", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.00027877578395418823, - "strict_f1": 0.006835665824509611, - "tolerance_f1": 0.10250537509480662, - "union_f1": 0.25743196970586, - "difference": 0.2505963038813504, - "predicted_positive_rate": 0.43306619174829325, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 2 - }, - { - "model_tag": "stormcast", - "label": "StormCast", - "seed": 42, - "scope": "train_fire_top10pct", - "threshold_key": "0.00027818422", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.00027818421949632466, - "strict_f1": 0.00356800070444239, - "tolerance_f1": 0.1112312688602769, - "union_f1": 0.2876668872191355, - "difference": 0.2840988865146931, - "predicted_positive_rate": 0.7009498367468091, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 2 - }, - { - "model_tag": "stormcast", - "label": "StormCast", - "seed": 7, - "scope": "full_domain", - "threshold_key": "0.00028929155", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.0002892915508709848, - "strict_f1": 0.0006714969233599232, - "tolerance_f1": 0.09264506128556889, - "union_f1": 0.25008945924787185, - "difference": 0.24941796232451194, - "predicted_positive_rate": 0.7037476808905381, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 2 - }, - { - "model_tag": "stormcast", - "label": "StormCast", - "seed": 7, - "scope": "train_fire_top05pct", - "threshold_key": "0.00028836745", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.0002883674460463226, - "strict_f1": 0.005648734389013078, - "tolerance_f1": 0.1601110414083686, - "union_f1": 0.39479065437879374, - "difference": 0.38914191998978065, - "predicted_positive_rate": 0.9213416444048679, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 2 - }, - { - "model_tag": "stormcast", - "label": "StormCast", - "seed": 7, - "scope": "train_fire_top10pct", - "threshold_key": "0.00028836745", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.0002883674460463226, - "strict_f1": 0.00275028568108307, - "tolerance_f1": 0.08722255267001126, - "union_f1": 0.23118547623512836, - "difference": 0.2284351905540453, - "predicted_positive_rate": 0.9565152864351439, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 2 - }, - { - "model_tag": "stormcast", - "label": "StormCast", - "seed": 99, - "scope": "full_domain", - "threshold_key": "0.00028191711", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.0002819171058945358, - "strict_f1": 0.0007136363665052921, - "tolerance_f1": 0.09390595738710937, - "union_f1": 0.253626654686362, - "difference": 0.2529130183198567, - "predicted_positive_rate": 0.5876512059369202, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 2 - }, - { - "model_tag": "stormcast", - "label": "StormCast", - "seed": 99, - "scope": "train_fire_top05pct", - "threshold_key": "0.00028293603", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.000282936030998826, - "strict_f1": 0.005865442686314773, - "tolerance_f1": 0.10517133941552075, - "union_f1": 0.2717309409136105, - "difference": 0.2658654982272957, - "predicted_positive_rate": 0.5093499554764025, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 2 - }, - { - "model_tag": "stormcast", - "label": "StormCast", - "seed": 99, - "scope": "train_fire_top10pct", - "threshold_key": "0.00028128058", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.00028128057601861656, - "strict_f1": 0.003485751823118051, - "tolerance_f1": 0.10982263054722373, - "union_f1": 0.28516330859718164, - "difference": 0.2816775567740636, - "predicted_positive_rate": 0.7423567824280202, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 2 - }, - { - "model_tag": "aurora", - "label": "Aurora", - "seed": 1, - "scope": "train_fire_top20pct", - "threshold_key": "0.00028478092", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.0002847809228114784, - "strict_f1": 0.026403975040443728, - "tolerance_f1": 0.23725772191057362, - "union_f1": 0.4711352810381827, - "difference": 0.44473130599773897, - "predicted_positive_rate": 0.02069016697588126, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 3 - }, - { - "model_tag": "aurora", - "label": "Aurora", - "seed": 123, - "scope": "train_fire_top20pct", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.0002, - "strict_f1": 0.001471035812113219, - "tolerance_f1": 0.06246138809231819, - "union_f1": 0.17013954201238674, - "difference": 0.16866850620027352, - "predicted_positive_rate": 0.9746196660482375, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 3 - }, - { - "model_tag": "aurora", - "label": "Aurora", - "seed": 42, - "scope": "train_fire_top20pct", - "threshold_key": "0.00028130069", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.00028130068676546216, - "strict_f1": 0.00196209796760366, - "tolerance_f1": 0.08141778117479781, - "union_f1": 0.21602191083593253, - "difference": 0.21405981286832887, - "predicted_positive_rate": 0.7109764996907854, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 3 - }, - { - "model_tag": "aurora", - "label": "Aurora", - "seed": 7, - "scope": "train_fire_top20pct", - "threshold_key": "0.00027013358", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.0002701335761230439, - "strict_f1": 0.0014870516331508276, - "tolerance_f1": 0.06425881062262738, - "union_f1": 0.1742547495584396, - "difference": 0.17276769792528876, - "predicted_positive_rate": 0.9366672850958565, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 3 - }, - { - "model_tag": "aurora", - "label": "Aurora", - "seed": 99, - "scope": "train_fire_top20pct", - "threshold_key": "0.00028269363", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.0002826936251949519, - "strict_f1": 0.0019489171350575599, - "tolerance_f1": 0.08112237765113199, - "union_f1": 0.21567017057663868, - "difference": 0.2137212534415811, - "predicted_positive_rate": 0.7215015460729747, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 3 - }, - { - "model_tag": "climax", - "label": "ClimaX", - "seed": 1, - "scope": "train_fire_top20pct", - "threshold_key": "0.0015083512", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.0015083511573617753, - "strict_f1": 0.011488825936418005, - "tolerance_f1": 0.30575189275627285, - "union_f1": 0.6099212745225361, - "difference": 0.5984324485861181, - "predicted_positive_rate": 0.030718614718614718, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 3 - }, - { - "model_tag": "climax", - "label": "ClimaX", - "seed": 123, - "scope": "train_fire_top20pct", - "threshold_key": "0.00068209981", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.0006820998141378855, - "strict_f1": 0.006734646906365392, - "tolerance_f1": 0.2833418180729231, - "union_f1": 0.5895764795021311, - "difference": 0.5828418325957657, - "predicted_positive_rate": 0.12288311688311689, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 3 - }, - { - "model_tag": "climax", - "label": "ClimaX", - "seed": 42, - "scope": "train_fire_top20pct", - "threshold_key": "0.0013755694", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.0013755693906898965, - "strict_f1": 0.013757803485310216, - "tolerance_f1": 0.32901058451573606, - "union_f1": 0.6288351083076325, - "difference": 0.6150773048223223, - "predicted_positive_rate": 0.041087198515769946, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 3 - }, - { - "model_tag": "climax", - "label": "ClimaX", - "seed": 7, - "scope": "train_fire_top20pct", - "threshold_key": "0.0013572138", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.0013572138352511622, - "strict_f1": 0.010760193101843772, - "tolerance_f1": 0.24066482674710968, - "union_f1": 0.48364281191756603, - "difference": 0.47288261881572224, - "predicted_positive_rate": 0.020547928262213976, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 3 - }, - { - "model_tag": "climax", - "label": "ClimaX", - "seed": 99, - "scope": "train_fire_top20pct", - "threshold_key": "0.00096331563", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.0009633156281329466, - "strict_f1": 0.008694912370256934, - "tolerance_f1": 0.35193063179186285, - "union_f1": 0.691273953290052, - "difference": 0.6825790409197952, - "predicted_positive_rate": 0.07197278911564625, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 3 - }, - { - "model_tag": "pangu_weather", - "label": "Pangu-Weather", - "seed": 1, - "scope": "train_fire_top20pct", - "threshold_key": "0.0002743864", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.00027438640245236456, - "strict_f1": 0.026530371070668874, - "tolerance_f1": 0.23561614605975847, - "union_f1": 0.4621544722787787, - "difference": 0.43562410120810985, - "predicted_positive_rate": 0.0202152133580705, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 3 - }, - { - "model_tag": "pangu_weather", - "label": "Pangu-Weather", - "seed": 123, - "scope": "train_fire_top20pct", - "threshold_key": "0.00023001364", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.0002300136402482167, - "strict_f1": 0.0014649840398829276, - "tolerance_f1": 0.062212235501567396, - "union_f1": 0.16949844621797366, - "difference": 0.16803346217809073, - "predicted_positive_rate": 0.9786487322201608, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 3 - }, - { - "model_tag": "pangu_weather", - "label": "Pangu-Weather", - "seed": 42, - "scope": "train_fire_top20pct", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.0002, - "strict_f1": 0.0014680041534393378, - "tolerance_f1": 0.062336581960243484, - "union_f1": 0.16981843825492582, - "difference": 0.16835043410148648, - "predicted_positive_rate": 0.9766338899196042, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 3 - }, - { - "model_tag": "pangu_weather", - "label": "Pangu-Weather", - "seed": 7, - "scope": "train_fire_top20pct", - "threshold_key": "0.00023887513", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.00023887513088993728, - "strict_f1": 0.0014589084597825474, - "tolerance_f1": 0.06196203978961812, - "union_f1": 0.1688543802092876, - "difference": 0.16739547174950506, - "predicted_positive_rate": 0.9827272727272728, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 3 - }, - { - "model_tag": "pangu_weather", - "label": "Pangu-Weather", - "seed": 99, - "scope": "train_fire_top20pct", - "threshold_key": "0.00028135377", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.00028135377215221524, - "strict_f1": 0.0021531819937041672, - "tolerance_f1": 0.031681254572138745, - "union_f1": 0.09062518509001295, - "difference": 0.08847200309630879, - "predicted_positive_rate": 0.34681447124304265, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 3 - }, - { - "model_tag": "prithvi_wxc", - "label": "Prithvi-WxC", - "seed": 1, - "scope": "train_fire_top20pct", - "threshold_key": "0.0002", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.0002, - "strict_f1": 0.001471035812113219, - "tolerance_f1": 0.06246138809231819, - "union_f1": 0.17013954201238674, - "difference": 0.16866850620027352, - "predicted_positive_rate": 0.9746196660482375, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 3 - }, - { - "model_tag": "prithvi_wxc", - "label": "Prithvi-WxC", - "seed": 123, - "scope": "train_fire_top20pct", - "threshold_key": "0.00024384175", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.0002438417504890822, - "strict_f1": 0.001471035812113219, - "tolerance_f1": 0.06246138809231819, - "union_f1": 0.17013954201238674, - "difference": 0.16866850620027352, - "predicted_positive_rate": 0.9746196660482375, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 3 - }, - { - "model_tag": "prithvi_wxc", - "label": "Prithvi-WxC", - "seed": 42, - "scope": "train_fire_top20pct", - "threshold_key": "0.00028831148", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.0002883114793803543, - "strict_f1": 0.025831485587583147, - "tolerance_f1": 0.23362286716686329, - "union_f1": 0.4676561751411253, - "difference": 0.44182468955354215, - "predicted_positive_rate": 0.021595547309833023, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 3 - }, - { - "model_tag": "prithvi_wxc", - "label": "Prithvi-WxC", - "seed": 7, - "scope": "train_fire_top20pct", - "threshold_key": "0.00026064578", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.000260645778325852, - "strict_f1": 0.001471035812113219, - "tolerance_f1": 0.06246138809231819, - "union_f1": 0.17013954201238674, - "difference": 0.16866850620027352, - "predicted_positive_rate": 0.9746196660482375, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 3 - }, - { - "model_tag": "prithvi_wxc", - "label": "Prithvi-WxC", - "seed": 99, - "scope": "train_fire_top20pct", - "threshold_key": "0.00026065261", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.0002606526104500517, - "strict_f1": 0.027352941176470587, - "tolerance_f1": 0.2365506772536198, - "union_f1": 0.4285211456102784, - "difference": 0.4011682044338078, - "predicted_positive_rate": 0.0034879406307977735, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 3 - }, - { - "model_tag": "stormcast", - "label": "StormCast", - "seed": 1, - "scope": "train_fire_top20pct", - "threshold_key": "0.0003011123", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.00030111230444163084, - "strict_f1": 0.022074159907300116, - "tolerance_f1": 0.23443811051591282, - "union_f1": 0.4646063802629719, - "difference": 0.4425322203556718, - "predicted_positive_rate": 0.020630797773654917, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 3 - }, - { - "model_tag": "stormcast", - "label": "StormCast", - "seed": 123, - "scope": "train_fire_top20pct", - "threshold_key": "0.00024970324", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.0002497032401151955, - "strict_f1": 0.0014637962799384197, - "tolerance_f1": 0.062163327857870446, - "union_f1": 0.16937256832950737, - "difference": 0.16790877204956894, - "predicted_positive_rate": 0.979443413729128, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 3 - }, - { - "model_tag": "stormcast", - "label": "StormCast", - "seed": 42, - "scope": "train_fire_top20pct", - "threshold_key": "0.00027813489", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.0002781348885037005, - "strict_f1": 0.0020248273888312522, - "tolerance_f1": 0.08079366172039935, - "union_f1": 0.21367238965986382, - "difference": 0.21164756227103257, - "predicted_positive_rate": 0.6907606679035251, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 3 - }, - { - "model_tag": "stormcast", - "label": "StormCast", - "seed": 7, - "scope": "train_fire_top20pct", - "threshold_key": "0.00021927932", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.00021927931811660528, - "strict_f1": 0.0014637962799384197, - "tolerance_f1": 0.062163327857870446, - "union_f1": 0.16937256832950737, - "difference": 0.16790877204956894, - "predicted_positive_rate": 0.979443413729128, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 3 - }, - { - "model_tag": "stormcast", - "label": "StormCast", - "seed": 99, - "scope": "train_fire_top20pct", - "threshold_key": "0.00028123951", - "threshold_selector": "strict_f1_nondegenerate", - "threshold": 0.0002812395105138421, - "strict_f1": 0.001948363237912615, - "tolerance_f1": 0.08122801407099456, - "union_f1": 0.21596595812335517, - "difference": 0.21401759488544256, - "predicted_positive_rate": 0.7217068645640075, - "tolerance_combo": "t0_s8", - "union_combo": "t3_s8", - "path": "", - "source_priority": 3 - } - ], - "summary": [ - { - "model_tag": "alphaearth", - "label": "AlphaEarth", - "scope": "full_domain", - "scope_label": "global", - "n": 5, - "strict_f1": { - "n": 5, - "mean": 0.02060627238126987, - "std": 0.004404166549372983, - "min": 0.015582982831138383, - "max": 0.02585193889541716 - }, - "tolerance_f1": { - "n": 5, - "mean": 0.2944756804111831, - "std": 0.06006423929515853, - "min": 0.19137568834619276, - "max": 0.3412019506417814 - }, - "union_f1": { - "n": 5, - "mean": 0.3742857130975682, - "std": 0.09945849036660906, - "min": 0.21572734820321318, - "max": 0.4879949976282558 - }, - "difference": { - "n": 5, - "mean": 0.3536794407162983, - "std": 0.10027055938000065, - "min": 0.1971458843464264, - "max": 0.4724120147971174 - }, - "predicted_positive_rate": { - "n": 5, - "mean": 0.000691230674087817, - "std": 0.0004992791583911452, - "min": 0.00030327767470624613, - "max": 0.0015512677798392085 - }, - "seed_values": { - "strict_f1": [ - { - "seed": 1, - "value": 0.01841620626151013, - "path": "" - }, - { - "seed": 7, - "value": 0.018581463856786766, - "path": "" - }, - { - "seed": 42, - "value": 0.024598770061496925, - "path": "" - }, - { - "seed": 99, - "value": 0.02585193889541716, - "path": "" - }, - { - "seed": 123, - "value": 0.015582982831138383, - "path": "" - } - ], - "tolerance_f1": [ - { - "seed": 1, - "value": 0.3233671389763082, - "path": "" - }, - { - "seed": 7, - "value": 0.19137568834619276, - "path": "" - }, - { - "seed": 42, - "value": 0.3224566312114399, - "path": "" - }, - { - "seed": 99, - "value": 0.29397699288019324, - "path": "" - }, - { - "seed": 123, - "value": 0.3412019506417814, - "path": "" - } - ], - "union_f1": [ - { - "seed": 1, - "value": 0.4075802286978974, - "path": "" - }, - { - "seed": 7, - "value": 0.21572734820321318, - "path": "" - }, - { - "seed": 42, - "value": 0.3927541342199395, - "path": "" - }, - { - "seed": 99, - "value": 0.36737185673853523, - "path": "" - }, - { - "seed": 123, - "value": 0.4879949976282558, - "path": "" - } - ], - "difference": [ - { - "seed": 1, - "value": 0.38916402243638726, - "path": "" - }, - { - "seed": 7, - "value": 0.1971458843464264, - "path": "" - }, - { - "seed": 42, - "value": 0.3681553641584426, - "path": "" - }, - { - "seed": 99, - "value": 0.3415199178431181, - "path": "" - }, - { - "seed": 123, - "value": 0.4724120147971174, - "path": "" - } - ], - "predicted_positive_rate": [ - { - "seed": 1, - "value": 0.0006305504019789734, - "path": "" - }, - { - "seed": 7, - "value": 0.00030327767470624613, - "path": "" - }, - { - "seed": 42, - "value": 0.0005820655534941249, - "path": "" - }, - { - "seed": 99, - "value": 0.00038899196042053183, - "path": "" - }, - { - "seed": 123, - "value": 0.0015512677798392085, - "path": "" - } - ] - }, - "thresholds": [ - 0.005, - 0.01, - 0.02 - ], - "paths": [ - "", - "", - "", - "", - "" - ] - }, - { - "model_tag": "alphaearth", - "label": "AlphaEarth", - "scope": "train_fire_top05pct", - "scope_label": "top 5%", - "n": 5, - "strict_f1": { - "n": 5, - "mean": 0.06913347589828964, - "std": 0.008450070468719153, - "min": 0.05801376597836775, - "max": 0.07774936061381074 - }, - "tolerance_f1": { - "n": 5, - "mean": 0.42878982650540687, - "std": 0.04608715441056064, - "min": 0.348489928787613, - "max": 0.46103410735073025 - }, - "union_f1": { - "n": 5, - "mean": 0.5174486091799061, - "std": 0.08732060818913379, - "min": 0.39499846060642174, - "max": 0.6362946372146648 - }, - "difference": { - "n": 5, - "mean": 0.44831513328161643, - "std": 0.09076276520267298, - "min": 0.32645966955454164, - "max": 0.572037609102215 - }, - "predicted_positive_rate": { - "n": 5, - "mean": 0.002825764321757198, - "std": 0.0014965985811042732, - "min": 0.001496487582863362, - "max": 0.005397249431087365 - }, - "seed_values": { - "strict_f1": [ - { - "seed": 1, - "value": 0.05801376597836775, - "path": "" - }, - { - "seed": 7, - "value": 0.06853879105188007, - "path": "" - }, - { - "seed": 42, - "value": 0.07774936061381074, - "path": "" - }, - { - "seed": 99, - "value": 0.07710843373493977, - "path": "" - }, - { - "seed": 123, - "value": 0.06425702811244981, - "path": "" - } - ], - "tolerance_f1": [ - { - "seed": 1, - "value": 0.45453465315678554, - "path": "" - }, - { - "seed": 7, - "value": 0.348489928787613, - "path": "" - }, - { - "seed": 42, - "value": 0.46103410735073025, - "path": "" - }, - { - "seed": 99, - "value": 0.4329934518241347, - "path": "" - }, - { - "seed": 123, - "value": 0.44689699140777084, - "path": "" - } - ], - "union_f1": [ - { - "seed": 1, - "value": 0.5487719544302176, - "path": "" - }, - { - "seed": 7, - "value": 0.39499846060642174, - "path": "" - }, - { - "seed": 42, - "value": 0.5064003599986154, - "path": "" - }, - { - "seed": 99, - "value": 0.5007776336496106, - "path": "" - }, - { - "seed": 123, - "value": 0.6362946372146648, - "path": "" - } - ], - "difference": [ - { - "seed": 1, - "value": 0.4907581884518499, - "path": "" - }, - { - "seed": 7, - "value": 0.32645966955454164, - "path": "" - }, - { - "seed": 42, - "value": 0.42865099938480467, - "path": "" - }, - { - "seed": 99, - "value": 0.42366919991467084, - "path": "" - }, - { - "seed": 123, - "value": 0.572037609102215, - "path": "" - } - ], - "predicted_positive_rate": [ - { - "seed": 1, - "value": 0.0024215889977243494, - "path": "" - }, - { - "seed": 7, - "value": 0.0025873157217769865, - "path": "" - }, - { - "seed": 42, - "value": 0.002226179875333927, - "path": "" - }, - { - "seed": 99, - "value": 0.001496487582863362, - "path": "" - }, - { - "seed": 123, - "value": 0.005397249431087365, - "path": "" - } - ] - }, - "thresholds": [ - 0.005, - 0.01 - ], - "paths": [ - "", - "", - "", - "", - "" - ] - }, - { - "model_tag": "alphaearth", - "label": "AlphaEarth", - "scope": "train_fire_top10pct", - "scope_label": "top 10%", - "n": 5, - "strict_f1": { - "n": 5, - "mean": 0.0663656378441958, - "std": 0.009901307723431575, - "min": 0.05725646123260437, - "max": 0.07711821410451548 - }, - "tolerance_f1": { - "n": 5, - "mean": 0.41898143755043915, - "std": 0.05945364415661427, - "min": 0.31417824074074074, - "max": 0.4570756272994282 - }, - "union_f1": { - "n": 5, - "mean": 0.5057119561083213, - "std": 0.10005714147402384, - "min": 0.3547275267815556, - "max": 0.6310773342718137 - }, - "difference": { - "n": 5, - "mean": 0.43934631826412546, - "std": 0.0991562015356911, - "min": 0.29747106554895125, - "max": 0.5674882177936724 - }, - "predicted_positive_rate": { - "n": 5, - "mean": 0.001524438507964777, - "std": 0.0007698172196033509, - "min": 0.0007507173246264965, - "max": 0.002728307113881468 - }, - "seed_values": { - "strict_f1": [ - { - "seed": 1, - "value": 0.05730937348227295, - "path": "" - }, - { - "seed": 7, - "value": 0.05725646123260437, - "path": "" - }, - { - "seed": 42, - "value": 0.07711821410451548, - "path": "" - }, - { - "seed": 99, - "value": 0.07655502392344497, - "path": "" - }, - { - "seed": 123, - "value": 0.06358911647814125, - "path": "" - } - ], - "tolerance_f1": [ - { - "seed": 1, - "value": 0.4503813713224908, - "path": "" - }, - { - "seed": 7, - "value": 0.31417824074074074, - "path": "" - }, - { - "seed": 42, - "value": 0.4570756272994282, - "path": "" - }, - { - "seed": 99, - "value": 0.4297532170909699, - "path": "" - }, - { - "seed": 123, - "value": 0.4435187312985661, - "path": "" - } - ], - "union_f1": [ - { - "seed": 1, - "value": 0.5437059200683907, - "path": "" - }, - { - "seed": 7, - "value": 0.3547275267815556, - "path": "" - }, - { - "seed": 42, - "value": 0.5020582486954701, - "path": "" - }, - { - "seed": 99, - "value": 0.4969907507243763, - "path": "" - }, - { - "seed": 123, - "value": 0.6310773342718137, - "path": "" - } - ], - "difference": [ - { - "seed": 1, - "value": 0.48639654658611775, - "path": "" - }, - { - "seed": 7, - "value": 0.29747106554895125, - "path": "" - }, - { - "seed": 42, - "value": 0.42494003459095464, - "path": "" - }, - { - "seed": 99, - "value": 0.42043572680093133, - "path": "" - }, - { - "seed": 123, - "value": 0.5674882177936724, - "path": "" - } - ], - "predicted_positive_rate": [ - { - "seed": 1, - "value": 0.0012293459978232908, - "path": "" - }, - { - "seed": 7, - "value": 0.001793311566241219, - "path": "" - }, - { - "seed": 42, - "value": 0.00112051053725141, - "path": "" - }, - { - "seed": 99, - "value": 0.0007507173246264965, - "path": "" - }, - { - "seed": 123, - "value": 0.002728307113881468, - "path": "" - } - ] - }, - "thresholds": [ - 0.005, - 0.01 - ], - "paths": [ - "", - "", - "", - "", - "" - ] - }, - { - "model_tag": "alphaearth", - "label": "AlphaEarth", - "scope": "train_fire_top20pct", - "scope_label": "top 20%", - "n": 5, - "strict_f1": { - "n": 5, - "mean": 0.06190818589945194, - "std": 0.011329878082076436, - "min": 0.047398843930635835, - "max": 0.07357212003872217 - }, - "tolerance_f1": { - "n": 5, - "mean": 0.38832498285553635, - "std": 0.07496564230623358, - "min": 0.25536195152859176, - "max": 0.42986611450839224 - }, - "union_f1": { - "n": 5, - "mean": 0.4638327916863426, - "std": 0.12169697847221474, - "min": 0.2676781973617166, - "max": 0.5992872540844005 - }, - "difference": { - "n": 5, - "mean": 0.4019246057868907, - "std": 0.11678824405387903, - "min": 0.22027935343108077, - "max": 0.5375844952591261 - }, - "predicted_positive_rate": { - "n": 5, - "mean": 0.0006557823129251701, - "std": 0.00041445475311508546, - "min": 0.00035250463821892396, - "max": 0.0013673469387755102 - }, - "seed_values": { - "strict_f1": [ - { - "seed": 1, - "value": 0.0544280442804428, - "path": "" - }, - { - "seed": 7, - "value": 0.047398843930635835, - "path": "" - }, - { - "seed": 42, - "value": 0.07357212003872217, - "path": "" - }, - { - "seed": 99, - "value": 0.0724391624221845, - "path": "" - }, - { - "seed": 123, - "value": 0.06170275882527441, - "path": "" - } - ], - "tolerance_f1": [ - { - "seed": 1, - "value": 0.42554776574916503, - "path": "" - }, - { - "seed": 7, - "value": 0.25536195152859176, - "path": "" - }, - { - "seed": 42, - "value": 0.42986611450839224, - "path": "" - }, - { - "seed": 99, - "value": 0.40493051697817745, - "path": "" - }, - { - "seed": 123, - "value": 0.4259185655133553, - "path": "" - } - ], - "union_f1": [ - { - "seed": 1, - "value": 0.5120841886230071, - "path": "" - }, - { - "seed": 7, - "value": 0.2676781973617166, - "path": "" - }, - { - "seed": 42, - "value": 0.4723374235406742, - "path": "" - }, - { - "seed": 99, - "value": 0.4677768948219149, - "path": "" - }, - { - "seed": 123, - "value": 0.5992872540844005, - "path": "" - } - ], - "difference": [ - { - "seed": 1, - "value": 0.4576561443425643, - "path": "" - }, - { - "seed": 7, - "value": 0.22027935343108077, - "path": "" - }, - { - "seed": 42, - "value": 0.39876530350195205, - "path": "" - }, - { - "seed": 99, - "value": 0.3953377323997304, - "path": "" - }, - { - "seed": 123, - "value": 0.5375844952591261, - "path": "" - } - ], - "predicted_positive_rate": [ - { - "seed": 1, - "value": 0.0006233766233766234, - "path": "" - }, - { - "seed": 7, - "value": 0.00035250463821892396, - "path": "" - }, - { - "seed": 42, - "value": 0.0005602968460111318, - "path": "" - }, - { - "seed": 99, - "value": 0.0003753865182436611, - "path": "" - }, - { - "seed": 123, - "value": 0.0013673469387755102, - "path": "" - } - ] - }, - "thresholds": [ - 0.005, - 0.01, - 0.02 - ], - "paths": [ - "", - "", - "", - "", - "" - ] - }, - { - "model_tag": "aurora", - "label": "Aurora", - "scope": "full_domain", - "scope_label": "global", - "n": 5, - "strict_f1": { - "n": 5, - "mean": 0.000655543322081318, - "std": 9.445781201845953e-05, - "min": 0.0004938156417811271, - "max": 0.0007150400212604667 - }, - "tolerance_f1": { - "n": 5, - "mean": 0.08500901067877495, - "std": 0.01959379095008962, - "min": 0.05024333401294363, - "max": 0.09742777619989017 - }, - "union_f1": { - "n": 5, - "mean": 0.23103726682311568, - "std": 0.04941752137736377, - "min": 0.14329601271947937, - "max": 0.2618373273867167 - }, - "difference": { - "n": 5, - "mean": 0.23038172350103436, - "std": 0.04932517127284255, - "min": 0.14280219707769823, - "max": 0.2611249045411472 - }, - "predicted_positive_rate": { - "n": 5, - "mean": 0.5558489053803339, - "std": 0.1975239087458258, - "min": 0.21716512059369203, - "max": 0.7199758812615955 - }, - "seed_values": { - "strict_f1": [ - { - "seed": 1, - "value": 0.0004938156417811271, - "path": "" - }, - { - "seed": 7, - "value": 0.0007124228455695601, - "path": "" - }, - { - "seed": 42, - "value": 0.0007076278562951189, - "path": "" - }, - { - "seed": 99, - "value": 0.0007150400212604667, - "path": "" - }, - { - "seed": 123, - "value": 0.0006488102455003172, - "path": "" - } - ], - "tolerance_f1": [ - { - "seed": 1, - "value": 0.05024333401294363, - "path": "" - }, - { - "seed": 7, - "value": 0.09742777619989017, - "path": "" - }, - { - "seed": 42, - "value": 0.09343380358557288, - "path": "" - }, - { - "seed": 99, - "value": 0.09353511619384626, - "path": "" - }, - { - "seed": 123, - "value": 0.09040502340162183, - "path": "" - } - ], - "union_f1": [ - { - "seed": 1, - "value": 0.14329601271947937, - "path": "" - }, - { - "seed": 7, - "value": 0.2618373273867167, - "path": "" - }, - { - "seed": 42, - "value": 0.2527418148996877, - "path": "" - }, - { - "seed": 99, - "value": 0.25248287808338293, - "path": "" - }, - { - "seed": 123, - "value": 0.24482830102631173, - "path": "" - } - ], - "difference": [ - { - "seed": 1, - "value": 0.14280219707769823, - "path": "" - }, - { - "seed": 7, - "value": 0.2611249045411472, - "path": "" - }, - { - "seed": 42, - "value": 0.2520341870433926, - "path": "" - }, - { - "seed": 99, - "value": 0.25176783806212244, - "path": "" - }, - { - "seed": 123, - "value": 0.24417949078081141, - "path": "" - } - ], - "predicted_positive_rate": [ - { - "seed": 1, - "value": 0.21716512059369203, - "path": "" - }, - { - "seed": 7, - "value": 0.6671256648113791, - "path": "" - }, - { - "seed": 42, - "value": 0.5915943104514533, - "path": "" - }, - { - "seed": 99, - "value": 0.5833835497835498, - "path": "" - }, - { - "seed": 123, - "value": 0.7199758812615955, - "path": "" - } - ] - }, - "thresholds": [ - 0.0002701166733984173, - 0.00028167077107355, - 0.00028207036666572094, - 0.0002832056488841772, - 0.0002848515287041664 - ], - "paths": [ - "", - "", - "", - "", - "" - ] - }, - { - "model_tag": "aurora", - "label": "Aurora", - "scope": "train_fire_top05pct", - "scope_label": "top 5%", - "n": 5, - "strict_f1": { - "n": 5, - "mean": 0.009858908695393357, - "std": 0.009298695250165839, - "min": 0.0053561461508737284, - "max": 0.026484309350641822 - }, - "tolerance_f1": { - "n": 5, - "mean": 0.1513369496836336, - "std": 0.06082068957436595, - "min": 0.08994268846280176, - "max": 0.24351172066402033 - }, - "union_f1": { - "n": 5, - "mean": 0.35483366745730893, - "std": 0.11019233024683481, - "min": 0.22524158122442267, - "max": 0.4964540293986394 - }, - "difference": { - "n": 5, - "mean": 0.3449747587619155, - "std": 0.10372797772336852, - "min": 0.21988543507354893, - "max": 0.4699697200479976 - }, - "predicted_positive_rate": { - "n": 5, - "mean": 0.572238052834669, - "std": 0.35339959438797136, - "min": 0.08275452656574651, - "max": 0.921086870485802 - }, - "seed_values": { - "strict_f1": [ - { - "seed": 1, - "value": 0.026484309350641822, - "path": "" - }, - { - "seed": 7, - "value": 0.00565521517042468, - "path": "" - }, - { - "seed": 42, - "value": 0.00618071474679909, - "path": "" - }, - { - "seed": 99, - "value": 0.0053561461508737284, - "path": "" - }, - { - "seed": 123, - "value": 0.005618158058227469, - "path": "" - } - ], - "tolerance_f1": [ - { - "seed": 1, - "value": 0.24351172066402033, - "path": "" - }, - { - "seed": 7, - "value": 0.16052688500868942, - "path": "" - }, - { - "seed": 42, - "value": 0.10264712599745741, - "path": "" - }, - { - "seed": 99, - "value": 0.08994268846280176, - "path": "" - }, - { - "seed": 123, - "value": 0.16005632828519903, - "path": "" - } - ], - "union_f1": [ - { - "seed": 1, - "value": 0.4964540293986394, - "path": "" - }, - { - "seed": 7, - "value": 0.39515679465182413, - "path": "" - }, - { - "seed": 42, - "value": 0.26261754818982164, - "path": "" - }, - { - "seed": 99, - "value": 0.22524158122442267, - "path": "" - }, - { - "seed": 123, - "value": 0.39469838382183675, - "path": "" - } - ], - "difference": [ - { - "seed": 1, - "value": 0.4699697200479976, - "path": "" - }, - { - "seed": 7, - "value": 0.38950157948139946, - "path": "" - }, - { - "seed": 42, - "value": 0.25643683344302254, - "path": "" - }, - { - "seed": 99, - "value": 0.21988543507354893, - "path": "" - }, - { - "seed": 123, - "value": 0.3890802257636093, - "path": "" - } - ], - "predicted_positive_rate": [ - { - "seed": 1, - "value": 0.08275452656574651, - "path": "" - }, - { - "seed": 7, - "value": 0.9150341347580885, - "path": "" - }, - { - "seed": 42, - "value": 0.48403581676066093, - "path": "" - }, - { - "seed": 99, - "value": 0.4582789156030474, - "path": "" - }, - { - "seed": 123, - "value": 0.921086870485802, - "path": "" - } - ] - }, - "thresholds": [ - 0.00026978866662830114, - 0.0002819437358994037, - 0.00028234132332727313, - 0.00028413085965439677, - 0.0002847809228114784 - ], - "paths": [ - "", - "", - "", - "", - "" - ] - }, - { - "model_tag": "aurora", - "label": "Aurora", - "scope": "train_fire_top10pct", - "scope_label": "top 10%", - "n": 5, - "strict_f1": { - "n": 5, - "mean": 0.007789958613164452, - "std": 0.01045266903938076, - "min": 0.00275028568108307, - "max": 0.026476637409113293 - }, - "tolerance_f1": { - "n": 5, - "mean": 0.12738113192754766, - "std": 0.06555769355379562, - "min": 0.08722255267001126, - "max": 0.24290193233931348 - }, - "union_f1": { - "n": 5, - "mean": 0.30530493054248076, - "std": 0.10884219611087281, - "min": 0.23118547623512836, - "max": 0.49392607486162804 - }, - "difference": { - "n": 5, - "mean": 0.2975149719293163, - "std": 0.09865622266744696, - "min": 0.2284351905540453, - "max": 0.46744943745251477 - }, - "predicted_positive_rate": { - "n": 5, - "mean": 0.687941773028594, - "std": 0.37695842871011653, - "min": 0.04137726328287326, - "max": 0.9565152864351439 - }, - "seed_values": { - "strict_f1": [ - { - "seed": 1, - "value": 0.026476637409113293, - "path": "" - }, - { - "seed": 7, - "value": 0.00275028568108307, - "path": "" - }, - { - "seed": 42, - "value": 0.0035136704264394467, - "path": "" - }, - { - "seed": 99, - "value": 0.0034589138681033788, - "path": "" - }, - { - "seed": 123, - "value": 0.00275028568108307, - "path": "" - } - ], - "tolerance_f1": [ - { - "seed": 1, - "value": 0.24290193233931348, - "path": "" - }, - { - "seed": 7, - "value": 0.08722255267001126, - "path": "" - }, - { - "seed": 42, - "value": 0.11052705132032803, - "path": "" - }, - { - "seed": 99, - "value": 0.10903157063807417, - "path": "" - }, - { - "seed": 123, - "value": 0.08722255267001126, - "path": "" - } - ], - "union_f1": [ - { - "seed": 1, - "value": 0.49392607486162804, - "path": "" - }, - { - "seed": 7, - "value": 0.23118547623512836, - "path": "" - }, - { - "seed": 42, - "value": 0.2868180777521314, - "path": "" - }, - { - "seed": 99, - "value": 0.28340954762838766, - "path": "" - }, - { - "seed": 123, - "value": 0.23118547623512836, - "path": "" - } - ], - "difference": [ - { - "seed": 1, - "value": 0.46744943745251477, - "path": "" - }, - { - "seed": 7, - "value": 0.2284351905540453, - "path": "" - }, - { - "seed": 42, - "value": 0.28330440732569195, - "path": "" - }, - { - "seed": 99, - "value": 0.2799506337602843, - "path": "" - }, - { - "seed": 123, - "value": 0.2284351905540453, - "path": "" - } - ], - "predicted_positive_rate": [ - { - "seed": 1, - "value": 0.04137726328287326, - "path": "" - }, - { - "seed": 7, - "value": 0.9565152864351439, - "path": "" - }, - { - "seed": 42, - "value": 0.735743791431681, - "path": "" - }, - { - "seed": 99, - "value": 0.749557237558128, - "path": "" - }, - { - "seed": 123, - "value": 0.9565152864351439, - "path": "" - } - ] - }, - "thresholds": [ - 0.00026914896443486214, - 0.00028129236306995153, - 0.00028190662851557136, - 0.000282647815765813, - 0.0002847809228114784 - ], - "paths": [ - "", - "", - "", - "", - "" - ] - }, - { - "model_tag": "aurora", - "label": "Aurora", - "scope": "train_fire_top20pct", - "scope_label": "top 20%", - "n": 5, - "strict_f1": { - "n": 5, - "mean": 0.0066546155176738, - "std": 0.011042800084784003, - "min": 0.001471035812113219, - "max": 0.026403975040443728 - }, - "tolerance_f1": { - "n": 5, - "mean": 0.1053036158902898, - "std": 0.07430895639255175, - "min": 0.06246138809231819, - "max": 0.23725772191057362 - }, - "union_f1": { - "n": 5, - "mean": 0.24944433080431602, - "std": 0.12584449940748835, - "min": 0.17013954201238674, - "max": 0.4711352810381827 - }, - "difference": { - "n": 5, - "mean": 0.24278971528664223, - "std": 0.11494327446365726, - "min": 0.16866850620027352, - "max": 0.44473130599773897 - }, - "predicted_positive_rate": { - "n": 5, - "mean": 0.6728910327767471, - "std": 0.38399138452827886, - "min": 0.02069016697588126, - "max": 0.9746196660482375 - }, - "seed_values": { - "strict_f1": [ - { - "seed": 1, - "value": 0.026403975040443728, - "path": "" - }, - { - "seed": 7, - "value": 0.0014870516331508276, - "path": "" - }, - { - "seed": 42, - "value": 0.00196209796760366, - "path": "" - }, - { - "seed": 99, - "value": 0.0019489171350575599, - "path": "" - }, - { - "seed": 123, - "value": 0.001471035812113219, - "path": "" - } - ], - "tolerance_f1": [ - { - "seed": 1, - "value": 0.23725772191057362, - "path": "" - }, - { - "seed": 7, - "value": 0.06425881062262738, - "path": "" - }, - { - "seed": 42, - "value": 0.08141778117479781, - "path": "" - }, - { - "seed": 99, - "value": 0.08112237765113199, - "path": "" - }, - { - "seed": 123, - "value": 0.06246138809231819, - "path": "" - } - ], - "union_f1": [ - { - "seed": 1, - "value": 0.4711352810381827, - "path": "" - }, - { - "seed": 7, - "value": 0.1742547495584396, - "path": "" - }, - { - "seed": 42, - "value": 0.21602191083593253, - "path": "" - }, - { - "seed": 99, - "value": 0.21567017057663868, - "path": "" - }, - { - "seed": 123, - "value": 0.17013954201238674, - "path": "" - } - ], - "difference": [ - { - "seed": 1, - "value": 0.44473130599773897, - "path": "" - }, - { - "seed": 7, - "value": 0.17276769792528876, - "path": "" - }, - { - "seed": 42, - "value": 0.21405981286832887, - "path": "" - }, - { - "seed": 99, - "value": 0.2137212534415811, - "path": "" - }, - { - "seed": 123, - "value": 0.16866850620027352, - "path": "" - } - ], - "predicted_positive_rate": [ - { - "seed": 1, - "value": 0.02069016697588126, - "path": "" - }, - { - "seed": 7, - "value": 0.9366672850958565, - "path": "" - }, - { - "seed": 42, - "value": 0.7109764996907854, - "path": "" - }, - { - "seed": 99, - "value": 0.7215015460729747, - "path": "" - }, - { - "seed": 123, - "value": 0.9746196660482375, - "path": "" - } - ] - }, - "thresholds": [ - 0.0002, - 0.0002701335761230439, - 0.00028130068676546216, - 0.0002826936251949519, - 0.0002847809228114784 - ], - "paths": [ - "", - "", - "", - "", - "" - ] - }, - { - "model_tag": "climax", - "label": "ClimaX", - "scope": "full_domain", - "scope_label": "global", - "n": 5, - "strict_f1": { - "n": 5, - "mean": 0.00347964131784203, - "std": 0.0007542402263439388, - "min": 0.0027501821392557145, - "max": 0.004637761701086003 - }, - "tolerance_f1": { - "n": 5, - "mean": 0.2975348750078338, - "std": 0.03607251580095264, - "min": 0.24162386260287075, - "max": 0.33641526956922785 - }, - "union_f1": { - "n": 5, - "mean": 0.6015057903864602, - "std": 0.07586493774699653, - "min": 0.4719733074426152, - "max": 0.6677488562714049 - }, - "difference": { - "n": 5, - "mean": 0.5980261490686182, - "std": 0.07545417523567537, - "min": 0.46922312530335947, - "max": 0.664031416861268 - }, - "predicted_positive_rate": { - "n": 5, - "mean": 0.056763042671614104, - "std": 0.03926077114304812, - "min": 0.025392455163883734, - "max": 0.12421632653061225 - }, - "seed_values": { - "strict_f1": [ - { - "seed": 1, - "value": 0.003394935139973715, - "path": "" - }, - { - "seed": 7, - "value": 0.0027501821392557145, - "path": "" - }, - { - "seed": 42, - "value": 0.004637761701086003, - "path": "" - }, - { - "seed": 99, - "value": 0.003717439410136951, - "path": "" - }, - { - "seed": 123, - "value": 0.002897888198757764, - "path": "" - } - ], - "tolerance_f1": [ - { - "seed": 1, - "value": 0.3112148878371163, - "path": "" - }, - { - "seed": 7, - "value": 0.24162386260287075, - "path": "" - }, - { - "seed": 42, - "value": 0.31289270592627744, - "path": "" - }, - { - "seed": 99, - "value": 0.33641526956922785, - "path": "" - }, - { - "seed": 123, - "value": 0.2855276491036767, - "path": "" - } - ], - "union_f1": [ - { - "seed": 1, - "value": 0.6396719244631184, - "path": "" - }, - { - "seed": 7, - "value": 0.4719733074426152, - "path": "" - }, - { - "seed": 42, - "value": 0.620505744456497, - "path": "" - }, - { - "seed": 99, - "value": 0.6677488562714049, - "path": "" - }, - { - "seed": 123, - "value": 0.6076291192986658, - "path": "" - } - ], - "difference": [ - { - "seed": 1, - "value": 0.6362769893231447, - "path": "" - }, - { - "seed": 7, - "value": 0.46922312530335947, - "path": "" - }, - { - "seed": 42, - "value": 0.615867982755411, - "path": "" - }, - { - "seed": 99, - "value": 0.664031416861268, - "path": "" - }, - { - "seed": 123, - "value": 0.604731231099908, - "path": "" - } - ], - "predicted_positive_rate": [ - { - "seed": 1, - "value": 0.041071861471861475, - "path": "" - }, - { - "seed": 7, - "value": 0.025392455163883734, - "path": "" - }, - { - "seed": 42, - "value": 0.037147804576376, - "path": "" - }, - { - "seed": 99, - "value": 0.055986765615337046, - "path": "" - }, - { - "seed": 123, - "value": 0.12421632653061225, - "path": "" - } - ] - }, - "thresholds": [ - 0.0005393968409013734, - 0.001021229882830197, - 0.0013572092986540852, - 0.001442534703828549, - 0.0014632086704014654 - ], - "paths": [ - "", - "", - "", - "", - "" - ] - }, - { - "model_tag": "climax", - "label": "ClimaX", - "scope": "train_fire_top05pct", - "scope_label": "top 5%", - "n": 5, - "strict_f1": { - "n": 5, - "mean": 0.012936887506760619, - "std": 0.0010861342333245315, - "min": 0.011492789729228419, - "max": 0.014246556800108287 - }, - "tolerance_f1": { - "n": 5, - "mean": 0.34579050632090236, - "std": 0.02377186314441771, - "min": 0.3186326966868806, - "max": 0.3665570668664193 - }, - "union_f1": { - "n": 5, - "mean": 0.6921863051786654, - "std": 0.057214573939719605, - "min": 0.6170788209868917, - "max": 0.7408743604065289 - }, - "difference": { - "n": 5, - "mean": 0.6792494176719048, - "std": 0.05726305372578923, - "min": 0.6028322641867834, - "max": 0.7272887839717904 - }, - "predicted_positive_rate": { - "n": 5, - "mean": 0.24009102602156923, - "std": 0.0928155581273325, - "min": 0.13341248639556744, - "max": 0.3090951815573365 - }, - "seed_values": { - "strict_f1": [ - { - "seed": 1, - "value": 0.011492789729228419, - "path": "" - }, - { - "seed": 7, - "value": 0.013106997858124742, - "path": "" - }, - { - "seed": 42, - "value": 0.014246556800108287, - "path": "" - }, - { - "seed": 99, - "value": 0.013585576434738445, - "path": "" - }, - { - "seed": 123, - "value": 0.012252516711603196, - "path": "" - } - ], - "tolerance_f1": [ - { - "seed": 1, - "value": 0.3186326966868806, - "path": "" - }, - { - "seed": 7, - "value": 0.3655334236435691, - "path": "" - }, - { - "seed": 42, - "value": 0.32161191611096424, - "path": "" - }, - { - "seed": 99, - "value": 0.3665570668664193, - "path": "" - }, - { - "seed": 123, - "value": 0.3566174282966785, - "path": "" - } - ], - "union_f1": [ - { - "seed": 1, - "value": 0.6456200196251293, - "path": "" - }, - { - "seed": 7, - "value": 0.7398705155472896, - "path": "" - }, - { - "seed": 42, - "value": 0.6170788209868917, - "path": "" - }, - { - "seed": 99, - "value": 0.7408743604065289, - "path": "" - }, - { - "seed": 123, - "value": 0.7174878093274878, - "path": "" - } - ], - "difference": [ - { - "seed": 1, - "value": 0.6341272298959009, - "path": "" - }, - { - "seed": 7, - "value": 0.7267635176891648, - "path": "" - }, - { - "seed": 42, - "value": 0.6028322641867834, - "path": "" - }, - { - "seed": 99, - "value": 0.7272887839717904, - "path": "" - }, - { - "seed": 123, - "value": 0.7052352926158846, - "path": "" - } - ], - "predicted_positive_rate": [ - { - "seed": 1, - "value": 0.13341248639556744, - "path": "" - }, - { - "seed": 7, - "value": 0.3068887899475611, - "path": "" - }, - { - "seed": 42, - "value": 0.14358118135945386, - "path": "" - }, - { - "seed": 99, - "value": 0.3090951815573365, - "path": "" - }, - { - "seed": 123, - "value": 0.3074774908479272, - "path": "" - } - ] - }, - "thresholds": [ - 0.0005436268660297483, - 0.0006166165592580956, - 0.0007465386633598005, - 0.0014632082671483923, - 0.0014882655838293205 - ], - "paths": [ - "", - "", - "", - "", - "" - ] - }, - { - "model_tag": "climax", - "label": "ClimaX", - "scope": "train_fire_top10pct", - "scope_label": "top 10%", - "n": 5, - "strict_f1": { - "n": 5, - "mean": 0.012522340011631166, - "std": 0.0016018805893483327, - "min": 0.010293697838827224, - "max": 0.014244146704560834 - }, - "tolerance_f1": { - "n": 5, - "mean": 0.34334086004373804, - "std": 0.022852052082273543, - "min": 0.3179193199949169, - "max": 0.36524448333846904 - }, - "union_f1": { - "n": 5, - "mean": 0.6857131503190195, - "std": 0.05537732130173235, - "min": 0.6141812807681712, - "max": 0.7365490884872157 - }, - "difference": { - "n": 5, - "mean": 0.6731908103073884, - "std": 0.05553767153675633, - "min": 0.5999371340636104, - "max": 0.7230514919507499 - }, - "predicted_positive_rate": { - "n": 5, - "mean": 0.1184503314534481, - "std": 0.04506675919516373, - "min": 0.06670624319778372, - "max": 0.15408504007123774 - }, - "seed_values": { - "strict_f1": [ - { - "seed": 1, - "value": 0.011490700169087834, - "path": "" - }, - { - "seed": 7, - "value": 0.01308555880921415, - "path": "" - }, - { - "seed": 42, - "value": 0.014244146704560834, - "path": "" - }, - { - "seed": 99, - "value": 0.013497596536465794, - "path": "" - }, - { - "seed": 123, - "value": 0.010293697838827224, - "path": "" - } - ], - "tolerance_f1": [ - { - "seed": 1, - "value": 0.3179193199949169, - "path": "" - }, - { - "seed": 7, - "value": 0.36399388919405107, - "path": "" - }, - { - "seed": 42, - "value": 0.3208182317552862, - "path": "" - }, - { - "seed": 99, - "value": 0.36524448333846904, - "path": "" - }, - { - "seed": 123, - "value": 0.34872837593596695, - "path": "" - } - ], - "union_f1": [ - { - "seed": 1, - "value": 0.6427222935095626, - "path": "" - }, - { - "seed": 7, - "value": 0.7361209272894985, - "path": "" - }, - { - "seed": 42, - "value": 0.6141812807681712, - "path": "" - }, - { - "seed": 99, - "value": 0.7365490884872157, - "path": "" - }, - { - "seed": 123, - "value": 0.6989921615406495, - "path": "" - } - ], - "difference": [ - { - "seed": 1, - "value": 0.6312315933404747, - "path": "" - }, - { - "seed": 7, - "value": 0.7230353684802844, - "path": "" - }, - { - "seed": 42, - "value": 0.5999371340636104, - "path": "" - }, - { - "seed": 99, - "value": 0.7230514919507499, - "path": "" - }, - { - "seed": 123, - "value": 0.6886984637018222, - "path": "" - } - ], - "predicted_positive_rate": [ - { - "seed": 1, - "value": 0.06670624319778372, - "path": "" - }, - { - "seed": 7, - "value": 0.15368556446027506, - "path": "" - }, - { - "seed": 42, - "value": 0.07179059067972693, - "path": "" - }, - { - "seed": 99, - "value": 0.15408504007123774, - "path": "" - }, - { - "seed": 123, - "value": 0.14598421885821708, - "path": "" - } - ] - }, - "thresholds": [ - 0.0006294738670532621, - 0.0006776005131691771, - 0.0007519771209341887, - 0.0014632082671483923, - 0.0014882655838293205 - ], - "paths": [ - "", - "", - "", - "", - "" - ] - }, - { - "model_tag": "climax", - "label": "ClimaX", - "scope": "train_fire_top20pct", - "scope_label": "top 20%", - "n": 5, - "strict_f1": { - "n": 5, - "mean": 0.010287276360038863, - "std": 0.002686481400804917, - "min": 0.006734646906365392, - "max": 0.013757803485310216 - }, - "tolerance_f1": { - "n": 5, - "mean": 0.3021399507767809, - "std": 0.042856591284064, - "min": 0.24066482674710968, - "max": 0.35193063179186285 - }, - "union_f1": { - "n": 5, - "mean": 0.6006499255079836, - "std": 0.07567424048289119, - "min": 0.48364281191756603, - "max": 0.691273953290052 - }, - "difference": { - "n": 5, - "mean": 0.5903626491479447, - "std": 0.07589123629900595, - "min": 0.47288261881572224, - "max": 0.6825790409197952 - }, - "predicted_positive_rate": { - "n": 5, - "mean": 0.057441929499072356, - "std": 0.04134148942976998, - "min": 0.020547928262213976, - "max": 0.12288311688311689 - }, - "seed_values": { - "strict_f1": [ - { - "seed": 1, - "value": 0.011488825936418005, - "path": "" - }, - { - "seed": 7, - "value": 0.010760193101843772, - "path": "" - }, - { - "seed": 42, - "value": 0.013757803485310216, - "path": "" - }, - { - "seed": 99, - "value": 0.008694912370256934, - "path": "" - }, - { - "seed": 123, - "value": 0.006734646906365392, - "path": "" - } - ], - "tolerance_f1": [ - { - "seed": 1, - "value": 0.30575189275627285, - "path": "" - }, - { - "seed": 7, - "value": 0.24066482674710968, - "path": "" - }, - { - "seed": 42, - "value": 0.32901058451573606, - "path": "" - }, - { - "seed": 99, - "value": 0.35193063179186285, - "path": "" - }, - { - "seed": 123, - "value": 0.2833418180729231, - "path": "" - } - ], - "union_f1": [ - { - "seed": 1, - "value": 0.6099212745225361, - "path": "" - }, - { - "seed": 7, - "value": 0.48364281191756603, - "path": "" - }, - { - "seed": 42, - "value": 0.6288351083076325, - "path": "" - }, - { - "seed": 99, - "value": 0.691273953290052, - "path": "" - }, - { - "seed": 123, - "value": 0.5895764795021311, - "path": "" - } - ], - "difference": [ - { - "seed": 1, - "value": 0.5984324485861181, - "path": "" - }, - { - "seed": 7, - "value": 0.47288261881572224, - "path": "" - }, - { - "seed": 42, - "value": 0.6150773048223223, - "path": "" - }, - { - "seed": 99, - "value": 0.6825790409197952, - "path": "" - }, - { - "seed": 123, - "value": 0.5828418325957657, - "path": "" - } - ], - "predicted_positive_rate": [ - { - "seed": 1, - "value": 0.030718614718614718, - "path": "" - }, - { - "seed": 7, - "value": 0.020547928262213976, - "path": "" - }, - { - "seed": 42, - "value": 0.041087198515769946, - "path": "" - }, - { - "seed": 99, - "value": 0.07197278911564625, - "path": "" - }, - { - "seed": 123, - "value": 0.12288311688311689, - "path": "" - } - ] - }, - "thresholds": [ - 0.0006820998141378855, - 0.0009633156281329466, - 0.0013572138352511622, - 0.0013755693906898965, - 0.0015083511573617753 - ], - "paths": [ - "", - "", - "", - "", - "" - ] - }, - { - "model_tag": "dlwp", - "label": "DLWP", - "scope": "full_domain", - "scope_label": "global", - "n": 5, - "strict_f1": { - "n": 5, - "mean": 0.001692736472985844, - "std": 0.0004188778823216006, - "min": 0.001008153911497155, - "max": 0.002078902199880332 - }, - "tolerance_f1": { - "n": 5, - "mean": 0.14914807478919873, - "std": 0.032446265301716536, - "min": 0.09460750931776095, - "max": 0.17299721297318826 - }, - "union_f1": { - "n": 5, - "mean": 0.281900673424129, - "std": 0.06965835259895045, - "min": 0.17657638185878205, - "max": 0.36723610814010926 - }, - "difference": { - "n": 5, - "mean": 0.2802079369511431, - "std": 0.06925745403651407, - "min": 0.17556822794728488, - "max": 0.36515720594022893 - }, - "predicted_positive_rate": { - "n": 5, - "mean": 0.04322077922077922, - "std": 0.015619298009606992, - "min": 0.02625751391465677, - "max": 0.0589282622139765 - }, - "seed_values": { - "strict_f1": [ - { - "seed": 7, - "value": 0.0016304484151205278, - "path": "" - }, - { - "seed": 17, - "value": 0.001786130143937306, - "path": "" - }, - { - "seed": 37, - "value": 0.0019600476944939, - "path": "" - }, - { - "seed": 57, - "value": 0.001008153911497155, - "path": "" - }, - { - "seed": 97, - "value": 0.002078902199880332, - "path": "" - } - ], - "tolerance_f1": [ - { - "seed": 7, - "value": 0.14442504878529014, - "path": "" - }, - { - "seed": 17, - "value": 0.16383974771641738, - "path": "" - }, - { - "seed": 37, - "value": 0.17299721297318826, - "path": "" - }, - { - "seed": 57, - "value": 0.09460750931776095, - "path": "" - }, - { - "seed": 97, - "value": 0.16987085515333694, - "path": "" - } - ], - "union_f1": [ - { - "seed": 7, - "value": 0.265022372628049, - "path": "" - }, - { - "seed": 17, - "value": 0.3062219870023089, - "path": "" - }, - { - "seed": 37, - "value": 0.2944465174913956, - "path": "" - }, - { - "seed": 57, - "value": 0.17657638185878205, - "path": "" - }, - { - "seed": 97, - "value": 0.36723610814010926, - "path": "" - } - ], - "difference": [ - { - "seed": 7, - "value": 0.26339192421292845, - "path": "" - }, - { - "seed": 17, - "value": 0.3044358568583716, - "path": "" - }, - { - "seed": 37, - "value": 0.2924864697969017, - "path": "" - }, - { - "seed": 57, - "value": 0.17556822794728488, - "path": "" - }, - { - "seed": 97, - "value": 0.36515720594022893, - "path": "" - } - ], - "predicted_positive_rate": [ - { - "seed": 7, - "value": 0.0589282622139765, - "path": "" - }, - { - "seed": 17, - "value": 0.057648732220160794, - "path": "" - }, - { - "seed": 37, - "value": 0.04519196042053185, - "path": "" - }, - { - "seed": 57, - "value": 0.02625751391465677, - "path": "" - }, - { - "seed": 97, - "value": 0.028077427334570193, - "path": "" - } - ] - }, - "thresholds": [ - 0.08, - 0.1 - ], - "paths": [ - "", - "", - "", - "", - "" - ] - }, - { - "model_tag": "dlwp", - "label": "DLWP", - "scope": "train_fire_top05pct", - "scope_label": "top 5%", - "n": 5, - "strict_f1": { - "n": 5, - "mean": 0.01805388290899331, - "std": 0.004835410031272547, - "min": 0.011684287006420426, - "max": 0.024960845732184808 - }, - "tolerance_f1": { - "n": 5, - "mean": 0.31723135564296057, - "std": 0.032922909485364145, - "min": 0.28520211660517514, - "max": 0.35323298220195054 - }, - "union_f1": { - "n": 5, - "mean": 0.5545957535300212, - "std": 0.05291976314318387, - "min": 0.47313209204823786, - "max": 0.6023409019864735 - }, - "difference": { - "n": 5, - "mean": 0.5365418706210278, - "std": 0.0547523973572839, - "min": 0.45626799482364494, - "max": 0.5906566149800531 - }, - "predicted_positive_rate": { - "n": 5, - "mean": 0.09400069258929455, - "std": 0.05080713313175061, - "min": 0.04792965271593945, - "max": 0.16420797467102008 - }, - "seed_values": { - "strict_f1": [ - { - "seed": 7, - "value": 0.016864097224592905, - "path": "" - }, - { - "seed": 17, - "value": 0.01972516815912186, - "path": "" - }, - { - "seed": 37, - "value": 0.024960845732184808, - "path": "" - }, - { - "seed": 57, - "value": 0.017035016422646553, - "path": "" - }, - { - "seed": 97, - "value": 0.011684287006420426, - "path": "" - } - ], - "tolerance_f1": [ - { - "seed": 7, - "value": 0.28520211660517514, - "path": "" - }, - { - "seed": 17, - "value": 0.3500819147736832, - "path": "" - }, - { - "seed": 37, - "value": 0.35323298220195054, - "path": "" - }, - { - "seed": 57, - "value": 0.3101459291213085, - "path": "" - }, - { - "seed": 97, - "value": 0.2874938355126855, - "path": "" - } - ], - "union_f1": [ - { - "seed": 7, - "value": 0.47313209204823786, - "path": "" - }, - { - "seed": 17, - "value": 0.5865273622259427, - "path": "" - }, - { - "seed": 37, - "value": 0.5303411060257172, - "path": "" - }, - { - "seed": 57, - "value": 0.5806373053637345, - "path": "" - }, - { - "seed": 97, - "value": 0.6023409019864735, - "path": "" - } - ], - "difference": [ - { - "seed": 7, - "value": 0.45626799482364494, - "path": "" - }, - { - "seed": 17, - "value": 0.5668021940668209, - "path": "" - }, - { - "seed": 37, - "value": 0.5053802602935323, - "path": "" - }, - { - "seed": 57, - "value": 0.5636022889410879, - "path": "" - }, - { - "seed": 97, - "value": 0.5906566149800531, - "path": "" - } - ], - "predicted_positive_rate": [ - { - "seed": 7, - "value": 0.06984763035519936, - "path": "" - }, - { - "seed": 17, - "value": 0.05733155238943306, - "path": "" - }, - { - "seed": 37, - "value": 0.04792965271593945, - "path": "" - }, - { - "seed": 57, - "value": 0.1306866528148808, - "path": "" - }, - { - "seed": 97, - "value": 0.16420797467102008, - "path": "" - } - ] - }, - "thresholds": [ - 0.05, - 0.08, - 0.1 - ], - "paths": [ - "", - "", - "", - "", - "" - ] - }, - { - "model_tag": "dlwp", - "label": "DLWP", - "scope": "train_fire_top10pct", - "scope_label": "top 10%", - "n": 5, - "strict_f1": { - "n": 5, - "mean": 0.016110218166136593, - "std": 0.005998581011452884, - "min": 0.008571862589787548, - "max": 0.023191305534082125 - }, - "tolerance_f1": { - "n": 5, - "mean": 0.27658069580542066, - "std": 0.05921597516408298, - "min": 0.2160234001333052, - "max": 0.3409571337826908 - }, - "union_f1": { - "n": 5, - "mean": 0.4712685600792209, - "std": 0.08011144927032286, - "min": 0.3704561794666731, - "max": 0.57270947387435 - }, - "difference": { - "n": 5, - "mean": 0.45515834191308435, - "std": 0.07792651255459898, - "min": 0.3591438068453628, - "max": 0.5537110636994692 - }, - "predicted_positive_rate": { - "n": 5, - "mean": 0.049700455130107846, - "std": 0.03684905017300331, - "min": 0.025880577817354308, - "max": 0.11266572672405263 - }, - "seed_values": { - "strict_f1": [ - { - "seed": 7, - "value": 0.011312372621310312, - "path": "" - }, - { - "seed": 17, - "value": 0.018998410174880763, - "path": "" - }, - { - "seed": 37, - "value": 0.023191305534082125, - "path": "" - }, - { - "seed": 57, - "value": 0.018477139910622206, - "path": "" - }, - { - "seed": 97, - "value": 0.008571862589787548, - "path": "" - } - ], - "tolerance_f1": [ - { - "seed": 7, - "value": 0.2160234001333052, - "path": "" - }, - { - "seed": 17, - "value": 0.3409571337826908, - "path": "" - }, - { - "seed": 37, - "value": 0.33689092573275314, - "path": "" - }, - { - "seed": 57, - "value": 0.2606560686639405, - "path": "" - }, - { - "seed": 97, - "value": 0.22837595071441363, - "path": "" - } - ], - "union_f1": [ - { - "seed": 7, - "value": 0.3704561794666731, - "path": "" - }, - { - "seed": 17, - "value": 0.57270947387435, - "path": "" - }, - { - "seed": 37, - "value": 0.5094071833388841, - "path": "" - }, - { - "seed": 57, - "value": 0.41307788619921965, - "path": "" - }, - { - "seed": 97, - "value": 0.4906920775169778, - "path": "" - } - ], - "difference": [ - { - "seed": 7, - "value": 0.3591438068453628, - "path": "" - }, - { - "seed": 17, - "value": 0.5537110636994692, - "path": "" - }, - { - "seed": 37, - "value": 0.486215877804802, - "path": "" - }, - { - "seed": 57, - "value": 0.3946007462885974, - "path": "" - }, - { - "seed": 97, - "value": 0.48212021492719026, - "path": "" - } - ], - "predicted_positive_rate": [ - { - "seed": 7, - "value": 0.052691204115959234, - "path": "" - }, - { - "seed": 17, - "value": 0.02979989116453943, - "path": "" - }, - { - "seed": 37, - "value": 0.025880577817354308, - "path": "" - }, - { - "seed": 57, - "value": 0.02746487582863362, - "path": "" - }, - { - "seed": 97, - "value": 0.11266572672405263, - "path": "" - } - ] - }, - "thresholds": [ - 0.05, - 0.08, - 0.1 - ], - "paths": [ - "", - "", - "", - "", - "" - ] - }, - { - "model_tag": "dlwp", - "label": "DLWP", - "scope": "train_fire_top20pct", - "scope_label": "top 20%", - "n": 5, - "strict_f1": { - "n": 5, - "mean": 0.015247655927156777, - "std": 0.008986797239635393, - "min": 0.007379212786746584, - "max": 0.028949691085613415 - }, - "tolerance_f1": { - "n": 5, - "mean": 0.2094029351195231, - "std": 0.047970894595050655, - "min": 0.1600767886426284, - "max": 0.28953047944835925 - }, - "union_f1": { - "n": 5, - "mean": 0.3493011789060551, - "std": 0.0784709395452888, - "min": 0.2769459546244767, - "max": 0.4666657840911348 - }, - "difference": { - "n": 5, - "mean": 0.33405352297889834, - "std": 0.0787597726808095, - "min": 0.26320327785336994, - "max": 0.44734760227295295 - }, - "predicted_positive_rate": { - "n": 5, - "mean": 0.019198021026592453, - "std": 0.014678247188046134, - "min": 0.0027860235003092145, - "max": 0.04168893011750155 - }, - "seed_values": { - "strict_f1": [ - { - "seed": 7, - "value": 0.007379212786746584, - "path": "" - }, - { - "seed": 17, - "value": 0.01931818181818182, - "path": "" - }, - { - "seed": 37, - "value": 0.028949691085613415, - "path": "" - }, - { - "seed": 57, - "value": 0.01226752201682564, - "path": "" - }, - { - "seed": 97, - "value": 0.008323671928416423, - "path": "" - } - ], - "tolerance_f1": [ - { - "seed": 7, - "value": 0.1600767886426284, - "path": "" - }, - { - "seed": 17, - "value": 0.28953047944835925, - "path": "" - }, - { - "seed": 37, - "value": 0.20397124965652888, - "path": "" - }, - { - "seed": 57, - "value": 0.19661692001462533, - "path": "" - }, - { - "seed": 97, - "value": 0.19681923783547378, - "path": "" - } - ], - "union_f1": [ - { - "seed": 7, - "value": 0.2769459546244767, - "path": "" - }, - { - "seed": 17, - "value": 0.4666657840911348, - "path": "" - }, - { - "seed": 37, - "value": 0.29215296893898335, - "path": "" - }, - { - "seed": 57, - "value": 0.3215961142158263, - "path": "" - }, - { - "seed": 97, - "value": 0.3891450726598544, - "path": "" - } - ], - "difference": [ - { - "seed": 7, - "value": 0.2695667418377301, - "path": "" - }, - { - "seed": 17, - "value": 0.44734760227295295, - "path": "" - }, - { - "seed": 37, - "value": 0.26320327785336994, - "path": "" - }, - { - "seed": 57, - "value": 0.30932859219900066, - "path": "" - }, - { - "seed": 97, - "value": 0.38082140073143794, - "path": "" - } - ], - "predicted_positive_rate": [ - { - "seed": 7, - "value": 0.04168893011750155, - "path": "" - }, - { - "seed": 17, - "value": 0.010166975881261595, - "path": "" - }, - { - "seed": 37, - "value": 0.0027860235003092145, - "path": "" - }, - { - "seed": 57, - "value": 0.021262213976499692, - "path": "" - }, - { - "seed": 97, - "value": 0.02008596165739023, - "path": "" - } - ] - }, - "thresholds": [ - 0.08, - 0.1, - 0.2 - ], - "paths": [ - "", - "", - "", - "", - "" - ] - }, - { - "model_tag": "fcn", - "label": "FCN", - "scope": "full_domain", - "scope_label": "global", - "n": 5, - "strict_f1": { - "n": 5, - "mean": 0.002829241015607336, - "std": 0.0008385182499278472, - "min": 0.002068681772914111, - "max": 0.00400312243549969 - }, - "tolerance_f1": { - "n": 5, - "mean": 0.19506132405627552, - "std": 0.03341186919631261, - "min": 0.16083569292964747, - "max": 0.23211789308126832 - }, - "union_f1": { - "n": 5, - "mean": 0.4006042571190199, - "std": 0.09370061309902981, - "min": 0.2863487755227932, - "max": 0.5057290416588791 - }, - "difference": { - "n": 5, - "mean": 0.39777501610341254, - "std": 0.09342259085449434, - "min": 0.2830040701913329, - "max": 0.5017259192233794 - }, - "predicted_positive_rate": { - "n": 5, - "mean": 0.01520235003092146, - "std": 0.013446242178130542, - "min": 0.0016064316635745208, - "max": 0.03192430426716141 - }, - "seed_values": { - "strict_f1": [ - { - "seed": 7, - "value": 0.0026467855605157606, - "path": "" - }, - { - "seed": 17, - "value": 0.002068681772914111, - "path": "" - }, - { - "seed": 37, - "value": 0.00208290997764682, - "path": "" - }, - { - "seed": 57, - "value": 0.0033447053314602988, - "path": "" - }, - { - "seed": 97, - "value": 0.00400312243549969, - "path": "" - } - ], - "tolerance_f1": [ - { - "seed": 7, - "value": 0.23211789308126832, - "path": "" - }, - { - "seed": 17, - "value": 0.19094441931639336, - "path": "" - }, - { - "seed": 37, - "value": 0.16485748333864078, - "path": "" - }, - { - "seed": 57, - "value": 0.16083569292964747, - "path": "" - }, - { - "seed": 97, - "value": 0.22655113161542761, - "path": "" - } - ], - "union_f1": [ - { - "seed": 7, - "value": 0.48493981229339306, - "path": "" - }, - { - "seed": 17, - "value": 0.38616767546901964, - "path": "" - }, - { - "seed": 37, - "value": 0.33983598065101456, - "path": "" - }, - { - "seed": 57, - "value": 0.2863487755227932, - "path": "" - }, - { - "seed": 97, - "value": 0.5057290416588791, - "path": "" - } - ], - "difference": [ - { - "seed": 7, - "value": 0.4822930267328773, - "path": "" - }, - { - "seed": 17, - "value": 0.3840989936961055, - "path": "" - }, - { - "seed": 37, - "value": 0.33775307067336774, - "path": "" - }, - { - "seed": 57, - "value": 0.2830040701913329, - "path": "" - }, - { - "seed": 97, - "value": 0.5017259192233794, - "path": "" - } - ], - "predicted_positive_rate": [ - { - "seed": 7, - "value": 0.027048113790970933, - "path": "" - }, - { - "seed": 17, - "value": 0.03192430426716141, - "path": "" - }, - { - "seed": 37, - "value": 0.009495980210265924, - "path": "" - }, - { - "seed": 57, - "value": 0.0016064316635745208, - "path": "" - }, - { - "seed": 97, - "value": 0.005936920222634508, - "path": "" - } - ] - }, - "thresholds": [ - 0.15, - 0.2, - 0.3, - 0.5 - ], - "paths": [ - "", - "", - "", - "", - "" - ] - }, - { - "model_tag": "fcn", - "label": "FCN", - "scope": "train_fire_top05pct", - "scope_label": "top 5%", - "n": 5, - "strict_f1": { - "n": 5, - "mean": 0.016231351161484737, - "std": 0.00506423236804874, - "min": 0.012095158293562608, - "max": 0.024537870112874206 - }, - "tolerance_f1": { - "n": 5, - "mean": 0.2937694425517297, - "std": 0.0276260699067859, - "min": 0.25572856051368525, - "max": 0.3296389749733903 - }, - "union_f1": { - "n": 5, - "mean": 0.5430325295439387, - "std": 0.0740889720448565, - "min": 0.47261322844070225, - "max": 0.6562774730922674 - }, - "difference": { - "n": 5, - "mean": 0.5268011783824539, - "std": 0.0743888677938, - "min": 0.46051807014713964, - "max": 0.6396713821034217 - }, - "predicted_positive_rate": { - "n": 5, - "mean": 0.047855941426733946, - "std": 0.029409395848307603, - "min": 0.01251113089937667, - "max": 0.07633570792520035 - }, - "seed_values": { - "strict_f1": [ - { - "seed": 7, - "value": 0.01222529471692621, - "path": "" - }, - { - "seed": 17, - "value": 0.012095158293562608, - "path": "" - }, - { - "seed": 37, - "value": 0.024537870112874206, - "path": "" - }, - { - "seed": 57, - "value": 0.01569234169521494, - "path": "" - }, - { - "seed": 97, - "value": 0.016606090988845718, - "path": "" - } - ], - "tolerance_f1": [ - { - "seed": 7, - "value": 0.3028035236186743, - "path": "" - }, - { - "seed": 17, - "value": 0.25572856051368525, - "path": "" - }, - { - "seed": 37, - "value": 0.27999632336313063, - "path": "" - }, - { - "seed": 57, - "value": 0.300679830289768, - "path": "" - }, - { - "seed": 97, - "value": 0.3296389749733903, - "path": "" - } - ], - "union_f1": [ - { - "seed": 7, - "value": 0.5768233236256973, - "path": "" - }, - { - "seed": 17, - "value": 0.47261322844070225, - "path": "" - }, - { - "seed": 37, - "value": 0.5116723674766515, - "path": "" - }, - { - "seed": 57, - "value": 0.49777625508437506, - "path": "" - }, - { - "seed": 97, - "value": 0.6562774730922674, - "path": "" - } - ], - "difference": [ - { - "seed": 7, - "value": 0.5645980289087711, - "path": "" - }, - { - "seed": 17, - "value": 0.46051807014713964, - "path": "" - }, - { - "seed": 37, - "value": 0.48713449736377723, - "path": "" - }, - { - "seed": 57, - "value": 0.4820839133891601, - "path": "" - }, - { - "seed": 97, - "value": 0.6396713821034217, - "path": "" - } - ], - "predicted_positive_rate": [ - { - "seed": 7, - "value": 0.06537300880577818, - "path": "" - }, - { - "seed": 17, - "value": 0.06528643514395963, - "path": "" - }, - { - "seed": 37, - "value": 0.01251113089937667, - "path": "" - }, - { - "seed": 57, - "value": 0.0197734243593549, - "path": "" - }, - { - "seed": 97, - "value": 0.07633570792520035, - "path": "" - } - ] - }, - "thresholds": [ - 0.15, - 0.2, - 0.3 - ], - "paths": [ - "", - "", - "", - "", - "" - ] - }, - { - "model_tag": "fcn", - "label": "FCN", - "scope": "train_fire_top10pct", - "scope_label": "top 10%", - "n": 5, - "strict_f1": { - "n": 5, - "mean": 0.011776711221769876, - "std": 0.005117894394726336, - "min": 0.006403642961329112, - "max": 0.019849146486701073 - }, - "tolerance_f1": { - "n": 5, - "mean": 0.22421694517240046, - "std": 0.03980326313080014, - "min": 0.16176542623012913, - "max": 0.2724598040570218 - }, - "union_f1": { - "n": 5, - "mean": 0.4345103901380763, - "std": 0.09251305768823673, - "min": 0.3108270369881256, - "max": 0.5655038878451817 - }, - "difference": { - "n": 5, - "mean": 0.4227336789163064, - "std": 0.0902513347968921, - "min": 0.30442339402679647, - "max": 0.5525178187961205 - }, - "predicted_positive_rate": { - "n": 5, - "mean": 0.0272573463935886, - "std": 0.01635259738911405, - "min": 0.00802908875037103, - "max": 0.049730384881765115 - }, - "seed_values": { - "strict_f1": [ - { - "seed": 7, - "value": 0.008918001736248123, - "path": "" - }, - { - "seed": 17, - "value": 0.006403642961329112, - "path": "" - }, - { - "seed": 37, - "value": 0.019849146486701073, - "path": "" - }, - { - "seed": 57, - "value": 0.010726695875509895, - "path": "" - }, - { - "seed": 97, - "value": 0.012986069049061175, - "path": "" - } - ], - "tolerance_f1": [ - { - "seed": 7, - "value": 0.2270292295163525, - "path": "" - }, - { - "seed": 17, - "value": 0.16176542623012913, - "path": "" - }, - { - "seed": 37, - "value": 0.23420382391023928, - "path": "" - }, - { - "seed": 57, - "value": 0.2256264421482596, - "path": "" - }, - { - "seed": 97, - "value": 0.2724598040570218, - "path": "" - } - ], - "union_f1": [ - { - "seed": 7, - "value": 0.4541508313100257, - "path": "" - }, - { - "seed": 17, - "value": 0.3108270369881256, - "path": "" - }, - { - "seed": 37, - "value": 0.4437163794060617, - "path": "" - }, - { - "seed": 57, - "value": 0.3983538151409869, - "path": "" - }, - { - "seed": 97, - "value": 0.5655038878451817, - "path": "" - } - ], - "difference": [ - { - "seed": 7, - "value": 0.4452328295737776, - "path": "" - }, - { - "seed": 17, - "value": 0.30442339402679647, - "path": "" - }, - { - "seed": 37, - "value": 0.4238672329193606, - "path": "" - }, - { - "seed": 57, - "value": 0.387627119265477, - "path": "" - }, - { - "seed": 97, - "value": 0.5525178187961205, - "path": "" - } - ], - "predicted_positive_rate": [ - { - "seed": 7, - "value": 0.030024982685267636, - "path": "" - }, - { - "seed": 17, - "value": 0.03344711586029484, - "path": "" - }, - { - "seed": 37, - "value": 0.00802908875037103, - "path": "" - }, - { - "seed": 57, - "value": 0.015055159790244386, - "path": "" - }, - { - "seed": 97, - "value": 0.049730384881765115, - "path": "" - } - ] - }, - "thresholds": [ - 0.15, - 0.2, - 0.3 - ], - "paths": [ - "", - "", - "", - "", - "" - ] - }, - { - "model_tag": "fcn", - "label": "FCN", - "scope": "train_fire_top20pct", - "scope_label": "top 20%", - "n": 5, - "strict_f1": { - "n": 5, - "mean": 0.009962019740560447, - "std": 0.004315216342666275, - "min": 0.004578192172408018, - "max": 0.016197783461210574 - }, - "tolerance_f1": { - "n": 5, - "mean": 0.1697921962176252, - "std": 0.03937109759451842, - "min": 0.10554731176319206, - "max": 0.20219147661108805 - }, - "union_f1": { - "n": 5, - "mean": 0.3408594825588104, - "std": 0.08261638767751367, - "min": 0.2167503101126067, - "max": 0.4415948106689657 - }, - "difference": { - "n": 5, - "mean": 0.3308974628182499, - "std": 0.07927520828147509, - "min": 0.2121721179401987, - "max": 0.4298567373898511 - }, - "predicted_positive_rate": { - "n": 5, - "mean": 0.008367841682127397, - "std": 0.0023583622377585994, - "min": 0.005085961657390229, - "max": 0.01035930735930736 - }, - "seed_values": { - "strict_f1": [ - { - "seed": 7, - "value": 0.008704984497972811, - "path": "" - }, - { - "seed": 17, - "value": 0.004578192172408018, - "path": "" - }, - { - "seed": 37, - "value": 0.016197783461210574, - "path": "" - }, - { - "seed": 57, - "value": 0.00859106529209622, - "path": "" - }, - { - "seed": 97, - "value": 0.011738073279114613, - "path": "" - } - ], - "tolerance_f1": [ - { - "seed": 7, - "value": 0.1596269732904185, - "path": "" - }, - { - "seed": 17, - "value": 0.10554731176319206, - "path": "" - }, - { - "seed": 37, - "value": 0.19512358967789506, - "path": "" - }, - { - "seed": 57, - "value": 0.18647162974553239, - "path": "" - }, - { - "seed": 97, - "value": 0.20219147661108805, - "path": "" - } - ], - "union_f1": [ - { - "seed": 7, - "value": 0.3313747927063083, - "path": "" - }, - { - "seed": 17, - "value": 0.2167503101126067, - "path": "" - }, - { - "seed": 37, - "value": 0.3809676016171751, - "path": "" - }, - { - "seed": 57, - "value": 0.33360989768899596, - "path": "" - }, - { - "seed": 97, - "value": 0.4415948106689657, - "path": "" - } - ], - "difference": [ - { - "seed": 7, - "value": 0.3226698082083355, - "path": "" - }, - { - "seed": 17, - "value": 0.2121721179401987, - "path": "" - }, - { - "seed": 37, - "value": 0.3647698181559645, - "path": "" - }, - { - "seed": 57, - "value": 0.3250188323968997, - "path": "" - }, - { - "seed": 97, - "value": 0.4298567373898511, - "path": "" - } - ], - "predicted_positive_rate": [ - { - "seed": 7, - "value": 0.009654916512059368, - "path": "" - }, - { - "seed": 17, - "value": 0.01035930735930736, - "path": "" - }, - { - "seed": 37, - "value": 0.005085961657390229, - "path": "" - }, - { - "seed": 57, - "value": 0.010080395794681509, - "path": "" - }, - { - "seed": 97, - "value": 0.006658627087198516, - "path": "" - } - ] - }, - "thresholds": [ - 0.2, - 0.3 - ], - "paths": [ - "", - "", - "", - "", - "" - ] - }, - { - "model_tag": "fengwu", - "label": "FengWu", - "scope": "full_domain", - "scope_label": "global", - "n": 5, - "strict_f1": { - "n": 5, - "mean": 0.0026125750208171442, - "std": 0.0007567108180601367, - "min": 0.0018414421399496025, - "max": 0.003438698584843275 - }, - "tolerance_f1": { - "n": 5, - "mean": 0.12004984556809643, - "std": 0.06023907872870804, - "min": 0.042906126190853267, - "max": 0.20002472134774485 - }, - "union_f1": { - "n": 5, - "mean": 0.24102217940052184, - "std": 0.13629274822774434, - "min": 0.06919785857013828, - "max": 0.4405198598043062 - }, - "difference": { - "n": 5, - "mean": 0.23840960437970468, - "std": 0.13573634707074209, - "min": 0.06735555938074993, - "max": 0.43726313068468925 - }, - "predicted_positive_rate": { - "n": 5, - "mean": 0.004276685219542362, - "std": 0.004829522662220775, - "min": 0.00042881880024737165, - "max": 0.012518243661100804 - }, - "seed_values": { - "strict_f1": [ - { - "seed": 7, - "value": 0.00268370607028754, - "path": "" - }, - { - "seed": 17, - "value": 0.003438698584843275, - "path": "" - }, - { - "seed": 37, - "value": 0.0018414421399496025, - "path": "" - }, - { - "seed": 57, - "value": 0.001842299189388357, - "path": "" - }, - { - "seed": 97, - "value": 0.0032567291196169464, - "path": "" - } - ], - "tolerance_f1": [ - { - "seed": 7, - "value": 0.09710793321083327, - "path": "" - }, - { - "seed": 17, - "value": 0.1569315132056949, - "path": "" - }, - { - "seed": 37, - "value": 0.10327893388535589, - "path": "" - }, - { - "seed": 57, - "value": 0.042906126190853267, - "path": "" - }, - { - "seed": 97, - "value": 0.20002472134774485, - "path": "" - } - ], - "union_f1": [ - { - "seed": 7, - "value": 0.186659362535767, - "path": "" - }, - { - "seed": 17, - "value": 0.283249876555753, - "path": "" - }, - { - "seed": 37, - "value": 0.2254839395366447, - "path": "" - }, - { - "seed": 57, - "value": 0.06919785857013828, - "path": "" - }, - { - "seed": 97, - "value": 0.4405198598043062, - "path": "" - } - ], - "difference": [ - { - "seed": 7, - "value": 0.18397565646547948, - "path": "" - }, - { - "seed": 17, - "value": 0.2798111779709097, - "path": "" - }, - { - "seed": 37, - "value": 0.2236424973966951, - "path": "" - }, - { - "seed": 57, - "value": 0.06735555938074993, - "path": "" - }, - { - "seed": 97, - "value": 0.43726313068468925, - "path": "" - } - ], - "predicted_positive_rate": [ - { - "seed": 7, - "value": 0.0016931354359925788, - "path": "" - }, - { - "seed": 17, - "value": 0.004433395176252319, - "path": "" - }, - { - "seed": 37, - "value": 0.0023098330241187385, - "path": "" - }, - { - "seed": 57, - "value": 0.00042881880024737165, - "path": "" - }, - { - "seed": 97, - "value": 0.012518243661100804, - "path": "" - } - ] - }, - "thresholds": [ - 0.2, - 0.3, - 0.5 - ], - "paths": [ - "", - "", - "", - "", - "" - ] - }, - { - "model_tag": "fengwu", - "label": "FengWu", - "scope": "train_fire_top05pct", - "scope_label": "top 5%", - "n": 5, - "strict_f1": { - "n": 5, - "mean": 0.015694746589097253, - "std": 0.003591671570543508, - "min": 0.01267427122940431, - "max": 0.0205686630369026 - }, - "tolerance_f1": { - "n": 5, - "mean": 0.16276279599467128, - "std": 0.03702437856278332, - "min": 0.12305221580797777, - "max": 0.21207047330588913 - }, - "union_f1": { - "n": 5, - "mean": 0.3010546945713337, - "std": 0.05010267654001306, - "min": 0.22237658124637444, - "max": 0.3374691422247865 - }, - "difference": { - "n": 5, - "mean": 0.28535994798223646, - "std": 0.04769577870036795, - "min": 0.20932175095916816, - "max": 0.31898912374476807 - }, - "predicted_positive_rate": { - "n": 5, - "mean": 0.00600375977045612, - "std": 0.0030413161402819498, - "min": 0.0030746017611556348, - "max": 0.010032650638171565 - }, - "seed_values": { - "strict_f1": [ - { - "seed": 7, - "value": 0.013054830287206266, - "path": "" - }, - { - "seed": 17, - "value": 0.01848001848001848, - "path": "" - }, - { - "seed": 37, - "value": 0.01267427122940431, - "path": "" - }, - { - "seed": 57, - "value": 0.0205686630369026, - "path": "" - }, - { - "seed": 97, - "value": 0.013695949911954608, - "path": "" - } - ], - "tolerance_f1": [ - { - "seed": 7, - "value": 0.12305221580797777, - "path": "" - }, - { - "seed": 17, - "value": 0.21207047330588913, - "path": "" - }, - { - "seed": 37, - "value": 0.13288915578187116, - "path": "" - }, - { - "seed": 57, - "value": 0.1866312847963942, - "path": "" - }, - { - "seed": 97, - "value": 0.15917085028122405, - "path": "" - } - ], - "union_f1": [ - { - "seed": 7, - "value": 0.22237658124637444, - "path": "" - }, - { - "seed": 17, - "value": 0.3374691422247865, - "path": "" - }, - { - "seed": 37, - "value": 0.27966639148093675, - "path": "" - }, - { - "seed": 57, - "value": 0.3374418848724159, - "path": "" - }, - { - "seed": 97, - "value": 0.3283194730321549, - "path": "" - } - ], - "difference": [ - { - "seed": 7, - "value": 0.20932175095916816, - "path": "" - }, - { - "seed": 17, - "value": 0.31898912374476807, - "path": "" - }, - { - "seed": 37, - "value": 0.26699212025153246, - "path": "" - }, - { - "seed": 57, - "value": 0.3168732218355133, - "path": "" - }, - { - "seed": 97, - "value": 0.3146235231202003, - "path": "" - } - ], - "predicted_positive_rate": [ - { - "seed": 7, - "value": 0.0030746017611556348, - "path": "" - }, - { - "seed": 17, - "value": 0.008098347679825863, - "path": "" - }, - { - "seed": 37, - "value": 0.0032452755515979025, - "path": "" - }, - { - "seed": 57, - "value": 0.005567923221529633, - "path": "" - }, - { - "seed": 97, - "value": 0.010032650638171565, - "path": "" - } - ] - }, - "thresholds": [ - 0.2, - 0.3 - ], - "paths": [ - "", - "", - "", - "", - "" - ] - }, - { - "model_tag": "fengwu", - "label": "FengWu", - "scope": "train_fire_top10pct", - "scope_label": "top 10%", - "n": 5, - "strict_f1": { - "n": 5, - "mean": 0.012426852806462965, - "std": 0.005333227666358275, - "min": 0.005460750853242321, - "max": 0.02015411973918198 - }, - "tolerance_f1": { - "n": 5, - "mean": 0.12950332830320144, - "std": 0.05605188948381181, - "min": 0.04684227229450968, - "max": 0.1840127014362662 - }, - "union_f1": { - "n": 5, - "mean": 0.24185379631697046, - "std": 0.0868538356837431, - "min": 0.10984387215996587, - "max": 0.33432003556596124 - }, - "difference": { - "n": 5, - "mean": 0.2294269435105075, - "std": 0.0818631441187217, - "min": 0.10438312130672354, - "max": 0.3141659158267793 - }, - "predicted_positive_rate": { - "n": 5, - "mean": 0.002608835460571881, - "std": 0.001935467532779746, - "min": 0.0004947066389630949, - "max": 0.005720045513010785 - }, - "seed_values": { - "strict_f1": [ - { - "seed": 7, - "value": 0.011185682326621925, - "path": "" - }, - { - "seed": 17, - "value": 0.014059753954305799, - "path": "" - }, - { - "seed": 37, - "value": 0.011273957158962795, - "path": "" - }, - { - "seed": 57, - "value": 0.02015411973918198, - "path": "" - }, - { - "seed": 97, - "value": 0.005460750853242321, - "path": "" - } - ], - "tolerance_f1": [ - { - "seed": 7, - "value": 0.11778287307219655, - "path": "" - }, - { - "seed": 17, - "value": 0.17969557482138315, - "path": "" - }, - { - "seed": 37, - "value": 0.1191832198916516, - "path": "" - }, - { - "seed": 57, - "value": 0.1840127014362662, - "path": "" - }, - { - "seed": 97, - "value": 0.04684227229450968, - "path": "" - } - ], - "union_f1": [ - { - "seed": 7, - "value": 0.21014056663719285, - "path": "" - }, - { - "seed": 17, - "value": 0.29555470142676327, - "path": "" - }, - { - "seed": 37, - "value": 0.259409805794969, - "path": "" - }, - { - "seed": 57, - "value": 0.33432003556596124, - "path": "" - }, - { - "seed": 97, - "value": 0.10984387215996587, - "path": "" - } - ], - "difference": [ - { - "seed": 7, - "value": 0.19895488431057093, - "path": "" - }, - { - "seed": 17, - "value": 0.28149494747245746, - "path": "" - }, - { - "seed": 37, - "value": 0.24813584863600618, - "path": "" - }, - { - "seed": 57, - "value": 0.3141659158267793, - "path": "" - }, - { - "seed": 97, - "value": 0.10438312130672354, - "path": "" - } - ], - "predicted_positive_rate": [ - { - "seed": 7, - "value": 0.0019998515880083113, - "path": "" - }, - { - "seed": 17, - "value": 0.005720045513010785, - "path": "" - }, - { - "seed": 37, - "value": 0.0019738794894627485, - "path": "" - }, - { - "seed": 57, - "value": 0.0028556940734144652, - "path": "" - }, - { - "seed": 97, - "value": 0.0004947066389630949, - "path": "" - } - ] - }, - "thresholds": [ - 0.2, - 0.3, - 0.5 - ], - "paths": [ - "", - "", - "", - "", - "" - ] - }, - { - "model_tag": "fengwu", - "label": "FengWu", - "scope": "train_fire_top20pct", - "scope_label": "top 20%", - "n": 5, - "strict_f1": { - "n": 5, - "mean": 0.01119211887721548, - "std": 0.005022919520207697, - "min": 0.004807692307692307, - "max": 0.018831348656881748 - }, - "tolerance_f1": { - "n": 5, - "mean": 0.11950764890450612, - "std": 0.05074475609731675, - "min": 0.04393081279657125, - "max": 0.17388386719173746 - }, - "union_f1": { - "n": 5, - "mean": 0.22785982657353704, - "std": 0.07911462942092182, - "min": 0.1067457375833951, - "max": 0.31802311631118385 - }, - "difference": { - "n": 5, - "mean": 0.21666770769632157, - "std": 0.07443767874640678, - "min": 0.1019380452757028, - "max": 0.2991917676543021 - }, - "predicted_positive_rate": { - "n": 5, - "mean": 0.0015006802721088436, - "std": 0.0012055357942519695, - "min": 0.0003116883116883117, - "max": 0.0035126777983920843 - }, - "seed_values": { - "strict_f1": [ - { - "seed": 7, - "value": 0.010309278350515464, - "path": "" - }, - { - "seed": 17, - "value": 0.011695906432748539, - "path": "" - }, - { - "seed": 37, - "value": 0.010316368638239341, - "path": "" - }, - { - "seed": 57, - "value": 0.018831348656881748, - "path": "" - }, - { - "seed": 97, - "value": 0.004807692307692307, - "path": "" - } - ], - "tolerance_f1": [ - { - "seed": 7, - "value": 0.11130870584431331, - "path": "" - }, - { - "seed": 17, - "value": 0.1579992352547558, - "path": "" - }, - { - "seed": 37, - "value": 0.11041562343515275, - "path": "" - }, - { - "seed": 57, - "value": 0.17388386719173746, - "path": "" - }, - { - "seed": 97, - "value": 0.04393081279657125, - "path": "" - } - ], - "union_f1": [ - { - "seed": 7, - "value": 0.20443519333920424, - "path": "" - }, - { - "seed": 17, - "value": 0.26532858639687246, - "path": "" - }, - { - "seed": 37, - "value": 0.24476649923702953, - "path": "" - }, - { - "seed": 57, - "value": 0.31802311631118385, - "path": "" - }, - { - "seed": 97, - "value": 0.1067457375833951, - "path": "" - } - ], - "difference": [ - { - "seed": 7, - "value": 0.19412591498868878, - "path": "" - }, - { - "seed": 17, - "value": 0.2536326799641239, - "path": "" - }, - { - "seed": 37, - "value": 0.2344501305987902, - "path": "" - }, - { - "seed": 57, - "value": 0.2991917676543021, - "path": "" - }, - { - "seed": 97, - "value": 0.1019380452757028, - "path": "" - } - ], - "predicted_positive_rate": [ - { - "seed": 7, - "value": 0.0010822510822510823, - "path": "" - }, - { - "seed": 17, - "value": 0.0035126777983920843, - "path": "" - }, - { - "seed": 37, - "value": 0.0010810142238713667, - "path": "" - }, - { - "seed": 57, - "value": 0.0015157699443413728, - "path": "" - }, - { - "seed": 97, - "value": 0.0003116883116883117, - "path": "" - } - ] - }, - "thresholds": [ - 0.2, - 0.3, - 0.5 - ], - "paths": [ - "", - "", - "", - "", - "" - ] - }, - { - "model_tag": "fuxi", - "label": "FuXi", - "scope": "full_domain", - "scope_label": "global", - "n": 5, - "strict_f1": { - "n": 5, - "mean": 0.0037736840271503577, - "std": 0.0012118699857535593, - "min": 0.0029424127784783522, - "max": 0.005897939994871356 - }, - "tolerance_f1": { - "n": 5, - "mean": 0.21032274703934553, - "std": 0.04821145875766877, - "min": 0.13539446199130012, - "max": 0.2682493391647262 - }, - "union_f1": { - "n": 5, - "mean": 0.37288797408747854, - "std": 0.09446975287636859, - "min": 0.22039566333970034, - "max": 0.4541140407970205 - }, - "difference": { - "n": 5, - "mean": 0.36911429006032814, - "std": 0.09432711805206372, - "min": 0.217453250561222, - "max": 0.45054053460571325 - }, - "predicted_positive_rate": { - "n": 5, - "mean": 0.004504539270253556, - "std": 0.00277275768973151, - "min": 0.001522943722943723, - "max": 0.008687322201607915 - }, - "seed_values": { - "strict_f1": [ - { - "seed": 7, - "value": 0.0033573873422136956, - "path": "" - }, - { - "seed": 17, - "value": 0.003097173828881146, - "path": "" - }, - { - "seed": 37, - "value": 0.0029424127784783522, - "path": "" - }, - { - "seed": 57, - "value": 0.005897939994871356, - "path": "" - }, - { - "seed": 97, - "value": 0.003573506191307238, - "path": "" - } - ], - "tolerance_f1": [ - { - "seed": 7, - "value": 0.2682493391647262, - "path": "" - }, - { - "seed": 17, - "value": 0.20441214417386547, - "path": "" - }, - { - "seed": 37, - "value": 0.13539446199130012, - "path": "" - }, - { - "seed": 57, - "value": 0.22566599998043468, - "path": "" - }, - { - "seed": 97, - "value": 0.21789178988640112, - "path": "" - } - ], - "union_f1": [ - { - "seed": 7, - "value": 0.44868694500233336, - "path": "" - }, - { - "seed": 17, - "value": 0.37735961055895995, - "path": "" - }, - { - "seed": 37, - "value": 0.22039566333970034, - "path": "" - }, - { - "seed": 57, - "value": 0.3638836107393784, - "path": "" - }, - { - "seed": 97, - "value": 0.4541140407970205, - "path": "" - } - ], - "difference": [ - { - "seed": 7, - "value": 0.44532955766011967, - "path": "" - }, - { - "seed": 17, - "value": 0.3742624367300788, - "path": "" - }, - { - "seed": 37, - "value": 0.217453250561222, - "path": "" - }, - { - "seed": 57, - "value": 0.35798567074450705, - "path": "" - }, - { - "seed": 97, - "value": 0.45054053460571325, - "path": "" - } - ], - "predicted_positive_rate": [ - { - "seed": 7, - "value": 0.005430797773654916, - "path": "" - }, - { - "seed": 17, - "value": 0.004230179344465059, - "path": "" - }, - { - "seed": 37, - "value": 0.001522943722943723, - "path": "" - }, - { - "seed": 57, - "value": 0.0026514533085961658, - "path": "" - }, - { - "seed": 97, - "value": 0.008687322201607915, - "path": "" - } - ] - }, - "thresholds": [ - 0.2, - 0.3 - ], - "paths": [ - "", - "", - "", - "", - "" - ] - }, - { - "model_tag": "fuxi", - "label": "FuXi", - "scope": "train_fire_top05pct", - "scope_label": "top 5%", - "n": 5, - "strict_f1": { - "n": 5, - "mean": 0.020306854641425946, - "std": 0.006799890593691795, - "min": 0.012024257632789628, - "max": 0.030370370370370364 - }, - "tolerance_f1": { - "n": 5, - "mean": 0.31894420698818526, - "std": 0.047331205244476655, - "min": 0.26086649919553234, - "max": 0.38200179460101663 - }, - "union_f1": { - "n": 5, - "mean": 0.5393080442029143, - "std": 0.08382153402586862, - "min": 0.4073509199876191, - "max": 0.6304826850687859 - }, - "difference": { - "n": 5, - "mean": 0.5190011895614883, - "std": 0.08687800582865718, - "min": 0.3862603850081952, - "max": 0.6138219771674631 - }, - "predicted_positive_rate": { - "n": 5, - "mean": 0.029315326011675073, - "std": 0.026391672443609994, - "min": 0.004068962105471455, - "max": 0.0665751459384585 - }, - "seed_values": { - "strict_f1": [ - { - "seed": 7, - "value": 0.016660707901322847, - "path": "" - }, - { - "seed": 17, - "value": 0.02138840232322301, - "path": "" - }, - { - "seed": 37, - "value": 0.02109053497942387, - "path": "" - }, - { - "seed": 57, - "value": 0.030370370370370364, - "path": "" - }, - { - "seed": 97, - "value": 0.012024257632789628, - "path": "" - } - ], - "tolerance_f1": [ - { - "seed": 7, - "value": 0.38200179460101663, - "path": "" - }, - { - "seed": 17, - "value": 0.30180702036802187, - "path": "" - }, - { - "seed": 37, - "value": 0.26086649919553234, - "path": "" - }, - { - "seed": 57, - "value": 0.3499638832422191, - "path": "" - }, - { - "seed": 97, - "value": 0.30008183753413636, - "path": "" - } - ], - "union_f1": [ - { - "seed": 7, - "value": 0.6304826850687859, - "path": "" - }, - { - "seed": 17, - "value": 0.551963463682504, - "path": "" - }, - { - "seed": 37, - "value": 0.4073509199876191, - "path": "" - }, - { - "seed": 57, - "value": 0.5231807249206608, - "path": "" - }, - { - "seed": 97, - "value": 0.5835624273550017, - "path": "" - } - ], - "difference": [ - { - "seed": 7, - "value": 0.6138219771674631, - "path": "" - }, - { - "seed": 17, - "value": 0.530575061359281, - "path": "" - }, - { - "seed": 37, - "value": 0.3862603850081952, - "path": "" - }, - { - "seed": 57, - "value": 0.49281035455029043, - "path": "" - }, - { - "seed": 97, - "value": 0.5715381697222122, - "path": "" - } - ], - "predicted_positive_rate": [ - { - "seed": 7, - "value": 0.0665751459384585, - "path": "" - }, - { - "seed": 17, - "value": 0.024220837043633127, - "path": "" - }, - { - "seed": 37, - "value": 0.007007519540912239, - "path": "" - }, - { - "seed": 57, - "value": 0.004068962105471455, - "path": "" - }, - { - "seed": 97, - "value": 0.04470416542990007, - "path": "" - } - ] - }, - "thresholds": [ - 0.1, - 0.15, - 0.2, - 0.3 - ], - "paths": [ - "", - "", - "", - "", - "" - ] - }, - { - "model_tag": "fuxi", - "label": "FuXi", - "scope": "train_fire_top10pct", - "scope_label": "top 10%", - "n": 5, - "strict_f1": { - "n": 5, - "mean": 0.016542308314081426, - "std": 0.007315696478545316, - "min": 0.008170871559633027, - "max": 0.026973684210526316 - }, - "tolerance_f1": { - "n": 5, - "mean": 0.24012808383234296, - "std": 0.05778381935942338, - "min": 0.1656142112861716, - "max": 0.3231296638237823 - }, - "union_f1": { - "n": 5, - "mean": 0.4021395209358275, - "std": 0.09930723729972839, - "min": 0.24700654033204764, - "max": 0.48397755354947186 - }, - "difference": { - "n": 5, - "mean": 0.38559721262174607, - "std": 0.09774430533636969, - "min": 0.23030507895417707, - "max": 0.45700386933894555 - }, - "predicted_positive_rate": { - "n": 5, - "mean": 0.00519664588898783, - "std": 0.006074020737200343, - "min": 0.0016448995745522905, - "max": 0.01593821114079351 - }, - "seed_values": { - "strict_f1": [ - { - "seed": 7, - "value": 0.011389521640091117, - "path": "" - }, - { - "seed": 17, - "value": 0.019476002782286114, - "path": "" - }, - { - "seed": 37, - "value": 0.016701461377870565, - "path": "" - }, - { - "seed": 57, - "value": 0.026973684210526316, - "path": "" - }, - { - "seed": 97, - "value": 0.008170871559633027, - "path": "" - } - ], - "tolerance_f1": [ - { - "seed": 7, - "value": 0.2251054852320675, - "path": "" - }, - { - "seed": 17, - "value": 0.26166119072060445, - "path": "" - }, - { - "seed": 37, - "value": 0.1656142112861716, - "path": "" - }, - { - "seed": 57, - "value": 0.3231296638237823, - "path": "" - }, - { - "seed": 97, - "value": 0.22512986809908905, - "path": "" - } - ], - "union_f1": [ - { - "seed": 7, - "value": 0.35879236229131267, - "path": "" - }, - { - "seed": 17, - "value": 0.45937946725189766, - "path": "" - }, - { - "seed": 37, - "value": 0.24700654033204764, - "path": "" - }, - { - "seed": 57, - "value": 0.48397755354947186, - "path": "" - }, - { - "seed": 97, - "value": 0.4615416812544075, - "path": "" - } - ], - "difference": [ - { - "seed": 7, - "value": 0.34740284065122157, - "path": "" - }, - { - "seed": 17, - "value": 0.43990346446961154, - "path": "" - }, - { - "seed": 37, - "value": 0.23030507895417707, - "path": "" - }, - { - "seed": 57, - "value": 0.45700386933894555, - "path": "" - }, - { - "seed": 97, - "value": 0.45337080969477445, - "path": "" - } - ], - "predicted_positive_rate": [ - { - "seed": 7, - "value": 0.0019404867913327397, - "path": "" - }, - { - "seed": 17, - "value": 0.0040170179083803305, - "path": "" - }, - { - "seed": 37, - "value": 0.0016448995745522905, - "path": "" - }, - { - "seed": 57, - "value": 0.002442614029880281, - "path": "" - }, - { - "seed": 97, - "value": 0.01593821114079351, - "path": "" - } - ] - }, - "thresholds": [ - 0.2, - 0.3 - ], - "paths": [ - "", - "", - "", - "", - "" - ] - }, - { - "model_tag": "fuxi", - "label": "FuXi", - "scope": "train_fire_top20pct", - "scope_label": "top 20%", - "n": 5, - "strict_f1": { - "n": 5, - "mean": 0.013646120927699812, - "std": 0.006773001125535398, - "min": 0.006516342573438146, - "max": 0.0239695995323005 - }, - "tolerance_f1": { - "n": 5, - "mean": 0.21954844569166232, - "std": 0.058600538898278126, - "min": 0.15340459969134643, - "max": 0.30046102485294895 - }, - "union_f1": { - "n": 5, - "mean": 0.3673143252863447, - "std": 0.10028910465565644, - "min": 0.23133715755118187, - "max": 0.4554233007850691 - }, - "difference": { - "n": 5, - "mean": 0.3536682043586449, - "std": 0.09922328477614482, - "min": 0.21606998197866278, - "max": 0.43708558281057147 - }, - "predicted_positive_rate": { - "n": 5, - "mean": 0.003620779220779221, - "std": 0.004346389540674709, - "min": 0.0009029066171923315, - "max": 0.011240568954854669 - }, - "seed_values": { - "strict_f1": [ - { - "seed": 7, - "value": 0.008802816901408451, - "path": "" - }, - { - "seed": 17, - "value": 0.013674670058832883, - "path": "" - }, - { - "seed": 37, - "value": 0.015267175572519083, - "path": "" - }, - { - "seed": 57, - "value": 0.0239695995323005, - "path": "" - }, - { - "seed": 97, - "value": 0.006516342573438146, - "path": "" - } - ], - "tolerance_f1": [ - { - "seed": 7, - "value": 0.18265891116039673, - "path": "" - }, - { - "seed": 17, - "value": 0.25519274478475146, - "path": "" - }, - { - "seed": 37, - "value": 0.15340459969134643, - "path": "" - }, - { - "seed": 57, - "value": 0.30046102485294895, - "path": "" - }, - { - "seed": 97, - "value": 0.20602494796886814, - "path": "" - } - ], - "union_f1": [ - { - "seed": 7, - "value": 0.29083501784981214, - "path": "" - }, - { - "seed": 17, - "value": 0.4153742248616508, - "path": "" - }, - { - "seed": 37, - "value": 0.23133715755118187, - "path": "" - }, - { - "seed": 57, - "value": 0.4554233007850691, - "path": "" - }, - { - "seed": 97, - "value": 0.4436019253840096, - "path": "" - } - ], - "difference": [ - { - "seed": 7, - "value": 0.2820322009484037, - "path": "" - }, - { - "seed": 17, - "value": 0.40169955480281794, - "path": "" - }, - { - "seed": 37, - "value": 0.21606998197866278, - "path": "" - }, - { - "seed": 57, - "value": 0.4314537012527686, - "path": "" - }, - { - "seed": 97, - "value": 0.43708558281057147, - "path": "" - } - ], - "predicted_positive_rate": [ - { - "seed": 7, - "value": 0.0013902288188002473, - "path": "" - }, - { - "seed": 17, - "value": 0.0031719233147804574, - "path": "" - }, - { - "seed": 37, - "value": 0.0009029066171923315, - "path": "" - }, - { - "seed": 57, - "value": 0.0013982683982683983, - "path": "" - }, - { - "seed": 97, - "value": 0.011240568954854669, - "path": "" - } - ] - }, - "thresholds": [ - 0.2, - 0.3 - ], - "paths": [ - "", - "", - "", - "", - "" - ] - }, - { - "model_tag": "pangu_weather", - "label": "Pangu-Weather", - "scope": "full_domain", - "scope_label": "global", - "n": 5, - "strict_f1": { - "n": 5, - "mean": 0.0006175979748444972, - "std": 6.723431511345116e-05, - "min": 0.0005395660023552084, - "max": 0.000687991383709005 - }, - "tolerance_f1": { - "n": 5, - "mean": 0.08180329398071068, - "std": 0.013544748931105265, - "min": 0.06046181002685303, - "max": 0.09170559714407109 - }, - "union_f1": { - "n": 5, - "mean": 0.22056244847642267, - "std": 0.03734327197007739, - "min": 0.16301936607754167, - "max": 0.2480828253168327 - }, - "difference": { - "n": 5, - "mean": 0.2199448505015782, - "std": 0.03728443961138173, - "min": 0.1624675064035813, - "max": 0.24742507585853188 - }, - "predicted_positive_rate": { - "n": 5, - "mean": 0.6338042300556587, - "std": 0.2507607336766575, - "min": 0.21222844774273344, - "max": 0.8827594310451453 - }, - "seed_values": { - "strict_f1": [ - { - "seed": 1, - "value": 0.0005518596739603732, - "path": "" - }, - { - "seed": 7, - "value": 0.0006577494583008349, - "path": "" - }, - { - "seed": 42, - "value": 0.0005395660023552084, - "path": "" - }, - { - "seed": 99, - "value": 0.000687991383709005, - "path": "" - }, - { - "seed": 123, - "value": 0.0006508233558970649, - "path": "" - } - ], - "tolerance_f1": [ - { - "seed": 1, - "value": 0.06046181002685303, - "path": "" - }, - { - "seed": 7, - "value": 0.09170559714407109, - "path": "" - }, - { - "seed": 42, - "value": 0.07610963330033767, - "path": "" - }, - { - "seed": 99, - "value": 0.08961425169301492, - "path": "" - }, - { - "seed": 123, - "value": 0.09112517773927671, - "path": "" - } - ], - "union_f1": [ - { - "seed": 1, - "value": 0.16301936607754167, - "path": "" - }, - { - "seed": 7, - "value": 0.2480828253168327, - "path": "" - }, - { - "seed": 42, - "value": 0.20220259920022512, - "path": "" - }, - { - "seed": 99, - "value": 0.24375811032725037, - "path": "" - }, - { - "seed": 123, - "value": 0.2457493414602635, - "path": "" - } - ], - "difference": [ - { - "seed": 1, - "value": 0.1624675064035813, - "path": "" - }, - { - "seed": 7, - "value": 0.24742507585853188, - "path": "" - }, - { - "seed": 42, - "value": 0.2016630331978699, - "path": "" - }, - { - "seed": 99, - "value": 0.24307011894354136, - "path": "" - }, - { - "seed": 123, - "value": 0.24509851810436645, - "path": "" - } - ], - "predicted_positive_rate": [ - { - "seed": 1, - "value": 0.21222844774273344, - "path": "" - }, - { - "seed": 7, - "value": 0.7071789734075449, - "path": "" - }, - { - "seed": 42, - "value": 0.8827594310451453, - "path": "" - }, - { - "seed": 99, - "value": 0.655947804576376, - "path": "" - }, - { - "seed": 123, - "value": 0.7109064935064935, - "path": "" - } - ] - }, - "thresholds": [ - 0.00024381681578233838, - 0.0002703598001971841, - 0.00027441306156106293, - 0.0002784867538139224, - 0.00027882601716555655 - ], - "paths": [ - "", - "", - "", - "", - "" - ] - }, - { - "model_tag": "pangu_weather", - "label": "Pangu-Weather", - "scope": "train_fire_top05pct", - "scope_label": "top 5%", - "n": 5, - "strict_f1": { - "n": 5, - "mean": 0.010646547680643631, - "std": 0.009091660121962298, - "min": 0.005639067545640032, - "max": 0.026612927125626056 - }, - "tolerance_f1": { - "n": 5, - "mean": 0.16674283792931957, - "std": 0.04706476664025482, - "min": 0.1110438397786047, - "max": 0.24179585238367046 - }, - "union_f1": { - "n": 5, - "mean": 0.3913796290737717, - "std": 0.07015617234020738, - "min": 0.2883417650922405, - "max": 0.4864922341150057 - }, - "difference": { - "n": 5, - "mean": 0.3807330813931281, - "std": 0.06491382662211008, - "min": 0.27868986084470576, - "max": 0.4598793069893796 - }, - "predicted_positive_rate": { - "n": 5, - "mean": 0.6241896705253784, - "std": 0.41065405269695054, - "min": 0.08085485307212822, - "max": 0.9213416444048679 - }, - "seed_values": { - "strict_f1": [ - { - "seed": 1, - "value": 0.026612927125626056, - "path": "" - }, - { - "seed": 7, - "value": 0.005648734389013078, - "path": "" - }, - { - "seed": 42, - "value": 0.00568010509540423, - "path": "" - }, - { - "seed": 99, - "value": 0.009651904247534756, - "path": "" - }, - { - "seed": 123, - "value": 0.005639067545640032, - "path": "" - } - ], - "tolerance_f1": [ - { - "seed": 1, - "value": 0.24179585238367046, - "path": "" - }, - { - "seed": 7, - "value": 0.1601110414083686, - "path": "" - }, - { - "seed": 42, - "value": 0.16063421440737302, - "path": "" - }, - { - "seed": 99, - "value": 0.1110438397786047, - "path": "" - }, - { - "seed": 123, - "value": 0.16012924166858108, - "path": "" - } - ], - "union_f1": [ - { - "seed": 1, - "value": 0.4864922341150057, - "path": "" - }, - { - "seed": 7, - "value": 0.39479065437879374, - "path": "" - }, - { - "seed": 42, - "value": 0.3925140767903833, - "path": "" - }, - { - "seed": 99, - "value": 0.2883417650922405, - "path": "" - }, - { - "seed": 123, - "value": 0.3947594149924353, - "path": "" - } - ], - "difference": [ - { - "seed": 1, - "value": 0.4598793069893796, - "path": "" - }, - { - "seed": 7, - "value": 0.38914191998978065, - "path": "" - }, - { - "seed": 42, - "value": 0.3868339716949791, - "path": "" - }, - { - "seed": 99, - "value": 0.27868986084470576, - "path": "" - }, - { - "seed": 123, - "value": 0.38912034744679525, - "path": "" - } - ], - "predicted_positive_rate": [ - { - "seed": 1, - "value": 0.08085485307212822, - "path": "" - }, - { - "seed": 7, - "value": 0.9213416444048679, - "path": "" - }, - { - "seed": 42, - "value": 0.9162387454239636, - "path": "" - }, - { - "seed": 99, - "value": 0.28134213911150685, - "path": "" - }, - { - "seed": 123, - "value": 0.9211709706144257, - "path": "" - } - ] - }, - "thresholds": [ - 0.0002431461471132934, - 0.0002697413437999785, - 0.00027438640245236456, - 0.0002780304348561913, - 0.00028155450127087533 - ], - "paths": [ - "", - "", - "", - "", - "" - ] - }, - { - "model_tag": "pangu_weather", - "label": "Pangu-Weather", - "scope": "train_fire_top10pct", - "scope_label": "top 10%", - "n": 5, - "strict_f1": { - "n": 5, - "mean": 0.007206875839483156, - "std": 0.010865089659576754, - "min": 0.0011827321111768183, - "max": 0.02660504251473943 - }, - "tolerance_f1": { - "n": 5, - "mean": 0.10446479485399655, - "std": 0.08187646601609166, - "min": 0.019439944457301548, - "max": 0.2411933776075557 - }, - "union_f1": { - "n": 5, - "mean": 0.24123553210507903, - "std": 0.1613799726515661, - "min": 0.029367206619275145, - "max": 0.4840644650115095 - }, - "difference": { - "n": 5, - "mean": 0.2340286562655959, - "std": 0.15198406003212697, - "min": 0.02818447450809833, - "max": 0.4574594224967701 - }, - "predicted_positive_rate": { - "n": 5, - "mean": 0.5820859305431878, - "std": 0.5127551339472974, - "min": 0.0007742158899772435, - "max": 0.9565152864351439 - }, - "seed_values": { - "strict_f1": [ - { - "seed": 1, - "value": 0.02660504251473943, - "path": "" - }, - { - "seed": 7, - "value": 0.00275028568108307, - "path": "" - }, - { - "seed": 42, - "value": 0.002750953470140531, - "path": "" - }, - { - "seed": 99, - "value": 0.0011827321111768183, - "path": "" - }, - { - "seed": 123, - "value": 0.0027453654202759313, - "path": "" - } - ], - "tolerance_f1": [ - { - "seed": 1, - "value": 0.2411933776075557, - "path": "" - }, - { - "seed": 7, - "value": 0.08722255267001126, - "path": "" - }, - { - "seed": 42, - "value": 0.08724283500251707, - "path": "" - }, - { - "seed": 99, - "value": 0.019439944457301548, - "path": "" - }, - { - "seed": 123, - "value": 0.08722526453259725, - "path": "" - } - ], - "union_f1": [ - { - "seed": 1, - "value": 0.4840644650115095, - "path": "" - }, - { - "seed": 7, - "value": 0.23118547623512836, - "path": "" - }, - { - "seed": 42, - "value": 0.23040938884616793, - "path": "" - }, - { - "seed": 99, - "value": 0.029367206619275145, - "path": "" - }, - { - "seed": 123, - "value": 0.23115112381331424, - "path": "" - } - ], - "difference": [ - { - "seed": 1, - "value": 0.4574594224967701, - "path": "" - }, - { - "seed": 7, - "value": 0.2284351905540453, - "path": "" - }, - { - "seed": 42, - "value": 0.2276584353760274, - "path": "" - }, - { - "seed": 99, - "value": 0.02818447450809833, - "path": "" - }, - { - "seed": 123, - "value": 0.2284057583930383, - "path": "" - } - ], - "predicted_positive_rate": [ - { - "seed": 1, - "value": 0.04042742653606411, - "path": "" - }, - { - "seed": 7, - "value": 0.9565152864351439, - "path": "" - }, - { - "seed": 42, - "value": 0.9562827743148313, - "path": "" - }, - { - "seed": 99, - "value": 0.0007742158899772435, - "path": "" - }, - { - "seed": 123, - "value": 0.9564299495399228, - "path": "" - } - ] - }, - "thresholds": [ - 0.00024276715703308582, - 0.0002697413437999785, - 0.00027438640245236456, - 0.0002780304348561913, - 0.000285720539977774 - ], - "paths": [ - "", - "", - "", - "", - "" - ] - }, - { - "model_tag": "pangu_weather", - "label": "Pangu-Weather", - "scope": "train_fire_top20pct", - "scope_label": "top 20%", - "n": 5, - "strict_f1": { - "n": 5, - "mean": 0.006615089943495571, - "std": 0.011136980463397231, - "min": 0.0014589084597825474, - "max": 0.026530371070668874 - }, - "tolerance_f1": { - "n": 5, - "mean": 0.09076165157666524, - "std": 0.08204539789375309, - "min": 0.031681254572138745, - "max": 0.23561614605975847 - }, - "union_f1": { - "n": 5, - "mean": 0.21219018441019574, - "std": 0.1438368307990196, - "min": 0.09062518509001295, - "max": 0.4621544722787787 - }, - "difference": { - "n": 5, - "mean": 0.20557509446670016, - "std": 0.13312438976032662, - "min": 0.08847200309630879, - "max": 0.43562410120810985 - }, - "predicted_positive_rate": { - "n": 5, - "mean": 0.6610079158936302, - "std": 0.4509300379554766, - "min": 0.0202152133580705, - "max": 0.9827272727272728 - }, - "seed_values": { - "strict_f1": [ - { - "seed": 1, - "value": 0.026530371070668874, - "path": "" - }, - { - "seed": 7, - "value": 0.0014589084597825474, - "path": "" - }, - { - "seed": 42, - "value": 0.0014680041534393378, - "path": "" - }, - { - "seed": 99, - "value": 0.0021531819937041672, - "path": "" - }, - { - "seed": 123, - "value": 0.0014649840398829276, - "path": "" - } - ], - "tolerance_f1": [ - { - "seed": 1, - "value": 0.23561614605975847, - "path": "" - }, - { - "seed": 7, - "value": 0.06196203978961812, - "path": "" - }, - { - "seed": 42, - "value": 0.062336581960243484, - "path": "" - }, - { - "seed": 99, - "value": 0.031681254572138745, - "path": "" - }, - { - "seed": 123, - "value": 0.062212235501567396, - "path": "" - } - ], - "union_f1": [ - { - "seed": 1, - "value": 0.4621544722787787, - "path": "" - }, - { - "seed": 7, - "value": 0.1688543802092876, - "path": "" - }, - { - "seed": 42, - "value": 0.16981843825492582, - "path": "" - }, - { - "seed": 99, - "value": 0.09062518509001295, - "path": "" - }, - { - "seed": 123, - "value": 0.16949844621797366, - "path": "" - } - ], - "difference": [ - { - "seed": 1, - "value": 0.43562410120810985, - "path": "" - }, - { - "seed": 7, - "value": 0.16739547174950506, - "path": "" - }, - { - "seed": 42, - "value": 0.16835043410148648, - "path": "" - }, - { - "seed": 99, - "value": 0.08847200309630879, - "path": "" - }, - { - "seed": 123, - "value": 0.16803346217809073, - "path": "" - } - ], - "predicted_positive_rate": [ - { - "seed": 1, - "value": 0.0202152133580705, - "path": "" - }, - { - "seed": 7, - "value": 0.9827272727272728, - "path": "" - }, - { - "seed": 42, - "value": 0.9766338899196042, - "path": "" - }, - { - "seed": 99, - "value": 0.34681447124304265, - "path": "" - }, - { - "seed": 123, - "value": 0.9786487322201608, - "path": "" - } - ] - }, - "thresholds": [ - 0.0002, - 0.0002300136402482167, - 0.00023887513088993728, - 0.00027438640245236456, - 0.00028135377215221524 - ], - "paths": [ - "", - "", - "", - "", - "" - ] - }, - { - "model_tag": "pangu_weather_h24", - "label": "Pangu-Weather", - "scope": "full_domain", - "scope_label": "global", - "n": 5, - "strict_f1": { - "n": 5, - "mean": 0.002839244726016104, - "std": 0.0014055201553382893, - "min": 0.0015570260801868433, - "max": 0.00458918723078781 - }, - "tolerance_f1": { - "n": 5, - "mean": 0.11344735239772843, - "std": 0.05215606001713916, - "min": 0.04422040571422221, - "max": 0.17658327178527805 - }, - "union_f1": { - "n": 5, - "mean": 0.26579704586756064, - "std": 0.12358583427828769, - "min": 0.1360311898534274, - "max": 0.4448076750202692 - }, - "difference": { - "n": 5, - "mean": 0.26295780114154454, - "std": 0.12366414839800409, - "min": 0.13447416377324056, - "max": 0.44278198559528875 - }, - "predicted_positive_rate": { - "n": 5, - "mean": 0.004966580086580087, - "std": 0.006165207367586338, - "min": 0.0013461966604823748, - "max": 0.015876932591218307 - }, - "seed_values": { - "strict_f1": [ - { - "seed": 7, - "value": 0.00412434125105018, - "path": "" - }, - { - "seed": 17, - "value": 0.0018999796430752525, - "path": "" - }, - { - "seed": 37, - "value": 0.002025689424980434, - "path": "" - }, - { - "seed": 57, - "value": 0.00458918723078781, - "path": "" - }, - { - "seed": 97, - "value": 0.0015570260801868433, - "path": "" - } - ], - "tolerance_f1": [ - { - "seed": 7, - "value": 0.08503531719948608, - "path": "" - }, - { - "seed": 17, - "value": 0.11199020227536667, - "path": "" - }, - { - "seed": 37, - "value": 0.17658327178527805, - "path": "" - }, - { - "seed": 57, - "value": 0.14940756501428915, - "path": "" - }, - { - "seed": 97, - "value": 0.04422040571422221, - "path": "" - } - ], - "union_f1": [ - { - "seed": 7, - "value": 0.1652119748219299, - "path": "" - }, - { - "seed": 17, - "value": 0.27178749222596443, - "path": "" - }, - { - "seed": 37, - "value": 0.4448076750202692, - "path": "" - }, - { - "seed": 57, - "value": 0.31114689741621243, - "path": "" - }, - { - "seed": 97, - "value": 0.1360311898534274, - "path": "" - } - ], - "difference": [ - { - "seed": 7, - "value": 0.1610876335708797, - "path": "" - }, - { - "seed": 17, - "value": 0.2698875125828892, - "path": "" - }, - { - "seed": 37, - "value": 0.44278198559528875, - "path": "" - }, - { - "seed": 57, - "value": 0.3065577101854246, - "path": "" - }, - { - "seed": 97, - "value": 0.13447416377324056, - "path": "" - } - ], - "predicted_positive_rate": [ - { - "seed": 7, - "value": 0.0013768707482993197, - "path": "" - }, - { - "seed": 17, - "value": 0.0034029684601113172, - "path": "" - }, - { - "seed": 37, - "value": 0.015876932591218307, - "path": "" - }, - { - "seed": 57, - "value": 0.0028299319727891157, - "path": "" - }, - { - "seed": 97, - "value": 0.0013461966604823748, - "path": "" - } - ] - }, - "thresholds": [ - 0.1, - 0.3, - 0.5 - ], - "paths": [ - "", - "", - "", - "", - "" - ] - }, - { - "model_tag": "pangu_weather_h24", - "label": "Pangu-Weather", - "scope": "train_fire_top05pct", - "scope_label": "top 5%", - "n": 5, - "strict_f1": { - "n": 5, - "mean": 0.010727828314232248, - "std": 0.0030802455090206595, - "min": 0.006323656223052602, - "max": 0.014334527877323108 - }, - "tolerance_f1": { - "n": 5, - "mean": 0.16440632872776462, - "std": 0.04420834148717055, - "min": 0.1236361475439318, - "max": 0.23786880092608761 - }, - "union_f1": { - "n": 5, - "mean": 0.36002559619997526, - "std": 0.11617972461830063, - "min": 0.26905489411020966, - "max": 0.5563138131518957 - }, - "difference": { - "n": 5, - "mean": 0.34929776788574307, - "std": 0.1137924578244252, - "min": 0.2627312378871571, - "max": 0.5419792852745725 - }, - "predicted_positive_rate": { - "n": 5, - "mean": 0.014698228950232511, - "std": 0.008898365132157035, - "min": 0.00599584446423271, - "max": 0.02707034728406055 - }, - "seed_values": { - "strict_f1": [ - { - "seed": 7, - "value": 0.00961323496534764, - "path": "" - }, - { - "seed": 17, - "value": 0.006323656223052602, - "path": "" - }, - { - "seed": 37, - "value": 0.014334527877323108, - "path": "" - }, - { - "seed": 57, - "value": 0.012796587576646227, - "path": "" - }, - { - "seed": 97, - "value": 0.010571134928791661, - "path": "" - } - ], - "tolerance_f1": [ - { - "seed": 7, - "value": 0.13752896018113672, - "path": "" - }, - { - "seed": 17, - "value": 0.1236361475439318, - "path": "" - }, - { - "seed": 37, - "value": 0.23786880092608761, - "path": "" - }, - { - "seed": 57, - "value": 0.15839502058327895, - "path": "" - }, - { - "seed": 97, - "value": 0.16460271440438795, - "path": "" - } - ], - "union_f1": [ - { - "seed": 7, - "value": 0.2838373343689194, - "path": "" - }, - { - "seed": 17, - "value": 0.26905489411020966, - "path": "" - }, - { - "seed": 37, - "value": 0.5563138131518957, - "path": "" - }, - { - "seed": 57, - "value": 0.3235832074044849, - "path": "" - }, - { - "seed": 97, - "value": 0.3673387319643668, - "path": "" - } - ], - "difference": [ - { - "seed": 7, - "value": 0.27422409940357173, - "path": "" - }, - { - "seed": 17, - "value": 0.2627312378871571, - "path": "" - }, - { - "seed": 37, - "value": 0.5419792852745725, - "path": "" - }, - { - "seed": 57, - "value": 0.3107866198278387, - "path": "" - }, - { - "seed": 97, - "value": 0.35676759703557515, - "path": "" - } - ], - "predicted_positive_rate": [ - { - "seed": 7, - "value": 0.019518650440288908, - "path": "" - }, - { - "seed": 17, - "value": 0.00599584446423271, - "path": "" - }, - { - "seed": 37, - "value": 0.02707034728406055, - "path": "" - }, - { - "seed": 57, - "value": 0.006668645493222519, - "path": "" - }, - { - "seed": 97, - "value": 0.014237657069357871, - "path": "" - } - ] - }, - "thresholds": [ - 0.1, - 0.3 - ], - "paths": [ - "", - "", - "", - "", - "" - ] - }, - { - "model_tag": "pangu_weather_h24", - "label": "Pangu-Weather", - "scope": "train_fire_top10pct", - "scope_label": "top 10%", - "n": 5, - "strict_f1": { - "n": 5, - "mean": 0.006807753305469981, - "std": 0.0036980018094160414, - "min": 0.002669336002669336, - "max": 0.010952623535404991 - }, - "tolerance_f1": { - "n": 5, - "mean": 0.10945088912540306, - "std": 0.06122864367998089, - "min": 0.03347640061525648, - "max": 0.19299696406724362 - }, - "union_f1": { - "n": 5, - "mean": 0.2511409611173737, - "std": 0.14418045901223492, - "min": 0.12011082544991639, - "max": 0.478233072358786 - }, - "difference": { - "n": 5, - "mean": 0.24433320781190368, - "std": 0.14096512937898192, - "min": 0.11744148944724705, - "max": 0.467280448823381 - }, - "predicted_positive_rate": { - "n": 5, - "mean": 0.006657019887206886, - "std": 0.006467521103800908, - "min": 0.002389433066191748, - "max": 0.018105026219451866 - }, - "seed_values": { - "strict_f1": [ - { - "seed": 7, - "value": 0.005579507988840983, - "path": "" - }, - { - "seed": 17, - "value": 0.004407051282051282, - "path": "" - }, - { - "seed": 37, - "value": 0.010952623535404991, - "path": "" - }, - { - "seed": 57, - "value": 0.010430247718383311, - "path": "" - }, - { - "seed": 97, - "value": 0.002669336002669336, - "path": "" - } - ], - "tolerance_f1": [ - { - "seed": 7, - "value": 0.07891517659613233, - "path": "" - }, - { - "seed": 17, - "value": 0.09804613935969869, - "path": "" - }, - { - "seed": 37, - "value": 0.19299696406724362, - "path": "" - }, - { - "seed": 57, - "value": 0.14381976498868418, - "path": "" - }, - { - "seed": 97, - "value": 0.03347640061525648, - "path": "" - } - ], - "union_f1": [ - { - "seed": 7, - "value": 0.14561269032771065, - "path": "" - }, - { - "seed": 17, - "value": 0.2156791196445671, - "path": "" - }, - { - "seed": 37, - "value": 0.478233072358786, - "path": "" - }, - { - "seed": 57, - "value": 0.2960690978058883, - "path": "" - }, - { - "seed": 97, - "value": 0.12011082544991639, - "path": "" - } - ], - "difference": [ - { - "seed": 7, - "value": 0.14003318233886966, - "path": "" - }, - { - "seed": 17, - "value": 0.2112720683625158, - "path": "" - }, - { - "seed": 37, - "value": 0.467280448823381, - "path": "" - }, - { - "seed": 57, - "value": 0.285638850087505, - "path": "" - }, - { - "seed": 97, - "value": 0.11744148944724705, - "path": "" - } - ], - "predicted_positive_rate": [ - { - "seed": 7, - "value": 0.0035594142673394677, - "path": "" - }, - { - "seed": 17, - "value": 0.004856782428020184, - "path": "" - }, - { - "seed": 37, - "value": 0.018105026219451866, - "path": "" - }, - { - "seed": 57, - "value": 0.004374443455031167, - "path": "" - }, - { - "seed": 97, - "value": 0.002389433066191748, - "path": "" - } - ] - }, - "thresholds": [ - 0.1, - 0.3, - 0.5 - ], - "paths": [ - "", - "", - "", - "", - "" - ] - }, - { - "model_tag": "pangu_weather_h24", - "label": "Pangu-Weather", - "scope": "train_fire_top20pct", - "scope_label": "top 20%", - "n": 5, - "strict_f1": { - "n": 5, - "mean": 0.004657283091657516, - "std": 0.002873310351989516, - "min": 0.002313475997686524, - "max": 0.0094525403702245 - }, - "tolerance_f1": { - "n": 5, - "mean": 0.08713410482710718, - "std": 0.03539578269908524, - "min": 0.036969834429575875, - "max": 0.1330139941503748 - }, - "union_f1": { - "n": 5, - "mean": 0.209055617580378, - "std": 0.08553820555850469, - "min": 0.12157774813170726, - "max": 0.3168721772329861 - }, - "difference": { - "n": 5, - "mean": 0.2043983344887205, - "std": 0.08451922309197234, - "min": 0.11926427213402073, - "max": 0.31356600176953114 - }, - "predicted_positive_rate": { - "n": 5, - "mean": 0.0027991341991341996, - "std": 0.0012806432499773903, - "min": 0.0014211502782931354, - "max": 0.004520098948670378 - }, - "seed_values": { - "strict_f1": [ - { - "seed": 7, - "value": 0.0051329911339244045, - "path": "" - }, - { - "seed": 17, - "value": 0.003081232492997199, - "path": "" - }, - { - "seed": 37, - "value": 0.003306175463454953, - "path": "" - }, - { - "seed": 57, - "value": 0.0094525403702245, - "path": "" - }, - { - "seed": 97, - "value": 0.002313475997686524, - "path": "" - } - ], - "tolerance_f1": [ - { - "seed": 7, - "value": 0.07335569213391166, - "path": "" - }, - { - "seed": 17, - "value": 0.0918614020950846, - "path": "" - }, - { - "seed": 37, - "value": 0.10046960132658891, - "path": "" - }, - { - "seed": 57, - "value": 0.1330139941503748, - "path": "" - }, - { - "seed": 97, - "value": 0.036969834429575875, - "path": "" - } - ], - "union_f1": [ - { - "seed": 7, - "value": 0.13544861662147875, - "path": "" - }, - { - "seed": 17, - "value": 0.19598081171127166, - "path": "" - }, - { - "seed": 37, - "value": 0.3168721772329861, - "path": "" - }, - { - "seed": 57, - "value": 0.2753987342044462, - "path": "" - }, - { - "seed": 97, - "value": 0.12157774813170726, - "path": "" - } - ], - "difference": [ - { - "seed": 7, - "value": 0.13031562548755435, - "path": "" - }, - { - "seed": 17, - "value": 0.19289957921827447, - "path": "" - }, - { - "seed": 37, - "value": 0.31356600176953114, - "path": "" - }, - { - "seed": 57, - "value": 0.2659461938342217, - "path": "" - }, - { - "seed": 97, - "value": 0.11926427213402073, - "path": "" - } - ], - "predicted_positive_rate": [ - { - "seed": 7, - "value": 0.0019332096474953618, - "path": "" - }, - { - "seed": 17, - "value": 0.0036982065553494126, - "path": "" - }, - { - "seed": 37, - "value": 0.004520098948670378, - "path": "" - }, - { - "seed": 57, - "value": 0.002423005565862709, - "path": "" - }, - { - "seed": 97, - "value": 0.0014211502782931354, - "path": "" - } - ] - }, - "thresholds": [ - 0.15, - 0.3, - 0.5 - ], - "paths": [ - "", - "", - "", - "", - "" - ] - }, - { - "model_tag": "pangu6", - "label": "Pangu-Weather", - "scope": "full_domain", - "scope_label": "global", - "n": 5, - "strict_f1": { - "n": 5, - "mean": 0.0027554617131344606, - "std": 0.0010888849544285033, - "min": 0.0014740203906154037, - "max": 0.003978746443051044 - }, - "tolerance_f1": { - "n": 5, - "mean": 0.17090859723170176, - "std": 0.04047686720194535, - "min": 0.12218430629860097, - "max": 0.2264875300093167 - }, - "union_f1": { - "n": 5, - "mean": 0.35638584510932614, - "std": 0.09032661075096597, - "min": 0.22268222965661022, - "max": 0.46795172614222125 - }, - "difference": { - "n": 5, - "mean": 0.3536303833961917, - "std": 0.0907743328516622, - "min": 0.21893843540252053, - "max": 0.4653493031095141 - }, - "predicted_positive_rate": { - "n": 5, - "mean": 0.010800964749536178, - "std": 0.011307793167633017, - "min": 0.0012771799628942485, - "max": 0.029129251700680273 - }, - "seed_values": { - "strict_f1": [ - { - "seed": 7, - "value": 0.0037437942540896885, - "path": "" - }, - { - "seed": 17, - "value": 0.0019783244452090142, - "path": "" - }, - { - "seed": 37, - "value": 0.0014740203906154037, - "path": "" - }, - { - "seed": 57, - "value": 0.003978746443051044, - "path": "" - }, - { - "seed": 97, - "value": 0.0026024230327071516, - "path": "" - } - ], - "tolerance_f1": [ - { - "seed": 7, - "value": 0.12218430629860097, - "path": "" - }, - { - "seed": 17, - "value": 0.1809729686535962, - "path": "" - }, - { - "seed": 37, - "value": 0.14188194761227685, - "path": "" - }, - { - "seed": 57, - "value": 0.18301623358471802, - "path": "" - }, - { - "seed": 97, - "value": 0.2264875300093167, - "path": "" - } - ], - "union_f1": [ - { - "seed": 7, - "value": 0.22268222965661022, - "path": "" - }, - { - "seed": 17, - "value": 0.40012531103066123, - "path": "" - }, - { - "seed": 37, - "value": 0.3364875741725597, - "path": "" - }, - { - "seed": 57, - "value": 0.3546823845445782, - "path": "" - }, - { - "seed": 97, - "value": 0.46795172614222125, - "path": "" - } - ], - "difference": [ - { - "seed": 7, - "value": 0.21893843540252053, - "path": "" - }, - { - "seed": 17, - "value": 0.3981469865854522, - "path": "" - }, - { - "seed": 37, - "value": 0.3350135537819443, - "path": "" - }, - { - "seed": 57, - "value": 0.35070363810152716, - "path": "" - }, - { - "seed": 97, - "value": 0.4653493031095141, - "path": "" - } - ], - "predicted_positive_rate": [ - { - "seed": 7, - "value": 0.0012771799628942485, - "path": "" - }, - { - "seed": 17, - "value": 0.01413716759431045, - "path": "" - }, - { - "seed": 37, - "value": 0.00479208410636982, - "path": "" - }, - { - "seed": 57, - "value": 0.004669140383426098, - "path": "" - }, - { - "seed": 97, - "value": 0.029129251700680273, - "path": "" - } - ] - }, - "thresholds": [ - 0.15, - 0.2, - 0.3 - ], - "paths": [ - "", - "", - "", - "", - "" - ] - }, - { - "model_tag": "pangu6", - "label": "Pangu-Weather", - "scope": "train_fire_top05pct", - "scope_label": "top 5%", - "n": 5, - "strict_f1": { - "n": 5, - "mean": 0.013655691986016547, - "std": 0.003064288092894302, - "min": 0.010515247108307044, - "max": 0.017845192951148784 - }, - "tolerance_f1": { - "n": 5, - "mean": 0.2222219561322302, - "std": 0.068612974451766, - "min": 0.12664654352782623, - "max": 0.3098064830268362 - }, - "union_f1": { - "n": 5, - "mean": 0.4342335956184513, - "std": 0.13238331958267657, - "min": 0.22723717713750308, - "max": 0.5952835470121373 - }, - "difference": { - "n": 5, - "mean": 0.42057790363243475, - "std": 0.13059925591392604, - "min": 0.21672193002919604, - "max": 0.5794332814358789 - }, - "predicted_positive_rate": { - "n": 5, - "mean": 0.02054862966261007, - "std": 0.02189321852493811, - "min": 0.004447412684278223, - "max": 0.056067576926882357 - }, - "seed_values": { - "strict_f1": [ - { - "seed": 7, - "value": 0.010515247108307044, - "path": "" - }, - { - "seed": 17, - "value": 0.011848926191063935, - "path": "" - }, - { - "seed": 37, - "value": 0.012218828103304637, - "path": "" - }, - { - "seed": 57, - "value": 0.017845192951148784, - "path": "" - }, - { - "seed": 97, - "value": 0.015850265576258328, - "path": "" - } - ], - "tolerance_f1": [ - { - "seed": 7, - "value": 0.12664654352782623, - "path": "" - }, - { - "seed": 17, - "value": 0.23514322359982898, - "path": "" - }, - { - "seed": 37, - "value": 0.1893747086942956, - "path": "" - }, - { - "seed": 57, - "value": 0.2501388218123639, - "path": "" - }, - { - "seed": 97, - "value": 0.3098064830268362, - "path": "" - } - ], - "union_f1": [ - { - "seed": 7, - "value": 0.22723717713750308, - "path": "" - }, - { - "seed": 17, - "value": 0.46975335070929664, - "path": "" - }, - { - "seed": 37, - "value": 0.4388313446697724, - "path": "" - }, - { - "seed": 57, - "value": 0.44006255856354737, - "path": "" - }, - { - "seed": 97, - "value": 0.5952835470121373, - "path": "" - } - ], - "difference": [ - { - "seed": 7, - "value": 0.21672193002919604, - "path": "" - }, - { - "seed": 17, - "value": 0.4579044245182327, - "path": "" - }, - { - "seed": 37, - "value": 0.42661251656646776, - "path": "" - }, - { - "seed": 57, - "value": 0.4222173656123986, - "path": "" - }, - { - "seed": 97, - "value": 0.5794332814358789, - "path": "" - } - ], - "predicted_positive_rate": [ - { - "seed": 7, - "value": 0.004447412684278223, - "path": "" - }, - { - "seed": 17, - "value": 0.027451271396062134, - "path": "" - }, - { - "seed": 37, - "value": 0.006297615514000198, - "path": "" - }, - { - "seed": 57, - "value": 0.008479271791827446, - "path": "" - }, - { - "seed": 97, - "value": 0.056067576926882357, - "path": "" - } - ] - }, - "thresholds": [ - 0.15, - 0.2, - 0.3 - ], - "paths": [ - "", - "", - "", - "", - "" - ] - }, - { - "model_tag": "pangu6", - "label": "Pangu-Weather", - "scope": "train_fire_top10pct", - "scope_label": "top 10%", - "n": 5, - "strict_f1": { - "n": 5, - "mean": 0.010931248966985132, - "std": 0.003535277885116421, - "min": 0.00788436268068331, - "max": 0.016403526758253024 - }, - "tolerance_f1": { - "n": 5, - "mean": 0.18933650687857217, - "std": 0.0593294088865109, - "min": 0.1079162350962175, - "max": 0.25501453744070673 - }, - "union_f1": { - "n": 5, - "mean": 0.3853250982462472, - "std": 0.1172207993601839, - "min": 0.19681189159430446, - "max": 0.5172553944576963 - }, - "difference": { - "n": 5, - "mean": 0.3743938492792621, - "std": 0.11526082075130117, - "min": 0.18892752891362113, - "max": 0.5050483949122122 - }, - "predicted_positive_rate": { - "n": 5, - "mean": 0.014029385574354408, - "std": 0.014738630997131075, - "min": 0.0033887404768972, - "max": 0.03677772830711388 - }, - "seed_values": { - "strict_f1": [ - { - "seed": 7, - "value": 0.00788436268068331, - "path": "" - }, - { - "seed": 17, - "value": 0.008011413520632135, - "path": "" - }, - { - "seed": 37, - "value": 0.010149942329873126, - "path": "" - }, - { - "seed": 57, - "value": 0.016403526758253024, - "path": "" - }, - { - "seed": 97, - "value": 0.012206999545484059, - "path": "" - } - ], - "tolerance_f1": [ - { - "seed": 7, - "value": 0.1079162350962175, - "path": "" - }, - { - "seed": 17, - "value": 0.18336249012390426, - "path": "" - }, - { - "seed": 37, - "value": 0.1623425297604306, - "path": "" - }, - { - "seed": 57, - "value": 0.23804674197160172, - "path": "" - }, - { - "seed": 97, - "value": 0.25501453744070673, - "path": "" - } - ], - "union_f1": [ - { - "seed": 7, - "value": 0.19681189159430446, - "path": "" - }, - { - "seed": 17, - "value": 0.3796403112945744, - "path": "" - }, - { - "seed": 37, - "value": 0.41150510005450436, - "path": "" - }, - { - "seed": 57, - "value": 0.4214127938301568, - "path": "" - }, - { - "seed": 97, - "value": 0.5172553944576963, - "path": "" - } - ], - "difference": [ - { - "seed": 7, - "value": 0.18892752891362113, - "path": "" - }, - { - "seed": 17, - "value": 0.37162889777394226, - "path": "" - }, - { - "seed": 37, - "value": 0.4013551577246312, - "path": "" - }, - { - "seed": 57, - "value": 0.40500926707190377, - "path": "" - }, - { - "seed": 97, - "value": 0.5050483949122122, - "path": "" - } - ], - "predicted_positive_rate": [ - { - "seed": 7, - "value": 0.0033887404768972, - "path": "" - }, - { - "seed": 17, - "value": 0.021221678044919364, - "path": "" - }, - { - "seed": 37, - "value": 0.004044226773523301, - "path": "" - }, - { - "seed": 57, - "value": 0.004714554269318294, - "path": "" - }, - { - "seed": 97, - "value": 0.03677772830711388, - "path": "" - } - ] - }, - "thresholds": [ - 0.15, - 0.2, - 0.3 - ], - "paths": [ - "", - "", - "", - "", - "" - ] - }, - { - "model_tag": "pangu6", - "label": "Pangu-Weather", - "scope": "train_fire_top20pct", - "scope_label": "top 20%", - "n": 5, - "strict_f1": { - "n": 5, - "mean": 0.008844166237577134, - "std": 0.0036010444504776733, - "min": 0.005117388748660143, - "max": 0.01471129091577786 - }, - "tolerance_f1": { - "n": 5, - "mean": 0.17017232303990823, - "std": 0.05485942193604147, - "min": 0.09937369519832985, - "max": 0.226729727407505 - }, - "union_f1": { - "n": 5, - "mean": 0.3456881777429305, - "std": 0.10293157698741554, - "min": 0.18187865016757276, - "max": 0.45415946247619876 - }, - "difference": { - "n": 5, - "mean": 0.3368440115053533, - "std": 0.10133432810494146, - "min": 0.17487748330642924, - "max": 0.44522188799641144 - }, - "predicted_positive_rate": { - "n": 5, - "mean": 0.010103277674706247, - "std": 0.011083627907574344, - "min": 0.001932591218305504, - "max": 0.026268398268398267 - }, - "seed_values": { - "strict_f1": [ - { - "seed": 7, - "value": 0.0070011668611435224, - "path": "" - }, - { - "seed": 17, - "value": 0.005117388748660143, - "path": "" - }, - { - "seed": 37, - "value": 0.008453410182516812, - "path": "" - }, - { - "seed": 57, - "value": 0.01471129091577786, - "path": "" - }, - { - "seed": 97, - "value": 0.008937574479787332, - "path": "" - } - ], - "tolerance_f1": [ - { - "seed": 7, - "value": 0.09937369519832985, - "path": "" - }, - { - "seed": 17, - "value": 0.14938342628742876, - "path": "" - }, - { - "seed": 37, - "value": 0.15039592221466835, - "path": "" - }, - { - "seed": 57, - "value": 0.2249788440916092, - "path": "" - }, - { - "seed": 97, - "value": 0.226729727407505, - "path": "" - } - ], - "union_f1": [ - { - "seed": 7, - "value": 0.18187865016757276, - "path": "" - }, - { - "seed": 17, - "value": 0.323212329597846, - "path": "" - }, - { - "seed": 37, - "value": 0.3734736878120269, - "path": "" - }, - { - "seed": 57, - "value": 0.3957167586610078, - "path": "" - }, - { - "seed": 97, - "value": 0.45415946247619876, - "path": "" - } - ], - "difference": [ - { - "seed": 7, - "value": 0.17487748330642924, - "path": "" - }, - { - "seed": 17, - "value": 0.3180949408491859, - "path": "" - }, - { - "seed": 37, - "value": 0.36502027762951006, - "path": "" - }, - { - "seed": 57, - "value": 0.3810054677452299, - "path": "" - }, - { - "seed": 97, - "value": 0.44522188799641144, - "path": "" - } - ], - "predicted_positive_rate": [ - { - "seed": 7, - "value": 0.001932591218305504, - "path": "" - }, - { - "seed": 17, - "value": 0.017168212739641312, - "path": "" - }, - { - "seed": 37, - "value": 0.0025015460729746444, - "path": "" - }, - { - "seed": 57, - "value": 0.0026456400742115027, - "path": "" - }, - { - "seed": 97, - "value": 0.026268398268398267, - "path": "" - } - ] - }, - "thresholds": [ - 0.15, - 0.2, - 0.3 - ], - "paths": [ - "", - "", - "", - "", - "" - ] - }, - { - "model_tag": "prithvi_wxc", - "label": "Prithvi-WxC", - "scope": "full_domain", - "scope_label": "global", - "n": 5, - "strict_f1": { - "n": 5, - "mean": 0.0005524855672037344, - "std": 3.864412600940357e-05, - "min": 0.0005057825179019543, - "max": 0.0006040339888888368 - }, - "tolerance_f1": { - "n": 5, - "mean": 0.07164938383885015, - "std": 0.006556565946161047, - "min": 0.06182533987250242, - "max": 0.07892160427759505 - }, - "union_f1": { - "n": 5, - "mean": 0.20185302299729843, - "std": 0.018299428665013905, - "min": 0.17269452770359542, - "max": 0.22218157532367563 - }, - "difference": { - "n": 5, - "mean": 0.20130053743009468, - "std": 0.018296989279232255, - "min": 0.17211673370575886, - "max": 0.2215775413347868 - }, - "predicted_positive_rate": { - "n": 5, - "mean": 0.6197105751391466, - "std": 0.30910050227102126, - "min": 0.22581076066790354, - "max": 0.9588571428571429 - }, - "seed_values": { - "strict_f1": [ - { - "seed": 1, - "value": 0.0006040339888888368, - "path": "" - }, - { - "seed": 7, - "value": 0.000532941187181528, - "path": "" - }, - { - "seed": 42, - "value": 0.0005777939978365554, - "path": "" - }, - { - "seed": 99, - "value": 0.0005418761442097979, - "path": "" - }, - { - "seed": 123, - "value": 0.0005057825179019543, - "path": "" - } - ], - "tolerance_f1": [ - { - "seed": 1, - "value": 0.07892160427759505, - "path": "" - }, - { - "seed": 7, - "value": 0.07586161173658511, - "path": "" - }, - { - "seed": 42, - "value": 0.06182533987250242, - "path": "" - }, - { - "seed": 99, - "value": 0.06950848282026648, - "path": "" - }, - { - "seed": 123, - "value": 0.07212988048730175, - "path": "" - } - ], - "union_f1": [ - { - "seed": 1, - "value": 0.22218157532367563, - "path": "" - }, - { - "seed": 7, - "value": 0.21037180175292197, - "path": "" - }, - { - "seed": 42, - "value": 0.17269452770359542, - "path": "" - }, - { - "seed": 99, - "value": 0.2033106164921145, - "path": "" - }, - { - "seed": 123, - "value": 0.20070659371418467, - "path": "" - } - ], - "difference": [ - { - "seed": 1, - "value": 0.2215775413347868, - "path": "" - }, - { - "seed": 7, - "value": 0.20983886056574044, - "path": "" - }, - { - "seed": 42, - "value": 0.17211673370575886, - "path": "" - }, - { - "seed": 99, - "value": 0.20276874034790468, - "path": "" - }, - { - "seed": 123, - "value": 0.20020081119628272, - "path": "" - } - ], - "predicted_positive_rate": [ - { - "seed": 1, - "value": 0.5006159554730983, - "path": "" - }, - { - "seed": 7, - "value": 0.9099814471243043, - "path": "" - }, - { - "seed": 42, - "value": 0.22581076066790354, - "path": "" - }, - { - "seed": 99, - "value": 0.5032875695732839, - "path": "" - }, - { - "seed": 123, - "value": 0.9588571428571429, - "path": "" - } - ] - }, - "thresholds": [ - 0.0002606518683023751, - 0.00028123805532231927, - 0.00028278783429414034, - 0.00028816089616157115, - 0.00028831177041865885 - ], - "paths": [ - "", - "", - "", - "", - "" - ] - }, - { - "model_tag": "prithvi_wxc", - "label": "Prithvi-WxC", - "scope": "train_fire_top05pct", - "scope_label": "top 5%", - "n": 5, - "strict_f1": { - "n": 5, - "mean": 0.01411939737783544, - "std": 0.011635188932059573, - "min": 0.005636060100166944, - "max": 0.027781926811053025 - }, - "tolerance_f1": { - "n": 5, - "mean": 0.19263594340017406, - "std": 0.04501855438328491, - "min": 0.15977956000729543, - "max": 0.24416957191197788 - }, - "union_f1": { - "n": 5, - "mean": 0.4257931093416185, - "std": 0.04549524292260173, - "min": 0.39407737525514647, - "max": 0.4925384357395131 - }, - "difference": { - "n": 5, - "mean": 0.4116737119637831, - "std": 0.034845536781985785, - "min": 0.38844131515497954, - "max": 0.46663155596188977 - }, - "predicted_positive_rate": { - "n": 5, - "mean": 0.5741169486494508, - "std": 0.47898702418670663, - "min": 0.013950727218759276, - "max": 0.9234194122885129 - }, - "seed_values": { - "strict_f1": [ - { - "seed": 1, - "value": 0.005636060100166944, - "path": "" - }, - { - "seed": 7, - "value": 0.005636060100166944, - "path": "" - }, - { - "seed": 42, - "value": 0.02590687977762335, - "path": "" - }, - { - "seed": 99, - "value": 0.027781926811053025, - "path": "" - }, - { - "seed": 123, - "value": 0.005636060100166944, - "path": "" - } - ], - "tolerance_f1": [ - { - "seed": 1, - "value": 0.15977956000729543, - "path": "" - }, - { - "seed": 7, - "value": 0.15977956000729543, - "path": "" - }, - { - "seed": 42, - "value": 0.23967146506700618, - "path": "" - }, - { - "seed": 99, - "value": 0.24416957191197788, - "path": "" - }, - { - "seed": 123, - "value": 0.15977956000729543, - "path": "" - } - ], - "union_f1": [ - { - "seed": 1, - "value": 0.39407737525514647, - "path": "" - }, - { - "seed": 7, - "value": 0.39407737525514647, - "path": "" - }, - { - "seed": 42, - "value": 0.4925384357395131, - "path": "" - }, - { - "seed": 99, - "value": 0.4541949852031399, - "path": "" - }, - { - "seed": 123, - "value": 0.39407737525514647, - "path": "" - } - ], - "difference": [ - { - "seed": 1, - "value": 0.38844131515497954, - "path": "" - }, - { - "seed": 7, - "value": 0.38844131515497954, - "path": "" - }, - { - "seed": 42, - "value": 0.46663155596188977, - "path": "" - }, - { - "seed": 99, - "value": 0.4264130583920869, - "path": "" - }, - { - "seed": 123, - "value": 0.38844131515497954, - "path": "" - } - ], - "predicted_positive_rate": [ - { - "seed": 1, - "value": 0.9234194122885129, - "path": "" - }, - { - "seed": 7, - "value": 0.9234194122885129, - "path": "" - }, - { - "seed": 42, - "value": 0.08637577916295637, - "path": "" - }, - { - "seed": 99, - "value": 0.013950727218759276, - "path": "" - }, - { - "seed": 123, - "value": 0.9234194122885129, - "path": "" - } - ] - }, - "thresholds": [ - 0.00018521470337873325, - 0.0002438416049699299, - 0.0002606456328066997, - 0.0002606526104500517, - 0.0002883114793803543 - ], - "paths": [ - "", - "", - "", - "", - "" - ] - }, - { - "model_tag": "prithvi_wxc", - "label": "Prithvi-WxC", - "scope": "train_fire_top10pct", - "scope_label": "top 10%", - "n": 5, - "strict_f1": { - "n": 5, - "mean": 0.012376418057865393, - "std": 0.013201257527312249, - "min": 0.002747305898968793, - "max": 0.02774049217002237 - }, - "tolerance_f1": { - "n": 5, - "mean": 0.14878019275284607, - "std": 0.08442915398703313, - "min": 0.08713204456539597, - "max": 0.24342288270264262 - }, - "union_f1": { - "n": 5, - "mean": 0.32691282462654847, - "std": 0.13208480223417152, - "min": 0.2309636203280157, - "max": 0.4900551948025372 - }, - "difference": { - "n": 5, - "mean": 0.31453640656868304, - "std": 0.11905339505419958, - "min": 0.2282163144290469, - "max": 0.46415551438013897 - }, - "predicted_positive_rate": { - "n": 5, - "mean": 0.5845651528643514, - "std": 0.5108966939133424, - "min": 0.006975363609379638, - "max": 0.9575541703769664 - }, - "seed_values": { - "strict_f1": [ - { - "seed": 1, - "value": 0.002747305898968793, - "path": "" - }, - { - "seed": 7, - "value": 0.002747305898968793, - "path": "" - }, - { - "seed": 42, - "value": 0.02589968042239822, - "path": "" - }, - { - "seed": 99, - "value": 0.02774049217002237, - "path": "" - }, - { - "seed": 123, - "value": 0.002747305898968793, - "path": "" - } - ], - "tolerance_f1": [ - { - "seed": 1, - "value": 0.08713204456539597, - "path": "" - }, - { - "seed": 7, - "value": 0.08713204456539597, - "path": "" - }, - { - "seed": 42, - "value": 0.23908194736539978, - "path": "" - }, - { - "seed": 99, - "value": 0.24342288270264262, - "path": "" - }, - { - "seed": 123, - "value": 0.08713204456539597, - "path": "" - } - ], - "union_f1": [ - { - "seed": 1, - "value": 0.2309636203280157, - "path": "" - }, - { - "seed": 7, - "value": 0.2309636203280157, - "path": "" - }, - { - "seed": 42, - "value": 0.4900551948025372, - "path": "" - }, - { - "seed": 99, - "value": 0.451618067346158, - "path": "" - }, - { - "seed": 123, - "value": 0.2309636203280157, - "path": "" - } - ], - "difference": [ - { - "seed": 1, - "value": 0.2282163144290469, - "path": "" - }, - { - "seed": 7, - "value": 0.2282163144290469, - "path": "" - }, - { - "seed": 42, - "value": 0.46415551438013897, - "path": "" - }, - { - "seed": 99, - "value": 0.4238775751761356, - "path": "" - }, - { - "seed": 123, - "value": 0.2282163144290469, - "path": "" - } - ], - "predicted_positive_rate": [ - { - "seed": 1, - "value": 0.9575541703769664, - "path": "" - }, - { - "seed": 7, - "value": 0.9575541703769664, - "path": "" - }, - { - "seed": 42, - "value": 0.043187889581478185, - "path": "" - }, - { - "seed": 99, - "value": 0.006975363609379638, - "path": "" - }, - { - "seed": 123, - "value": 0.9575541703769664, - "path": "" - } - ] - }, - "thresholds": [ - 0.00018521470337873325, - 0.0002438416049699299, - 0.0002606456328066997, - 0.0002606526104500517, - 0.0002883114793803543 - ], - "paths": [ - "", - "", - "", - "", - "" - ] - }, - { - "model_tag": "prithvi_wxc", - "label": "Prithvi-WxC", - "scope": "train_fire_top20pct", - "scope_label": "top 20%", - "n": 5, - "strict_f1": { - "n": 5, - "mean": 0.011519506840078677, - "std": 0.013769946337526496, - "min": 0.001471035812113219, - "max": 0.027352941176470587 - }, - "tolerance_f1": { - "n": 5, - "mean": 0.13151154173948754, - "std": 0.09455648302229831, - "min": 0.06246138809231819, - "max": 0.2365506772536198 - }, - "union_f1": { - "n": 5, - "mean": 0.2813191893577128, - "std": 0.15286646962866243, - "min": 0.17013954201238674, - "max": 0.4676561751411253 - }, - "difference": { - "n": 5, - "mean": 0.2697996825176341, - "std": 0.13922359360588243, - "min": 0.16866850620027352, - "max": 0.44182468955354215 - }, - "predicted_positive_rate": { - "n": 5, - "mean": 0.5897884972170686, - "std": 0.5269906679651803, - "min": 0.0034879406307977735, - "max": 0.9746196660482375 - }, - "seed_values": { - "strict_f1": [ - { - "seed": 1, - "value": 0.001471035812113219, - "path": "" - }, - { - "seed": 7, - "value": 0.001471035812113219, - "path": "" - }, - { - "seed": 42, - "value": 0.025831485587583147, - "path": "" - }, - { - "seed": 99, - "value": 0.027352941176470587, - "path": "" - }, - { - "seed": 123, - "value": 0.001471035812113219, - "path": "" - } - ], - "tolerance_f1": [ - { - "seed": 1, - "value": 0.06246138809231819, - "path": "" - }, - { - "seed": 7, - "value": 0.06246138809231819, - "path": "" - }, - { - "seed": 42, - "value": 0.23362286716686329, - "path": "" - }, - { - "seed": 99, - "value": 0.2365506772536198, - "path": "" - }, - { - "seed": 123, - "value": 0.06246138809231819, - "path": "" - } - ], - "union_f1": [ - { - "seed": 1, - "value": 0.17013954201238674, - "path": "" - }, - { - "seed": 7, - "value": 0.17013954201238674, - "path": "" - }, - { - "seed": 42, - "value": 0.4676561751411253, - "path": "" - }, - { - "seed": 99, - "value": 0.4285211456102784, - "path": "" - }, - { - "seed": 123, - "value": 0.17013954201238674, - "path": "" - } - ], - "difference": [ - { - "seed": 1, - "value": 0.16866850620027352, - "path": "" - }, - { - "seed": 7, - "value": 0.16866850620027352, - "path": "" - }, - { - "seed": 42, - "value": 0.44182468955354215, - "path": "" - }, - { - "seed": 99, - "value": 0.4011682044338078, - "path": "" - }, - { - "seed": 123, - "value": 0.16866850620027352, - "path": "" - } - ], - "predicted_positive_rate": [ - { - "seed": 1, - "value": 0.9746196660482375, - "path": "" - }, - { - "seed": 7, - "value": 0.9746196660482375, - "path": "" - }, - { - "seed": 42, - "value": 0.021595547309833023, - "path": "" - }, - { - "seed": 99, - "value": 0.0034879406307977735, - "path": "" - }, - { - "seed": 123, - "value": 0.9746196660482375, - "path": "" - } - ] - }, - "thresholds": [ - 0.0002, - 0.0002438417504890822, - 0.000260645778325852, - 0.0002606526104500517, - 0.0002883114793803543 - ], - "paths": [ - "", - "", - "", - "", - "" - ] - }, - { - "model_tag": "reference", - "label": "Reference", - "scope": "full_domain", - "scope_label": "global", - "n": 5, - "strict_f1": { - "n": 5, - "mean": 0.004546429694675032, - "std": 0.0014120999298872481, - "min": 0.0026426577586601383, - "max": 0.006568199246490847 - }, - "tolerance_f1": { - "n": 5, - "mean": 0.29748392511772237, - "std": 0.012868439357804603, - "min": 0.2756667293290642, - "max": 0.30820726968158313 - }, - "union_f1": { - "n": 5, - "mean": 0.5906557578801308, - "std": 0.02737158059714251, - "min": 0.5521558725617307, - "max": 0.6239373622157508 - }, - "difference": { - "n": 5, - "mean": 0.5861093281854558, - "std": 0.026945326610355286, - "min": 0.5480197595988799, - "max": 0.6194077454910121 - }, - "predicted_positive_rate": { - "n": 5, - "mean": 0.016807866419294992, - "std": 0.0036836114267075385, - "min": 0.012341125541125542, - "max": 0.021380705009276437 - }, - "seed_values": { - "strict_f1": [ - { - "seed": 1, - "value": 0.004136112962850736, - "path": "" - }, - { - "seed": 7, - "value": 0.006568199246490847, - "path": "" - }, - { - "seed": 42, - "value": 0.0045296167247386755, - "path": "" - }, - { - "seed": 99, - "value": 0.004855561780634761, - "path": "" - }, - { - "seed": 123, - "value": 0.0026426577586601383, - "path": "" - } - ], - "tolerance_f1": [ - { - "seed": 1, - "value": 0.2985122125101588, - "path": "" - }, - { - "seed": 7, - "value": 0.29937890424270136, - "path": "" - }, - { - "seed": 42, - "value": 0.3056545098251043, - "path": "" - }, - { - "seed": 99, - "value": 0.30820726968158313, - "path": "" - }, - { - "seed": 123, - "value": 0.2756667293290642, - "path": "" - } - ], - "union_f1": [ - { - "seed": 1, - "value": 0.5521558725617307, - "path": "" - }, - { - "seed": 7, - "value": 0.6079171724719479, - "path": "" - }, - { - "seed": 42, - "value": 0.6239373622157508, - "path": "" - }, - { - "seed": 99, - "value": 0.5803661485742072, - "path": "" - }, - { - "seed": 123, - "value": 0.5889022335770174, - "path": "" - } - ], - "difference": [ - { - "seed": 1, - "value": 0.5480197595988799, - "path": "" - }, - { - "seed": 7, - "value": 0.601348973225457, - "path": "" - }, - { - "seed": 42, - "value": 0.6194077454910121, - "path": "" - }, - { - "seed": 99, - "value": 0.5755105867935725, - "path": "" - }, - { - "seed": 123, - "value": 0.5862595758183573, - "path": "" - } - ], - "predicted_positive_rate": [ - { - "seed": 1, - "value": 0.016084972170686457, - "path": "" - }, - { - "seed": 7, - "value": 0.014596289424860853, - "path": "" - }, - { - "seed": 42, - "value": 0.019636239950525665, - "path": "" - }, - { - "seed": 99, - "value": 0.012341125541125542, - "path": "" - }, - { - "seed": 123, - "value": 0.021380705009276437, - "path": "" - } - ] - }, - "thresholds": [ - 0.95 - ], - "paths": [ - "", - "", - "", - "", - "" - ] - }, - { - "model_tag": "reference", - "label": "Reference", - "scope": "train_fire_top05pct", - "scope_label": "top 5%", - "n": 5, - "strict_f1": { - "n": 5, - "mean": 0.03560431762946002, - "std": 0.008809158358157916, - "min": 0.02462935239291061, - "max": 0.04873596787441818 - }, - "tolerance_f1": { - "n": 5, - "mean": 0.3926165265132609, - "std": 0.014010934164100928, - "min": 0.3702273155255325, - "max": 0.4063154181985604 - }, - "union_f1": { - "n": 5, - "mean": 0.7282799455103544, - "std": 0.025784171509335416, - "min": 0.7028883266246737, - "max": 0.7623590438272105 - }, - "difference": { - "n": 5, - "mean": 0.6926756278808944, - "std": 0.0199602553757671, - "min": 0.6696707231715443, - "max": 0.7136230759527924 - }, - "predicted_positive_rate": { - "n": 5, - "mean": 0.030619372711981795, - "std": 0.010925458415897562, - "min": 0.022801028989809045, - "max": 0.04860987434451371 - }, - "seed_values": { - "strict_f1": [ - { - "seed": 1, - "value": 0.03321760345312939, - "path": "" - }, - { - "seed": 7, - "value": 0.04873596787441818, - "path": "" - }, - { - "seed": 42, - "value": 0.038147512864494, - "path": "" - }, - { - "seed": 99, - "value": 0.0332911515623479, - "path": "" - }, - { - "seed": 123, - "value": 0.02462935239291061, - "path": "" - } - ], - "tolerance_f1": [ - { - "seed": 1, - "value": 0.39555976109165375, - "path": "" - }, - { - "seed": 7, - "value": 0.40145293653054454, - "path": "" - }, - { - "seed": 42, - "value": 0.4063154181985604, - "path": "" - }, - { - "seed": 99, - "value": 0.3895272012200133, - "path": "" - }, - { - "seed": 123, - "value": 0.3702273155255325, - "path": "" - } - ], - "union_f1": [ - { - "seed": 1, - "value": 0.7028883266246737, - "path": "" - }, - { - "seed": 7, - "value": 0.7623590438272105, - "path": "" - }, - { - "seed": 42, - "value": 0.7476654358764496, - "path": "" - }, - { - "seed": 99, - "value": 0.707788551758277, - "path": "" - }, - { - "seed": 123, - "value": 0.7206983694651612, - "path": "" - } - ], - "difference": [ - { - "seed": 1, - "value": 0.6696707231715443, - "path": "" - }, - { - "seed": 7, - "value": 0.7136230759527924, - "path": "" - }, - { - "seed": 42, - "value": 0.7095179230119556, - "path": "" - }, - { - "seed": 99, - "value": 0.6744974001959292, - "path": "" - }, - { - "seed": 123, - "value": 0.6960690170722506, - "path": "" - } - ], - "predicted_positive_rate": [ - { - "seed": 1, - "value": 0.023750865736618186, - "path": "" - }, - { - "seed": 7, - "value": 0.02449292569506283, - "path": "" - }, - { - "seed": 42, - "value": 0.033442168793905214, - "path": "" - }, - { - "seed": 99, - "value": 0.022801028989809045, - "path": "" - }, - { - "seed": 123, - "value": 0.04860987434451371, - "path": "" - } - ] - }, - "thresholds": [ - 0.9, - 0.95 - ], - "paths": [ - "", - "", - "", - "", - "" - ] - }, - { - "model_tag": "reference", - "label": "Reference", - "scope": "train_fire_top10pct", - "scope_label": "top 10%", - "n": 5, - "strict_f1": { - "n": 5, - "mean": 0.03557511803103047, - "std": 0.008798648622457047, - "min": 0.02461746391852102, - "max": 0.0486915291328531 - }, - "tolerance_f1": { - "n": 5, - "mean": 0.391664739051225, - "std": 0.013905680686091083, - "min": 0.369479820868587, - "max": 0.4052951105632299 - }, - "union_f1": { - "n": 5, - "mean": 0.7252037371000112, - "std": 0.025669721937652358, - "min": 0.699798626339235, - "max": 0.7590968995963846 - }, - "difference": { - "n": 5, - "mean": 0.6896286190689807, - "std": 0.019887948249560174, - "min": 0.6666121634162013, - "max": 0.7104053704635315 - }, - "predicted_positive_rate": { - "n": 5, - "mean": 0.015309686355990898, - "std": 0.005462729207948781, - "min": 0.011400514494904522, - "max": 0.024304937172256853 - }, - "seed_values": { - "strict_f1": [ - { - "seed": 1, - "value": 0.033186462923033656, - "path": "" - }, - { - "seed": 7, - "value": 0.0486915291328531, - "path": "" - }, - { - "seed": 42, - "value": 0.0381213575591361, - "path": "" - }, - { - "seed": 99, - "value": 0.033258776621608484, - "path": "" - }, - { - "seed": 123, - "value": 0.02461746391852102, - "path": "" - } - ], - "tolerance_f1": [ - { - "seed": 1, - "value": 0.3945515652074689, - "path": "" - }, - { - "seed": 7, - "value": 0.4004617740449798, - "path": "" - }, - { - "seed": 42, - "value": 0.4052951105632299, - "path": "" - }, - { - "seed": 99, - "value": 0.38853542457185924, - "path": "" - }, - { - "seed": 123, - "value": 0.369479820868587, - "path": "" - } - ], - "union_f1": [ - { - "seed": 1, - "value": 0.699798626339235, - "path": "" - }, - { - "seed": 7, - "value": 0.7590968995963846, - "path": "" - }, - { - "seed": 42, - "value": 0.7444554956076256, - "path": "" - }, - { - "seed": 99, - "value": 0.7047690151929771, - "path": "" - }, - { - "seed": 123, - "value": 0.7178986487638336, - "path": "" - } - ], - "difference": [ - { - "seed": 1, - "value": 0.6666121634162013, - "path": "" - }, - { - "seed": 7, - "value": 0.7104053704635315, - "path": "" - }, - { - "seed": 42, - "value": 0.7063341380484894, - "path": "" - }, - { - "seed": 99, - "value": 0.6715102385713686, - "path": "" - }, - { - "seed": 123, - "value": 0.6932811848453125, - "path": "" - } - ], - "predicted_positive_rate": [ - { - "seed": 1, - "value": 0.011875432868309093, - "path": "" - }, - { - "seed": 7, - "value": 0.012246462847531414, - "path": "" - }, - { - "seed": 42, - "value": 0.016721084396952607, - "path": "" - }, - { - "seed": 99, - "value": 0.011400514494904522, - "path": "" - }, - { - "seed": 123, - "value": 0.024304937172256853, - "path": "" - } - ] - }, - "thresholds": [ - 0.9, - 0.95 - ], - "paths": [ - "", - "", - "", - "", - "" - ] - }, - { - "model_tag": "reference", - "label": "Reference", - "scope": "train_fire_top20pct", - "scope_label": "top 20%", - "n": 5, - "strict_f1": { - "n": 5, - "mean": 0.035300192316067396, - "std": 0.008700007560989128, - "min": 0.024505093215452623, - "max": 0.04827336828783222 - }, - "tolerance_f1": { - "n": 5, - "mean": 0.3828491649394263, - "std": 0.012952356930876381, - "min": 0.36252633147981417, - "max": 0.39585180444145995 - }, - "union_f1": { - "n": 5, - "mean": 0.6972278451335353, - "std": 0.02466447592239507, - "min": 0.6717469182906158, - "max": 0.7294445251003491 - }, - "difference": { - "n": 5, - "mean": 0.6619276528174679, - "std": 0.019273149109644262, - "min": 0.6388534040739275, - "max": 0.6811711568125168 - }, - "predicted_positive_rate": { - "n": 5, - "mean": 0.0076554112554112555, - "std": 0.0027315673026463, - "min": 0.005700680272108844, - "max": 0.012153370439084725 - }, - "seed_values": { - "strict_f1": [ - { - "seed": 1, - "value": 0.032893514216688347, - "path": "" - }, - { - "seed": 7, - "value": 0.04827336828783222, - "path": "" - }, - { - "seed": 42, - "value": 0.03787465940054495, - "path": "" - }, - { - "seed": 99, - "value": 0.03295432645981885, - "path": "" - }, - { - "seed": 123, - "value": 0.024505093215452623, - "path": "" - } - ], - "tolerance_f1": [ - { - "seed": 1, - "value": 0.3852239693382854, - "path": "" - }, - { - "seed": 7, - "value": 0.39128423640663973, - "path": "" - }, - { - "seed": 42, - "value": 0.39585180444145995, - "path": "" - }, - { - "seed": 99, - "value": 0.37935948303093225, - "path": "" - }, - { - "seed": 123, - "value": 0.36252633147981417, - "path": "" - } - ], - "union_f1": [ - { - "seed": 1, - "value": 0.6717469182906158, - "path": "" - }, - { - "seed": 7, - "value": 0.7294445251003491, - "path": "" - }, - { - "seed": 42, - "value": 0.7152818658418959, - "path": "" - }, - { - "seed": 99, - "value": 0.6773183297441491, - "path": "" - }, - { - "seed": 123, - "value": 0.6923475866906666, - "path": "" - } - ], - "difference": [ - { - "seed": 1, - "value": 0.6388534040739275, - "path": "" - }, - { - "seed": 7, - "value": 0.6811711568125168, - "path": "" - }, - { - "seed": 42, - "value": 0.677407206441351, - "path": "" - }, - { - "seed": 99, - "value": 0.6443640032843302, - "path": "" - }, - { - "seed": 123, - "value": 0.6678424934752141, - "path": "" - } - ], - "predicted_positive_rate": [ - { - "seed": 1, - "value": 0.0059381570810142234, - "path": "" - }, - { - "seed": 7, - "value": 0.006123685837971553, - "path": "" - }, - { - "seed": 42, - "value": 0.008361162646876932, - "path": "" - }, - { - "seed": 99, - "value": 0.005700680272108844, - "path": "" - }, - { - "seed": 123, - "value": 0.012153370439084725, - "path": "" - } - ] - }, - "thresholds": [ - 0.9, - 0.95 - ], - "paths": [ - "", - "", - "", - "", - "" - ] - }, - { - "model_tag": "stormcast", - "label": "StormCast", - "scope": "full_domain", - "scope_label": "global", - "n": 5, - "strict_f1": { - "n": 5, - "mean": 0.0006255534463925153, - "std": 0.00011899761411994709, - "min": 0.00043296285460247506, - "max": 0.0007179477702826634 - }, - "tolerance_f1": { - "n": 5, - "mean": 0.08195132072062675, - "std": 0.021894994384989055, - "min": 0.04396525865017935, - "max": 0.09668169596405224 - }, - "union_f1": { - "n": 5, - "mean": 0.2238166234389721, - "std": 0.05429431793170512, - "min": 0.12984783035809752, - "max": 0.2609912375807996 - }, - "difference": { - "n": 5, - "mean": 0.2231910699925796, - "std": 0.05417763955331957, - "min": 0.12941486750349504, - "max": 0.26027328981051695 - }, - "predicted_positive_rate": { - "n": 5, - "mean": 0.6065068645640074, - "std": 0.17489509665246694, - "min": 0.3248534322820037, - "max": 0.7873692022263451 - }, - "seed_values": { - "strict_f1": [ - { - "seed": 1, - "value": 0.00043296285460247506, - "path": "" - }, - { - "seed": 7, - "value": 0.0006714969233599232, - "path": "" - }, - { - "seed": 42, - "value": 0.0007179477702826634, - "path": "" - }, - { - "seed": 99, - "value": 0.0007136363665052921, - "path": "" - }, - { - "seed": 123, - "value": 0.0005917233172122231, - "path": "" - } - ], - "tolerance_f1": [ - { - "seed": 1, - "value": 0.04396525865017935, - "path": "" - }, - { - "seed": 7, - "value": 0.09264506128556889, - "path": "" - }, - { - "seed": 42, - "value": 0.09668169596405224, - "path": "" - }, - { - "seed": 99, - "value": 0.09390595738710937, - "path": "" - }, - { - "seed": 123, - "value": 0.08255863031622387, - "path": "" - } - ], - "union_f1": [ - { - "seed": 1, - "value": 0.12984783035809752, - "path": "" - }, - { - "seed": 7, - "value": 0.25008945924787185, - "path": "" - }, - { - "seed": 42, - "value": 0.2609912375807996, - "path": "" - }, - { - "seed": 99, - "value": 0.253626654686362, - "path": "" - }, - { - "seed": 123, - "value": 0.22452793532172965, - "path": "" - } - ], - "difference": [ - { - "seed": 1, - "value": 0.12941486750349504, - "path": "" - }, - { - "seed": 7, - "value": 0.24941796232451194, - "path": "" - }, - { - "seed": 42, - "value": 0.26027328981051695, - "path": "" - }, - { - "seed": 99, - "value": 0.2529130183198567, - "path": "" - }, - { - "seed": 123, - "value": 0.22393621200451744, - "path": "" - } - ], - "predicted_positive_rate": [ - { - "seed": 1, - "value": 0.3248534322820037, - "path": "" - }, - { - "seed": 7, - "value": 0.7037476808905381, - "path": "" - }, - { - "seed": 42, - "value": 0.62891280148423, - "path": "" - }, - { - "seed": 99, - "value": 0.5876512059369202, - "path": "" - }, - { - "seed": 123, - "value": 0.7873692022263451, - "path": "" - } - ] - }, - "thresholds": [ - 0.0002731947461143136, - 0.0002782046503853053, - 0.0002819171058945358, - 0.0002892915508709848, - 0.000301085616229102 - ], - "paths": [ - "", - "", - "", - "", - "" - ] - }, - { - "model_tag": "stormcast", - "label": "StormCast", - "scope": "train_fire_top05pct", - "scope_label": "top 5%", - "n": 5, - "strict_f1": { - "n": 5, - "mean": 0.009572526967992595, - "std": 0.00801147738269429, - "min": 0.005636060100166944, - "max": 0.023876731839958566 - }, - "tolerance_f1": { - "n": 5, - "mean": 0.15321899635692177, - "std": 0.05533660427697729, - "min": 0.10250537509480662, - "max": 0.23852766585861737 - }, - "union_f1": { - "n": 5, - "mean": 0.36185682619841214, - "std": 0.09733119467598991, - "min": 0.25743196970586, - "max": 0.49125319073865004 - }, - "difference": { - "n": 5, - "mean": 0.35228429923041954, - "std": 0.09181595284950021, - "min": 0.2505963038813504, - "max": 0.46737645889869145 - }, - "predicted_positive_rate": { - "n": 5, - "mean": 0.5760166221430691, - "std": 0.3529211324352006, - "min": 0.09290590679726922, - "max": 0.9234194122885129 - }, - "seed_values": { - "strict_f1": [ - { - "seed": 1, - "value": 0.023876731839958566, - "path": "" - }, - { - "seed": 7, - "value": 0.005648734389013078, - "path": "" - }, - { - "seed": 42, - "value": 0.006835665824509611, - "path": "" - }, - { - "seed": 99, - "value": 0.005865442686314773, - "path": "" - }, - { - "seed": 123, - "value": 0.005636060100166944, - "path": "" - } - ], - "tolerance_f1": [ - { - "seed": 1, - "value": 0.23852766585861737, - "path": "" - }, - { - "seed": 7, - "value": 0.1601110414083686, - "path": "" - }, - { - "seed": 42, - "value": 0.10250537509480662, - "path": "" - }, - { - "seed": 99, - "value": 0.10517133941552075, - "path": "" - }, - { - "seed": 123, - "value": 0.15977956000729543, - "path": "" - } - ], - "union_f1": [ - { - "seed": 1, - "value": 0.49125319073865004, - "path": "" - }, - { - "seed": 7, - "value": 0.39479065437879374, - "path": "" - }, - { - "seed": 42, - "value": 0.25743196970586, - "path": "" - }, - { - "seed": 99, - "value": 0.2717309409136105, - "path": "" - }, - { - "seed": 123, - "value": 0.39407737525514647, - "path": "" - } - ], - "difference": [ - { - "seed": 1, - "value": 0.46737645889869145, - "path": "" - }, - { - "seed": 7, - "value": 0.38914191998978065, - "path": "" - }, - { - "seed": 42, - "value": 0.2505963038813504, - "path": "" - }, - { - "seed": 99, - "value": 0.2658654982272957, - "path": "" - }, - { - "seed": 123, - "value": 0.38844131515497954, - "path": "" - } - ], - "predicted_positive_rate": [ - { - "seed": 1, - "value": 0.09290590679726922, - "path": "" - }, - { - "seed": 7, - "value": 0.9213416444048679, - "path": "" - }, - { - "seed": 42, - "value": 0.43306619174829325, - "path": "" - }, - { - "seed": 99, - "value": 0.5093499554764025, - "path": "" - }, - { - "seed": 123, - "value": 0.9234194122885129, - "path": "" - } - ] - }, - "thresholds": [ - 0.0002716928720474243, - 0.00027877578395418823, - 0.000282936030998826, - 0.0002883674460463226, - 0.000301085616229102 - ], - "paths": [ - "", - "", - "", - "", - "" - ] - }, - { - "model_tag": "stormcast", - "label": "StormCast", - "scope": "train_fire_top10pct", - "scope_label": "top 10%", - "n": 5, - "strict_f1": { - "n": 5, - "mean": 0.0072843788538849204, - "std": 0.009280155500075412, - "min": 0.002747305898968793, - "max": 0.023870550161812297 - }, - "tolerance_f1": { - "n": 5, - "mean": 0.12666948350566662, - "std": 0.06328963590343438, - "min": 0.08713204456539597, - "max": 0.23793892088542526 - }, - "union_f1": { - "n": 5, - "mean": 0.30474835114719656, - "std": 0.1065272580858829, - "min": 0.2309636203280157, - "max": 0.4887624633565217 - }, - "difference": { - "n": 5, - "mean": 0.2974639722933117, - "std": 0.09749379807889277, - "min": 0.2282163144290469, - "max": 0.4648919131947094 - }, - "predicted_positive_rate": { - "n": 5, - "mean": 0.6807658058771149, - "std": 0.37389945609232794, - "min": 0.04645295339863461, - "max": 0.9575541703769664 - }, - "seed_values": { - "strict_f1": [ - { - "seed": 1, - "value": 0.023870550161812297, - "path": "" - }, - { - "seed": 7, - "value": 0.00275028568108307, - "path": "" - }, - { - "seed": 42, - "value": 0.00356800070444239, - "path": "" - }, - { - "seed": 99, - "value": 0.003485751823118051, - "path": "" - }, - { - "seed": 123, - "value": 0.002747305898968793, - "path": "" - } - ], - "tolerance_f1": [ - { - "seed": 1, - "value": 0.23793892088542526, - "path": "" - }, - { - "seed": 7, - "value": 0.08722255267001126, - "path": "" - }, - { - "seed": 42, - "value": 0.1112312688602769, - "path": "" - }, - { - "seed": 99, - "value": 0.10982263054722373, - "path": "" - }, - { - "seed": 123, - "value": 0.08713204456539597, - "path": "" - } - ], - "union_f1": [ - { - "seed": 1, - "value": 0.4887624633565217, - "path": "" - }, - { - "seed": 7, - "value": 0.23118547623512836, - "path": "" - }, - { - "seed": 42, - "value": 0.2876668872191355, - "path": "" - }, - { - "seed": 99, - "value": 0.28516330859718164, - "path": "" - }, - { - "seed": 123, - "value": 0.2309636203280157, - "path": "" - } - ], - "difference": [ - { - "seed": 1, - "value": 0.4648919131947094, - "path": "" - }, - { - "seed": 7, - "value": 0.2284351905540453, - "path": "" - }, - { - "seed": 42, - "value": 0.2840988865146931, - "path": "" - }, - { - "seed": 99, - "value": 0.2816775567740636, - "path": "" - }, - { - "seed": 123, - "value": 0.2282163144290469, - "path": "" - } - ], - "predicted_positive_rate": [ - { - "seed": 1, - "value": 0.04645295339863461, - "path": "" - }, - { - "seed": 7, - "value": 0.9565152864351439, - "path": "" - }, - { - "seed": 42, - "value": 0.7009498367468091, - "path": "" - }, - { - "seed": 99, - "value": 0.7423567824280202, - "path": "" - }, - { - "seed": 123, - "value": 0.9575541703769664, - "path": "" - } - ] - }, - "thresholds": [ - 0.0002716928720474243, - 0.00027818421949632466, - 0.00028128057601861656, - 0.0002883674460463226, - 0.000301085616229102 - ], - "paths": [ - "", - "", - "", - "", - "" - ] - }, - { - "model_tag": "stormcast", - "label": "StormCast", - "scope": "train_fire_top20pct", - "scope_label": "top 20%", - "n": 5, - "strict_f1": { - "n": 5, - "mean": 0.005794988618784165, - "std": 0.009104127010779225, - "min": 0.0014637962799384197, - "max": 0.022074159907300116 - }, - "tolerance_f1": { - "n": 5, - "mean": 0.10415728840460951, - "std": 0.07343651804127242, - "min": 0.062163327857870446, - "max": 0.23443811051591282 - }, - "union_f1": { - "n": 5, - "mean": 0.2465979729410411, - "std": 0.12397339055627195, - "min": 0.16937256832950737, - "max": 0.4646063802629719 - }, - "difference": { - "n": 5, - "mean": 0.24080298432225694, - "std": 0.11498838284648957, - "min": 0.16790877204956894, - "max": 0.4425322203556718 - }, - "predicted_positive_rate": { - "n": 5, - "mean": 0.6783970315398886, - "std": 0.39241013872037767, - "min": 0.020630797773654917, - "max": 0.979443413729128 - }, - "seed_values": { - "strict_f1": [ - { - "seed": 1, - "value": 0.022074159907300116, - "path": "" - }, - { - "seed": 7, - "value": 0.0014637962799384197, - "path": "" - }, - { - "seed": 42, - "value": 0.0020248273888312522, - "path": "" - }, - { - "seed": 99, - "value": 0.001948363237912615, - "path": "" - }, - { - "seed": 123, - "value": 0.0014637962799384197, - "path": "" - } - ], - "tolerance_f1": [ - { - "seed": 1, - "value": 0.23443811051591282, - "path": "" - }, - { - "seed": 7, - "value": 0.062163327857870446, - "path": "" - }, - { - "seed": 42, - "value": 0.08079366172039935, - "path": "" - }, - { - "seed": 99, - "value": 0.08122801407099456, - "path": "" - }, - { - "seed": 123, - "value": 0.062163327857870446, - "path": "" - } - ], - "union_f1": [ - { - "seed": 1, - "value": 0.4646063802629719, - "path": "" - }, - { - "seed": 7, - "value": 0.16937256832950737, - "path": "" - }, - { - "seed": 42, - "value": 0.21367238965986382, - "path": "" - }, - { - "seed": 99, - "value": 0.21596595812335517, - "path": "" - }, - { - "seed": 123, - "value": 0.16937256832950737, - "path": "" - } - ], - "difference": [ - { - "seed": 1, - "value": 0.4425322203556718, - "path": "" - }, - { - "seed": 7, - "value": 0.16790877204956894, - "path": "" - }, - { - "seed": 42, - "value": 0.21164756227103257, - "path": "" - }, - { - "seed": 99, - "value": 0.21401759488544256, - "path": "" - }, - { - "seed": 123, - "value": 0.16790877204956894, - "path": "" - } - ], - "predicted_positive_rate": [ - { - "seed": 1, - "value": 0.020630797773654917, - "path": "" - }, - { - "seed": 7, - "value": 0.979443413729128, - "path": "" - }, - { - "seed": 42, - "value": 0.6907606679035251, - "path": "" - }, - { - "seed": 99, - "value": 0.7217068645640075, - "path": "" - }, - { - "seed": 123, - "value": 0.979443413729128, - "path": "" - } - ] - }, - "thresholds": [ - 0.00021927931811660528, - 0.0002497032401151955, - 0.0002781348885037005, - 0.0002812395105138421, - 0.00030111230444163084 - ], - "paths": [ - "", - "", - "", - "", - "" - ] - } - ] -} diff --git a/artifacts/results/pixel_context_tune3_summary.md b/artifacts/results/pixel_context_tune3_summary.md new file mode 100644 index 0000000000000000000000000000000000000000..37e03fd7ec9d8a2f8158e88612a2648151e4afca --- /dev/null +++ b/artifacts/results/pixel_context_tune3_summary.md @@ -0,0 +1,15 @@ +# FireWx-FM Downstream Tune2 Diagnostic + +## Spread +### wave1 +- prithvi_wxc: strict_f1 20.72 +/- 3.80 (n=5); spatial_s4_f1 57.32 +/- 2.14 (n=5); AP 5.04 +/- 0.40 (n=5); Brier 0.01 +/- 0.01 (n=5) +- reference: strict_f1 20.32 +/- 1.15 (n=5); spatial_s4_f1 68.58 +/- 1.98 (n=5); AP 30.44 +/- 1.81 (n=5); Brier 0.06 +/- 0.01 (n=5) + +### tune2 +- pending + +## Occupancy +- pixel_context_tune3: strict_f1 0.69 +/- 0.12 (n=5); t0_s3_f1 14.14 +/- 1.37 (n=5); t3_s3_f1 38.51 +/- 2.34 (n=5); PR-AUC 1.01 +/- 0.31 (n=5) + +## Head Control +- pending diff --git a/artifacts/results/selection_regret_all_backbones_20260504.csv b/artifacts/results/selection_regret_all_backbones_20260504.csv deleted file mode 100644 index ea8a02982a8c66171b1344716be24df46cd272c8..0000000000000000000000000000000000000000 --- a/artifacts/results/selection_regret_all_backbones_20260504.csv +++ /dev/null @@ -1,25 +0,0 @@ -model_tag,label,scope,n,seeds,exact_regret_mean,exact_regret_std,tolerated_regret_mean,tolerated_regret_std,union_regret_mean,union_regret_std -reference,Reference,global,5,1 7 42 99 123,0.0,0.0,0.08783024981138902,0.09670495645481135,0.08783024981138902,0.09670495645481135 -reference,Reference,fire_prone,5,1 7 42 99 123,0.0,0.0,0.03402707057672223,0.032044658643147844,0.03402707057672223,0.032044658643147844 -prithvi_wxc,Prithvi-WxC,global,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0 -prithvi_wxc,Prithvi-WxC,fire_prone,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0 -aurora,Aurora,global,5,1 7 42 99 123,0.00020004882767231798,0.00026703384456332115,0.09851983041506818,0.1298781980037557,0.09851983041506818,0.1298781980037557 -aurora,Aurora,fire_prone,5,1 7 42 99 123,0.008202508825959588,0.01834136732088763,0.14391889430974364,0.32121904665016227,0.14391889430974364,0.32121904665016227 -climax,ClimaX,global,5,1 7 42 99 123,3.0287686240700486e-06,4.147312242167625e-06,0.0012959969982639485,0.0017746169760203706,0.0012959969982639485,0.0017746169760203706 -climax,ClimaX,fire_prone,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0 -stormcast,StormCast,global,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0 -stormcast,StormCast,fire_prone,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0 -pangu_weather,Pangu-Weather,global,5,1 7 42 99 123,0.00013033979247265275,0.0002685372203690466,0.048806713097574374,0.10733308684741971,0.048806713097574374,0.10733308684741971 -pangu_weather,Pangu-Weather,fire_prone,5,1 7 42 99 123,0.027875386332505546,0.02348779386900393,0.43111948243387105,0.39355644251497235,0.43111948243387105,0.39355644251497235 -dlwp,DLWP,global,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0 -dlwp,DLWP,fire_prone,5,1 7 42 99 123,0.0007702319787454587,0.0010995336594539604,0.043265915053601556,0.04332331365579739,0.043265915053601556,0.04332331365579739 -fcn,FCN,global,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0 -fcn,FCN,fire_prone,5,1 7 42 99 123,5.960229415004348e-06,1.3327478133443526e-05,0.011679805987441694,0.019872372458657642,0.011679805987441694,0.019872372458657642 -fengwu,FengWu,global,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0 -fengwu,FengWu,fire_prone,5,1 7 42 99 123,0.0006908222234409067,0.0011910586589384115,0.005222389249812243,0.0062394095558402415,0.005222389249812243,0.0062394095558402415 -fuxi,FuXi,global,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0 -fuxi,FuXi,fire_prone,5,1 7 42 99 123,0.0,0.0,0.0010839188523199318,0.0017288780545672386,0.0010839188523199318,0.0017288780545672386 -pangu6,Pangu-Weather,global,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0 -pangu6,Pangu-Weather,fire_prone,5,1 7 42 99 123,0.0007280423771922354,0.001178746460551365,0.0018491271881979853,0.0032630386057089294,0.0018491271881979853,0.0032630386057089294 -alphaearth,AlphaEarth,global,5,1 7 42 99 123,0.0,0.0,0.1722171037486726,0.08849214830495522,0.1722171037486726,0.08849214830495522 -alphaearth,AlphaEarth,fire_prone,5,1 7 42 99 123,0.0,0.0,0.038803552655092256,0.0594825313313219,0.038803552655092256,0.0594825313313219 diff --git a/artifacts/results/selection_regret_all_backbones_20260504.json b/artifacts/results/selection_regret_all_backbones_20260504.json deleted file mode 100644 index c5d58018b40925a115f13542184a090ab6582af2..0000000000000000000000000000000000000000 --- a/artifacts/results/selection_regret_all_backbones_20260504.json +++ /dev/null @@ -1,5196 +0,0 @@ -{ - "rows": [ - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.004193290734824281", - "exact_ranking_score": "0.004193290734824281", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.0056438127090301", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.0056438127090301", - "family": "AlphaEarth", - "model_tag": "alphaearth", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.0006549130347299629", - "ranking_selected_val_pr_auc": "0.0007883068263073185", - "scope": "global", - "seed": 1, - "tolerated_decision_head": "pixel MLP head", - "tolerated_decision_head_arch": "pixel_mlp", - "tolerated_decision_score": "0.6337627266658229", - "tolerated_ranking_score": "0.40561891947698747", - "tolerated_regret": 0.22814380718883542, - "tolerated_test_gap": "0.22814380718883542", - "tolerated_val_decision_score": "0.6238973446702573", - "tolerated_val_gap": "0.1715500493543587", - "tolerated_val_ranking_score": "0.45234729531589857", - "union_decision_head": "pixel MLP head", - "union_decision_head_arch": "pixel_mlp", - "union_decision_score": "0.6337627266658229", - "union_ranking_score": "0.40561891947698747", - "union_regret": 0.22814380718883542, - "union_test_gap": "0.22814380718883542", - "union_val_decision_score": "0.6238973446702573", - "union_val_gap": "0.1715500493543587", - "union_val_ranking_score": "0.45234729531589857", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.09715762273901808", - "exact_ranking_score": "0.09715762273901808", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.13462469733656174", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.13462469733656174", - "family": "AlphaEarth", - "model_tag": "alphaearth", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.04390614036305261", - "ranking_selected_val_pr_auc": "0.05734457583552234", - "scope": "fire_prone", - "seed": 1, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7863702028280513", - "tolerated_ranking_score": "0.7863702028280513", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.758785818734687", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.758785818734687", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7863702028280513", - "union_ranking_score": "0.7863702028280513", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.758785818734687", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.758785818734687", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.006959088991986505", - "exact_ranking_score": "0.006959088991986505", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.006399069226294357", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.006399069226294357", - "family": "AlphaEarth", - "model_tag": "alphaearth", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.0007047712457371991", - "ranking_selected_val_pr_auc": "0.0007434343102653092", - "scope": "global", - "seed": 123, - "tolerated_decision_head": "pixel MLP head", - "tolerated_decision_head_arch": "pixel_mlp", - "tolerated_decision_score": "0.5604635792586619", - "tolerated_ranking_score": "0.4427625907752311", - "tolerated_regret": 0.11770098848343075, - "tolerated_test_gap": "0.11770098848343075", - "tolerated_val_decision_score": "0.5441785824390972", - "tolerated_val_gap": "0.056405166058839995", - "tolerated_val_ranking_score": "0.48777341638025723", - "union_decision_head": "pixel MLP head", - "union_decision_head_arch": "pixel_mlp", - "union_decision_score": "0.5604635792586619", - "union_ranking_score": "0.4427625907752311", - "union_regret": 0.11770098848343075, - "union_test_gap": "0.11770098848343075", - "union_val_decision_score": "0.5441785824390972", - "union_val_gap": "0.056405166058839995", - "union_val_ranking_score": "0.48777341638025723", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.10531544957774468", - "exact_ranking_score": "0.10531544957774468", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.12267343485617596", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.12267343485617596", - "family": "AlphaEarth", - "model_tag": "alphaearth", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.045809049876129763", - "ranking_selected_val_pr_auc": "0.05470980065861018", - "scope": "fire_prone", - "seed": 123, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7789089693560928", - "tolerated_ranking_score": "0.7789089693560928", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.749825256630312", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.749825256630312", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7789089693560928", - "union_ranking_score": "0.7789089693560928", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.749825256630312", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.749825256630312", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.004809747755451047", - "exact_ranking_score": "0.004809747755451047", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.004359099620457705", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.004359099620457705", - "family": "AlphaEarth", - "model_tag": "alphaearth", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.0005634701573991865", - "ranking_selected_val_pr_auc": "0.0005920938943373892", - "scope": "global", - "seed": 42, - "tolerated_decision_head": "pixel MLP head", - "tolerated_decision_head_arch": "pixel_mlp", - "tolerated_decision_score": "0.6812131506751973", - "tolerated_ranking_score": "0.4087444681515033", - "tolerated_regret": 0.272468682523694, - "tolerated_test_gap": "0.272468682523694", - "tolerated_val_decision_score": "0.7447325774946227", - "tolerated_val_gap": "0.3021550064919567", - "tolerated_val_ranking_score": "0.44257757100266604", - "union_decision_head": "pixel MLP head", - "union_decision_head_arch": "pixel_mlp", - "union_decision_score": "0.6812131506751973", - "union_ranking_score": "0.4087444681515033", - "union_regret": 0.272468682523694, - "union_test_gap": "0.272468682523694", - "union_val_decision_score": "0.7447325774946227", - "union_val_gap": "0.3021550064919567", - "union_val_ranking_score": "0.44257757100266604", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.08702469619756958", - "exact_ranking_score": "0.08702469619756958", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.10515392597717052", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.10515392597717052", - "family": "AlphaEarth", - "model_tag": "alphaearth", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.038083070948941686", - "ranking_selected_val_pr_auc": "0.04505214599515043", - "scope": "fire_prone", - "seed": 42, - "tolerated_decision_head": "pixel MLP head", - "tolerated_decision_head_arch": "pixel_mlp", - "tolerated_decision_score": "0.8461131676361712", - "tolerated_ranking_score": "0.7112901458230849", - "tolerated_regret": 0.13482302181308625, - "tolerated_test_gap": "0.13482302181308625", - "tolerated_val_decision_score": "0.806760850261024", - "tolerated_val_gap": "0.07815518654517495", - "tolerated_val_ranking_score": "0.728605663715849", - "union_decision_head": "pixel MLP head", - "union_decision_head_arch": "pixel_mlp", - "union_decision_score": "0.8461131676361712", - "union_ranking_score": "0.7112901458230849", - "union_regret": 0.13482302181308625, - "union_test_gap": "0.13482302181308625", - "union_val_decision_score": "0.806760850261024", - "union_val_gap": "0.07815518654517495", - "union_val_ranking_score": "0.728605663715849", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.010460251046025104", - "exact_ranking_score": "0.010460251046025104", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.009547591069330199", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.009547591069330199", - "family": "AlphaEarth", - "model_tag": "alphaearth", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.001005722733868245", - "ranking_selected_val_pr_auc": "0.0009777524224237678", - "scope": "global", - "seed": 7, - "tolerated_decision_head": "pixel MLP head", - "tolerated_decision_head_arch": "pixel_mlp", - "tolerated_decision_score": "0.6691395427484861", - "tolerated_ranking_score": "0.6184842128568402", - "tolerated_regret": 0.050655329891645895, - "tolerated_test_gap": "0.050655329891645895", - "tolerated_val_decision_score": "0.677404587086621", - "tolerated_val_gap": "0.011527156971746999", - "tolerated_val_ranking_score": "0.665877430114874", - "union_decision_head": "pixel MLP head", - "union_decision_head_arch": "pixel_mlp", - "union_decision_score": "0.6691395427484861", - "union_ranking_score": "0.6184842128568402", - "union_regret": 0.050655329891645895, - "union_test_gap": "0.050655329891645895", - "union_val_decision_score": "0.677404587086621", - "union_val_gap": "0.011527156971746999", - "union_val_ranking_score": "0.665877430114874", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.12514898688915374", - "exact_ranking_score": "0.12514898688915374", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.14325685506435365", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.14325685506435365", - "family": "AlphaEarth", - "model_tag": "alphaearth", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.05955397140893137", - "ranking_selected_val_pr_auc": "0.06058697421048699", - "scope": "fire_prone", - "seed": 7, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.8294499693616161", - "tolerated_ranking_score": "0.8294499693616161", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.8107581951083846", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.8107581951083846", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.8294499693616161", - "union_ranking_score": "0.8294499693616161", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.8107581951083846", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.8107581951083846", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.006780481898534931", - "exact_ranking_score": "0.006780481898534931", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.005964994898359627", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.005964994898359627", - "family": "AlphaEarth", - "model_tag": "alphaearth", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.0006577120081349608", - "ranking_selected_val_pr_auc": "0.0007340310681749519", - "scope": "global", - "seed": 99, - "tolerated_decision_head": "pixel MLP head", - "tolerated_decision_head_arch": "pixel_mlp", - "tolerated_decision_score": "0.5842714676652996", - "tolerated_ranking_score": "0.3921547570095426", - "tolerated_regret": 0.19211671065575697, - "tolerated_test_gap": "0.19211671065575697", - "tolerated_val_decision_score": "0.5663548190132099", - "tolerated_val_gap": "0.10549666519652506", - "tolerated_val_ranking_score": "0.4608581538166849", - "union_decision_head": "pixel MLP head", - "union_decision_head_arch": "pixel_mlp", - "union_decision_score": "0.5842714676652996", - "union_ranking_score": "0.3921547570095426", - "union_regret": 0.19211671065575697, - "union_test_gap": "0.19211671065575697", - "union_val_decision_score": "0.5663548190132099", - "union_val_gap": "0.10549666519652506", - "union_val_ranking_score": "0.4608581538166849", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.10365251727541955", - "exact_ranking_score": "0.10365251727541955", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.12251941328731666", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.12251941328731666", - "family": "AlphaEarth", - "model_tag": "alphaearth", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.0458102699699856", - "ranking_selected_val_pr_auc": "0.0554648890254577", - "scope": "fire_prone", - "seed": 99, - "tolerated_decision_head": "pixel MLP head", - "tolerated_decision_head_arch": "pixel_mlp", - "tolerated_decision_score": "0.8350245452333586", - "tolerated_ranking_score": "0.7758298037709835", - "tolerated_regret": 0.05919474146237502, - "tolerated_test_gap": "0.05919474146237502", - "tolerated_val_decision_score": "0.7736506356603177", - "tolerated_val_gap": "0.027586681737492236", - "tolerated_val_ranking_score": "0.7460639539228254", - "union_decision_head": "pixel MLP head", - "union_decision_head_arch": "pixel_mlp", - "union_decision_score": "0.8350245452333586", - "union_ranking_score": "0.7758298037709835", - "union_regret": 0.05919474146237502, - "union_test_gap": "0.05919474146237502", - "union_val_decision_score": "0.7736506356603177", - "union_val_gap": "0.027586681737492236", - "union_val_ranking_score": "0.7460639539228254", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0004925501476206198", - "exact_ranking_score": "0.00048497822606044473", - "exact_regret": 7.571921560175121e-06, - "family": "Aurora", - "model_tag": "aurora", - "ranking_selected_head": "linear probe", - "ranking_selected_head_arch": "linear", - "ranking_selected_pr_auc": "0.00024254792826221397", - "scope": "global", - "seed": 1, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.240793572992212", - "tolerated_ranking_score": "0.23755358049655212", - "tolerated_regret": 0.0032399924956598714, - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.240793572992212", - "union_ranking_score": "0.23755358049655212", - "union_regret": 0.0032399924956598714, - "path": "" - }, - { - "exact_decision_head": "linear probe", - "exact_decision_head_arch": "linear", - "exact_decision_score": "0.04101254412979794", - "exact_ranking_score": "0.04101254412979794", - "exact_regret": 0.0, - "family": "Aurora", - "model_tag": "aurora", - "ranking_selected_head": "linear probe", - "ranking_selected_head_arch": "linear", - "ranking_selected_pr_auc": "0.02093558282208589", - "scope": "fire_prone", - "seed": 1, - "tolerated_decision_head": "linear probe", - "tolerated_decision_head_arch": "linear", - "tolerated_decision_score": "0.7185324707231184", - "tolerated_ranking_score": "0.7185324707231184", - "tolerated_regret": 0.0, - "union_decision_head": "linear probe", - "union_decision_head_arch": "linear", - "union_decision_score": "0.7185324707231184", - "union_ranking_score": "0.7185324707231184", - "union_regret": 0.0, - "path": "" - }, - { - "exact_decision_head": "pixel MLP head", - "exact_decision_head_arch": "pixel_mlp", - "exact_decision_score": "0.0005311562430574265", - "exact_ranking_score": "0.0005311562430574265", - "exact_regret": 0.0, - "family": "Aurora", - "model_tag": "aurora", - "ranking_selected_head": "pixel MLP head", - "ranking_selected_head_arch": "pixel_mlp", - "ranking_selected_pr_auc": "0.00031683315961488916", - "scope": "global", - "seed": 123, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.240793572992212", - "tolerated_ranking_score": "0.23647112940979162", - "tolerated_regret": 0.004322443582420371, - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.240793572992212", - "union_ranking_score": "0.23647112940979162", - "union_regret": 0.004322443582420371, - "path": "" - }, - { - "exact_decision_head": "pixel MLP head", - "exact_decision_head_arch": "pixel_mlp", - "exact_decision_score": "0.04541038665445361", - "exact_ranking_score": "0.04541038665445361", - "exact_regret": 0.0, - "family": "Aurora", - "model_tag": "aurora", - "ranking_selected_head": "pixel MLP head", - "ranking_selected_head_arch": "pixel_mlp", - "ranking_selected_pr_auc": "0.03014151567817997", - "scope": "fire_prone", - "seed": 123, - "tolerated_decision_head": "linear probe", - "tolerated_decision_head_arch": "linear", - "tolerated_decision_score": "0.7185324707231184", - "tolerated_ranking_score": "0.7175172112337449", - "tolerated_regret": 0.0010152594893735323, - "union_decision_head": "linear probe", - "union_decision_head_arch": "linear", - "union_decision_score": "0.7185324707231184", - "union_ranking_score": "0.7175172112337449", - "union_regret": 0.0010152594893735323, - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0004925501476206198", - "exact_ranking_score": "0.00048497822606044473", - "exact_regret": 7.571921560175121e-06, - "family": "Aurora", - "model_tag": "aurora", - "ranking_selected_head": "linear probe", - "ranking_selected_head_arch": "linear", - "ranking_selected_pr_auc": "0.0002876372030063385", - "scope": "global", - "seed": 42, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.240793572992212", - "tolerated_ranking_score": "0.0", - "tolerated_regret": 0.240793572992212, - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.240793572992212", - "union_ranking_score": "0.0", - "union_regret": 0.240793572992212, - "path": "" - }, - { - "exact_decision_head": "linear probe", - "exact_decision_head_arch": "linear", - "exact_decision_score": "0.04101254412979794", - "exact_ranking_score": "0.04101254412979794", - "exact_regret": 0.0, - "family": "Aurora", - "model_tag": "aurora", - "ranking_selected_head": "linear probe", - "ranking_selected_head_arch": "linear", - "ranking_selected_pr_auc": "0.02613792907867929", - "scope": "fire_prone", - "seed": 42, - "tolerated_decision_head": "linear probe", - "tolerated_decision_head_arch": "linear", - "tolerated_decision_score": "0.7185324707231184", - "tolerated_ranking_score": "0.7185324707231184", - "tolerated_regret": 0.0, - "union_decision_head": "linear probe", - "union_decision_head_arch": "linear", - "union_decision_score": "0.7185324707231184", - "union_ranking_score": "0.7185324707231184", - "union_regret": 0.0, - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0004925501476206198", - "exact_ranking_score": "0.0", - "exact_regret": 0.0004925501476206198, - "family": "Aurora", - "model_tag": "aurora", - "ranking_selected_head": "linear probe", - "ranking_selected_head_arch": "linear", - "ranking_selected_pr_auc": "0.00027660331739269843", - "scope": "global", - "seed": 7, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.240793572992212", - "tolerated_ranking_score": "0.23734400297937533", - "tolerated_regret": 0.0034495700128366613, - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.240793572992212", - "union_ranking_score": "0.23734400297937533", - "union_regret": 0.0034495700128366613, - "path": "" - }, - { - "exact_decision_head": "linear probe", - "exact_decision_head_arch": "linear", - "exact_decision_score": "0.0413500618483831", - "exact_ranking_score": "0.0413500618483831", - "exact_regret": 0.0, - "family": "Aurora", - "model_tag": "aurora", - "ranking_selected_head": "linear probe", - "ranking_selected_head_arch": "linear", - "ranking_selected_pr_auc": "0.024802820904513342", - "scope": "fire_prone", - "seed": 7, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7185324707231184", - "tolerated_ranking_score": "0.7184857293868923", - "tolerated_regret": 4.674133622617482e-05, - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7185324707231184", - "union_ranking_score": "0.7184857293868923", - "union_regret": 4.674133622617482e-05, - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0004925501476206198", - "exact_ranking_score": "0.0", - "exact_regret": 0.0004925501476206198, - "family": "Aurora", - "model_tag": "aurora", - "ranking_selected_head": "linear probe", - "ranking_selected_head_arch": "linear", - "ranking_selected_pr_auc": "0.00024254792826221397", - "scope": "global", - "seed": 99, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.240793572992212", - "tolerated_ranking_score": "0.0", - "tolerated_regret": 0.240793572992212, - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.240793572992212", - "union_ranking_score": "0.0", - "union_regret": 0.240793572992212, - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.04101254412979794", - "exact_ranking_score": "0.0", - "exact_regret": 0.04101254412979794, - "family": "Aurora", - "model_tag": "aurora", - "ranking_selected_head": "linear probe", - "ranking_selected_head_arch": "linear", - "ranking_selected_pr_auc": "0.02093558282208589", - "scope": "fire_prone", - "seed": 99, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7185324707231184", - "tolerated_ranking_score": "0.0", - "tolerated_regret": 0.7185324707231184, - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7185324707231184", - "union_ranking_score": "0.0", - "union_regret": 0.7185324707231184, - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0004925501476206198", - "exact_ranking_score": "0.00048497822606044473", - "exact_regret": 7.571921560175121e-06, - "family": "ClimaX", - "model_tag": "climax", - "ranking_selected_head": "linear probe", - "ranking_selected_head_arch": "linear", - "ranking_selected_pr_auc": "0.0002543464414550104", - "scope": "global", - "seed": 1, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.240793572992212", - "tolerated_ranking_score": "0.23755358049655212", - "tolerated_regret": 0.0032399924956598714, - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.240793572992212", - "union_ranking_score": "0.23755358049655212", - "union_regret": 0.0032399924956598714, - "path": "" - }, - { - "exact_decision_head": "linear probe", - "exact_decision_head_arch": "linear", - "exact_decision_score": "0.04101254412979794", - "exact_ranking_score": "0.04101254412979794", - "exact_regret": 0.0, - "family": "ClimaX", - "model_tag": "climax", - "ranking_selected_head": "linear probe", - "ranking_selected_head_arch": "linear", - "ranking_selected_pr_auc": "0.02281272244151735", - "scope": "fire_prone", - "seed": 1, - "tolerated_decision_head": "linear probe", - "tolerated_decision_head_arch": "linear", - "tolerated_decision_score": "0.7185324707231184", - "tolerated_ranking_score": "0.7185324707231184", - "tolerated_regret": 0.0, - "union_decision_head": "linear probe", - "union_decision_head_arch": "linear", - "union_decision_score": "0.7185324707231184", - "union_ranking_score": "0.7185324707231184", - "union_regret": 0.0, - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0004925501476206198", - "exact_ranking_score": "0.0004925501476206198", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.0004727121748000204", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.0004727121748000204", - "family": "ClimaX", - "model_tag": "climax", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.00025952340213823634", - "ranking_selected_val_pr_auc": "0.00028511175967657057", - "scope": "global", - "seed": 123, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.240793572992212", - "tolerated_ranking_score": "0.240793572992212", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.2179368807313983", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.2179368807313983", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.240793572992212", - "union_ranking_score": "0.240793572992212", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.2179368807313983", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.2179368807313983", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.04101254412979794", - "exact_ranking_score": "0.04101254412979794", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.04253189892419314", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.04253189892419314", - "family": "ClimaX", - "model_tag": "climax", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.024930358757410707", - "ranking_selected_val_pr_auc": "0.03159877521821671", - "scope": "fire_prone", - "seed": 123, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7185324707231184", - "tolerated_ranking_score": "0.7185324707231184", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.6906754133476136", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.6906754133476136", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7185324707231184", - "union_ranking_score": "0.7185324707231184", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.6906754133476136", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.6906754133476136", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0004925501476206198", - "exact_ranking_score": "0.0004925501476206198", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.0004727121748000204", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.0004727121748000204", - "family": "ClimaX", - "model_tag": "climax", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.00023723426605001756", - "ranking_selected_val_pr_auc": "0.00023206733426223207", - "scope": "global", - "seed": 42, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.240793572992212", - "tolerated_ranking_score": "0.240793572992212", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.2179368807313983", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.2179368807313983", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.240793572992212", - "union_ranking_score": "0.240793572992212", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.2179368807313983", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.2179368807313983", - "path": "" - }, - { - "exact_decision_head": "linear probe", - "exact_decision_head_arch": "linear", - "exact_decision_score": "0.04101254412979794", - "exact_ranking_score": "0.04101254412979794", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.04253189892419314", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.04253189892419314", - "family": "ClimaX", - "model_tag": "climax", - "ranking_selected_head": "linear probe", - "ranking_selected_head_arch": "linear", - "ranking_selected_test_pr_auc": "0.021516770872035896", - "ranking_selected_val_pr_auc": "0.023234418719582723", - "scope": "fire_prone", - "seed": 42, - "tolerated_decision_head": "linear probe", - "tolerated_decision_head_arch": "linear", - "tolerated_decision_score": "0.7185324707231184", - "tolerated_ranking_score": "0.7185324707231184", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.6906754133476136", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.6906754133476136", - "union_decision_head": "linear probe", - "union_decision_head_arch": "linear", - "union_decision_score": "0.7185324707231184", - "union_ranking_score": "0.7185324707231184", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.6906754133476136", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.6906754133476136", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0004925501476206198", - "exact_ranking_score": "0.00048497822606044473", - "exact_regret": 7.571921560175121e-06, - "exact_test_gap": "7.571921560175121e-06", - "exact_val_decision_score": "0.0004727121748000204", - "exact_val_gap": "7.267025482891767e-06", - "exact_val_ranking_score": "0.00046544514931712863", - "family": "ClimaX", - "model_tag": "climax", - "ranking_selected_head": "pixel MLP head", - "ranking_selected_head_arch": "pixel_mlp", - "ranking_selected_test_pr_auc": "0.00025423546642937565", - "ranking_selected_val_pr_auc": "0.00023120715224267713", - "scope": "global", - "seed": 7, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.240793572992212", - "tolerated_ranking_score": "0.23755358049655212", - "tolerated_regret": 0.0032399924956598714, - "tolerated_test_gap": "0.0032399924956598714", - "tolerated_val_decision_score": "0.2179368807313983", - "tolerated_val_gap": "0.00299097714352789", - "tolerated_val_ranking_score": "0.2149459035878704", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.240793572992212", - "union_ranking_score": "0.23755358049655212", - "union_regret": 0.0032399924956598714, - "union_test_gap": "0.0032399924956598714", - "union_val_decision_score": "0.2179368807313983", - "union_val_gap": "0.00299097714352789", - "union_val_ranking_score": "0.2149459035878704", - "path": "" - }, - { - "exact_decision_head": "linear probe", - "exact_decision_head_arch": "linear", - "exact_decision_score": "0.04101254412979794", - "exact_ranking_score": "0.04101254412979794", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.04253189892419314", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.04253189892419314", - "family": "ClimaX", - "model_tag": "climax", - "ranking_selected_head": "linear probe", - "ranking_selected_head_arch": "linear", - "ranking_selected_test_pr_auc": "0.021317405351800135", - "ranking_selected_val_pr_auc": "0.02320587721933335", - "scope": "fire_prone", - "seed": 7, - "tolerated_decision_head": "linear probe", - "tolerated_decision_head_arch": "linear", - "tolerated_decision_score": "0.7185324707231184", - "tolerated_ranking_score": "0.7185324707231184", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.6906754133476136", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.6906754133476136", - "union_decision_head": "linear probe", - "union_decision_head_arch": "linear", - "union_decision_score": "0.7185324707231184", - "union_ranking_score": "0.7185324707231184", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.6906754133476136", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.6906754133476136", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.00048696535779102983", - "exact_ranking_score": "0.00048696535779102983", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.00046735226565276225", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.00046735226565276225", - "family": "ClimaX", - "model_tag": "climax", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.0002340075376021003", - "ranking_selected_val_pr_auc": "0.00023166839498120457", - "scope": "global", - "seed": 99, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.2384111045733381", - "tolerated_ranking_score": "0.2384111045733381", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.21573180756095836", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.21573180756095836", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.2384111045733381", - "union_ranking_score": "0.2384111045733381", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.21573180756095836", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.21573180756095836", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.04101254412979794", - "exact_ranking_score": "0.04101254412979794", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.04253189892419314", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.04253189892419314", - "family": "ClimaX", - "model_tag": "climax", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.02099123219536693", - "ranking_selected_val_pr_auc": "0.023142108180759017", - "scope": "fire_prone", - "seed": 99, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7185324707231184", - "tolerated_ranking_score": "0.7185324707231184", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.6906754133476136", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.6906754133476136", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7185324707231184", - "union_ranking_score": "0.7185324707231184", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.6906754133476136", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.6906754133476136", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.00487022180273714", - "exact_ranking_score": "0.00487022180273714", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.0013194321729398599", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.0013194321729398599", - "family": "DLWP", - "model_tag": "dlwp", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.0006257446338466172", - "ranking_selected_val_pr_auc": "0.00041081035639410466", - "scope": "global", - "seed": 1, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.38023285660836226", - "tolerated_ranking_score": "0.38023285660836226", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.40408253980815234", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.40408253980815234", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.38023285660836226", - "union_ranking_score": "0.38023285660836226", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.40408253980815234", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.40408253980815234", - "path": "" - }, - { - "exact_decision_head": "linear probe", - "exact_decision_head_arch": "linear", - "exact_decision_score": "0.043506471331489265", - "exact_ranking_score": "0.043506471331489265", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.04461624715984538", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.04461624715984538", - "family": "DLWP", - "model_tag": "dlwp", - "ranking_selected_head": "linear probe", - "ranking_selected_head_arch": "linear", - "ranking_selected_test_pr_auc": "0.019747545663020845", - "ranking_selected_val_pr_auc": "0.019740435565559492", - "scope": "fire_prone", - "seed": 1, - "tolerated_decision_head": "linear probe", - "tolerated_decision_head_arch": "linear", - "tolerated_decision_score": "0.7280364139105968", - "tolerated_ranking_score": "0.7280364139105968", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.7120828721033882", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.7120828721033882", - "union_decision_head": "linear probe", - "union_decision_head_arch": "linear", - "union_decision_score": "0.7280364139105968", - "union_ranking_score": "0.7280364139105968", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.7120828721033882", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.7120828721033882", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0", - "exact_ranking_score": "0.0", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.0025026814444047193", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.0025026814444047193", - "family": "DLWP", - "model_tag": "dlwp", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.0007129763973023342", - "ranking_selected_val_pr_auc": "0.0004370316771032176", - "scope": "global", - "seed": 123, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.3797689460796109", - "tolerated_ranking_score": "0.3797689460796109", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.4257234764346952", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.4257234764346952", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.3797689460796109", - "union_ranking_score": "0.3797689460796109", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.4257234764346952", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.4257234764346952", - "path": "" - }, - { - "exact_decision_head": "linear probe", - "exact_decision_head_arch": "linear", - "exact_decision_score": "0.04249854057209574", - "exact_ranking_score": "0.04101254412979794", - "exact_regret": 0.001485996442297799, - "exact_test_gap": "0.001485996442297799", - "exact_val_decision_score": "0.04371409485637943", - "exact_val_gap": "0.0011821959321862904", - "exact_val_ranking_score": "0.04253189892419314", - "family": "DLWP", - "model_tag": "dlwp", - "ranking_selected_head": "pixel MLP head", - "ranking_selected_head_arch": "pixel_mlp", - "ranking_selected_test_pr_auc": "0.021118517500793188", - "ranking_selected_val_pr_auc": "0.02303236785318257", - "scope": "fire_prone", - "seed": 123, - "tolerated_decision_head": "linear probe", - "tolerated_decision_head_arch": "linear", - "tolerated_decision_score": "0.7321459738801157", - "tolerated_ranking_score": "0.7185324707231184", - "tolerated_regret": 0.013613503156997275, - "tolerated_test_gap": "0.013613503156997275", - "tolerated_val_decision_score": "0.7128995538153613", - "tolerated_val_gap": "0.022224140467747677", - "tolerated_val_ranking_score": "0.6906754133476136", - "union_decision_head": "linear probe", - "union_decision_head_arch": "linear", - "union_decision_score": "0.7321459738801157", - "union_ranking_score": "0.7185324707231184", - "union_regret": 0.013613503156997275, - "union_test_gap": "0.013613503156997275", - "union_val_decision_score": "0.7128995538153613", - "union_val_gap": "0.022224140467747677", - "union_val_ranking_score": "0.6906754133476136", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0", - "exact_ranking_score": "0.0", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.0020120724346076456", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.0020120724346076456", - "family": "DLWP", - "model_tag": "dlwp", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.0006203713852571992", - "ranking_selected_val_pr_auc": "0.0004707444097300293", - "scope": "global", - "seed": 42, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.3405125814370199", - "tolerated_ranking_score": "0.3405125814370199", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.474350305376108", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.474350305376108", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.3405125814370199", - "union_ranking_score": "0.3405125814370199", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.474350305376108", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.474350305376108", - "path": "" - }, - { - "exact_decision_head": "pixel MLP head", - "exact_decision_head_arch": "pixel_mlp", - "exact_decision_score": "0.04637177602565815", - "exact_ranking_score": "0.04637177602565815", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.044021164021164016", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.044021164021164016", - "family": "DLWP", - "model_tag": "dlwp", - "ranking_selected_head": "pixel MLP head", - "ranking_selected_head_arch": "pixel_mlp", - "ranking_selected_test_pr_auc": "0.020762153794205103", - "ranking_selected_val_pr_auc": "0.020894537350314525", - "scope": "fire_prone", - "seed": 42, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7728400679088107", - "tolerated_ranking_score": "0.6606900017471591", - "tolerated_regret": 0.11215006616165157, - "tolerated_test_gap": "0.11215006616165157", - "tolerated_val_decision_score": "0.7122584178150692", - "tolerated_val_gap": "0.009110393702543962", - "tolerated_val_ranking_score": "0.7031480241125252", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7728400679088107", - "union_ranking_score": "0.6606900017471591", - "union_regret": 0.11215006616165157, - "union_test_gap": "0.11215006616165157", - "union_val_decision_score": "0.7122584178150692", - "union_val_gap": "0.009110393702543962", - "union_val_ranking_score": "0.7031480241125252", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0", - "exact_ranking_score": "0.0", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.0010056989607777405", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.0010056989607777405", - "family": "DLWP", - "model_tag": "dlwp", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.0005264872646085452", - "ranking_selected_val_pr_auc": "0.00035284338324857376", - "scope": "global", - "seed": 7, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.3432315705541329", - "tolerated_ranking_score": "0.3432315705541329", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.3469162743670964", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.3469162743670964", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.3432315705541329", - "union_ranking_score": "0.3432315705541329", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.3469162743670964", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.3469162743670964", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.043377707581227436", - "exact_ranking_score": "0.04101254412979794", - "exact_regret": 0.0023651634514294945, - "exact_test_gap": "0.0023651634514294945", - "exact_val_decision_score": "0.04303681559882578", - "exact_val_gap": "0.0005049166746326425", - "exact_val_ranking_score": "0.04253189892419314", - "family": "DLWP", - "model_tag": "dlwp", - "ranking_selected_head": "pixel MLP head", - "ranking_selected_head_arch": "pixel_mlp", - "ranking_selected_test_pr_auc": "0.018519739310339497", - "ranking_selected_val_pr_auc": "0.01953230111607746", - "scope": "fire_prone", - "seed": 7, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.762536895087842", - "tolerated_ranking_score": "0.7185324707231184", - "tolerated_regret": 0.044004424364723516, - "tolerated_test_gap": "0.044004424364723516", - "tolerated_val_decision_score": "0.7274867592832499", - "tolerated_val_gap": "0.036811345935636286", - "tolerated_val_ranking_score": "0.6906754133476136", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.762536895087842", - "union_ranking_score": "0.7185324707231184", - "union_regret": 0.044004424364723516, - "union_test_gap": "0.044004424364723516", - "union_val_decision_score": "0.7274867592832499", - "union_val_gap": "0.036811345935636286", - "union_val_ranking_score": "0.6906754133476136", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0", - "exact_ranking_score": "0.0", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.002074688796680498", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.002074688796680498", - "family": "DLWP", - "model_tag": "dlwp", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.0007477128447471452", - "ranking_selected_val_pr_auc": "0.00047085132311599663", - "scope": "global", - "seed": 99, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.3979559626836394", - "tolerated_ranking_score": "0.3979559626836394", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.4277982050028173", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.4277982050028173", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.3979559626836394", - "union_ranking_score": "0.3979559626836394", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.4277982050028173", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.4277982050028173", - "path": "" - }, - { - "exact_decision_head": "linear probe", - "exact_decision_head_arch": "linear", - "exact_decision_score": "0.04819843096725701", - "exact_ranking_score": "0.04819843096725701", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.05605272285813389", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.05605272285813389", - "family": "DLWP", - "model_tag": "dlwp", - "ranking_selected_head": "linear probe", - "ranking_selected_head_arch": "linear", - "ranking_selected_test_pr_auc": "0.02136633936888583", - "ranking_selected_val_pr_auc": "0.024917289620763165", - "scope": "fire_prone", - "seed": 99, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7653346239087763", - "tolerated_ranking_score": "0.7187730423241409", - "tolerated_regret": 0.046561581584635414, - "tolerated_test_gap": "0.046561581584635414", - "tolerated_val_decision_score": "0.7238142274110074", - "tolerated_val_gap": "0.032313773339499696", - "tolerated_val_ranking_score": "0.6915004540715077", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7653346239087763", - "union_ranking_score": "0.7187730423241409", - "union_regret": 0.046561581584635414, - "union_test_gap": "0.046561581584635414", - "union_val_decision_score": "0.7238142274110074", - "union_val_gap": "0.032313773339499696", - "union_val_ranking_score": "0.6915004540715077", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0009319664492078285", - "exact_ranking_score": "0.0009319664492078285", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.004399472063352399", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.004399472063352399", - "family": "FCN", - "model_tag": "fcn", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.00037256097806901117", - "ranking_selected_val_pr_auc": "0.00045069184813668276", - "scope": "global", - "seed": 1, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.31167484413093016", - "tolerated_ranking_score": "0.31167484413093016", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.3052875742065006", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.3052875742065006", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.31167484413093016", - "union_ranking_score": "0.31167484413093016", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.3052875742065006", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.3052875742065006", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.042068766252528166", - "exact_ranking_score": "0.042068766252528166", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.043150583715087926", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.043150583715087926", - "family": "FCN", - "model_tag": "fcn", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.01844011667219625", - "ranking_selected_val_pr_auc": "0.019762912264716534", - "scope": "fire_prone", - "seed": 1, - "tolerated_decision_head": "linear probe", - "tolerated_decision_head_arch": "linear", - "tolerated_decision_score": "0.7182175622542595", - "tolerated_ranking_score": "0.7185324707231184", - "tolerated_regret": 0.0, - "tolerated_test_gap": "-0.0003149084688589543", - "tolerated_val_decision_score": "0.6906967717983766", - "tolerated_val_gap": "2.135845076300935e-05", - "tolerated_val_ranking_score": "0.6906754133476136", - "union_decision_head": "linear probe", - "union_decision_head_arch": "linear", - "union_decision_score": "0.7182175622542595", - "union_ranking_score": "0.7185324707231184", - "union_regret": 0.0, - "union_test_gap": "-0.0003149084688589543", - "union_val_decision_score": "0.6906967717983766", - "union_val_gap": "2.135845076300935e-05", - "union_val_ranking_score": "0.6906754133476136", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0006535947712418301", - "exact_ranking_score": "0.0006535947712418301", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.0037359900373599006", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.0037359900373599006", - "family": "FCN", - "model_tag": "fcn", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.000387412312932838", - "ranking_selected_val_pr_auc": "0.0004364526611974742", - "scope": "global", - "seed": 123, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.3096850885545486", - "tolerated_ranking_score": "0.3096850885545486", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.2992790399255972", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.2992790399255972", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.3096850885545486", - "union_ranking_score": "0.3096850885545486", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.2992790399255972", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.2992790399255972", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.04104234527687296", - "exact_ranking_score": "0.04101254412979794", - "exact_regret": 2.980114707502174e-05, - "exact_test_gap": "2.980114707502174e-05", - "exact_val_decision_score": "0.04257237303415807", - "exact_val_gap": "4.047410996493167e-05", - "exact_val_ranking_score": "0.04253189892419314", - "family": "FCN", - "model_tag": "fcn", - "ranking_selected_head": "pixel MLP head", - "ranking_selected_head_arch": "pixel_mlp", - "ranking_selected_test_pr_auc": "0.019943278646881796", - "ranking_selected_val_pr_auc": "0.0240285163314118", - "scope": "fire_prone", - "seed": 123, - "tolerated_decision_head": "linear probe", - "tolerated_decision_head_arch": "linear", - "tolerated_decision_score": "0.7310509974227326", - "tolerated_ranking_score": "0.7185324707231184", - "tolerated_regret": 0.012518526699614174, - "tolerated_test_gap": "0.012518526699614174", - "tolerated_val_decision_score": "0.7058203125", - "tolerated_val_gap": "0.01514489915238637", - "tolerated_val_ranking_score": "0.6906754133476136", - "union_decision_head": "linear probe", - "union_decision_head_arch": "linear", - "union_decision_score": "0.7310509974227326", - "union_ranking_score": "0.7185324707231184", - "union_regret": 0.012518526699614174, - "union_test_gap": "0.012518526699614174", - "union_val_decision_score": "0.7058203125", - "union_val_gap": "0.01514489915238637", - "union_val_ranking_score": "0.6906754133476136", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0007027406886858749", - "exact_ranking_score": "0.0007027406886858749", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.0031868882312770315", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.0031868882312770315", - "family": "FCN", - "model_tag": "fcn", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.00041063897933390575", - "ranking_selected_val_pr_auc": "0.00042489429428535383", - "scope": "global", - "seed": 42, - "tolerated_decision_head": "pixel MLP head", - "tolerated_decision_head_arch": "pixel_mlp", - "tolerated_decision_score": "0.2870596305028149", - "tolerated_ranking_score": "0.31987973649439366", - "tolerated_regret": 0.0, - "tolerated_test_gap": "-0.03282010599157875", - "tolerated_val_decision_score": "0.31159367647460456", - "tolerated_val_gap": "0.0054887421624861266", - "tolerated_val_ranking_score": "0.30610493431211844", - "union_decision_head": "pixel MLP head", - "union_decision_head_arch": "pixel_mlp", - "union_decision_score": "0.2870596305028149", - "union_ranking_score": "0.31987973649439366", - "union_regret": 0.0, - "union_test_gap": "-0.03282010599157875", - "union_val_decision_score": "0.31159367647460456", - "union_val_gap": "0.0054887421624861266", - "union_val_ranking_score": "0.30610493431211844", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0414596444738876", - "exact_ranking_score": "0.0414596444738876", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.04356664102845841", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.04356664102845841", - "family": "FCN", - "model_tag": "fcn", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.018030815062946615", - "ranking_selected_val_pr_auc": "0.018978807537809048", - "scope": "fire_prone", - "seed": 42, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7197180735022655", - "tolerated_ranking_score": "0.7197180735022655", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.6914092197111065", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.6914092197111065", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7197180735022655", - "union_ranking_score": "0.7197180735022655", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.6914092197111065", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.6914092197111065", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.001086071137659517", - "exact_ranking_score": "0.001086071137659517", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.003349460364719018", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.003349460364719018", - "family": "FCN", - "model_tag": "fcn", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.0003268363416054406", - "ranking_selected_val_pr_auc": "0.00039156287297214436", - "scope": "global", - "seed": 7, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.3051941376005135", - "tolerated_ranking_score": "0.3051941376005135", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.2948179787588662", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.2948179787588662", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.3051941376005135", - "union_ranking_score": "0.3051941376005135", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.2948179787588662", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.2948179787588662", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.040288313241635454", - "exact_ranking_score": "0.04101254412979794", - "exact_regret": 0.0, - "exact_test_gap": "-0.0007242308881624873", - "exact_val_decision_score": "0.04306006752022422", - "exact_val_gap": "0.0005281685960310811", - "exact_val_ranking_score": "0.04253189892419314", - "family": "FCN", - "model_tag": "fcn", - "ranking_selected_head": "pixel MLP head", - "ranking_selected_head_arch": "pixel_mlp", - "ranking_selected_test_pr_auc": "0.02050876208409485", - "ranking_selected_val_pr_auc": "0.02590116599560572", - "scope": "fire_prone", - "seed": 7, - "tolerated_decision_head": "linear probe", - "tolerated_decision_head_arch": "linear", - "tolerated_decision_score": "0.7644129739607127", - "tolerated_ranking_score": "0.7185324707231184", - "tolerated_regret": 0.045880503237594294, - "tolerated_test_gap": "0.045880503237594294", - "tolerated_val_decision_score": "0.6962583126352054", - "tolerated_val_gap": "0.005582899287591814", - "tolerated_val_ranking_score": "0.6906754133476136", - "union_decision_head": "linear probe", - "union_decision_head_arch": "linear", - "union_decision_score": "0.7644129739607127", - "union_ranking_score": "0.7185324707231184", - "union_regret": 0.045880503237594294, - "union_test_gap": "0.045880503237594294", - "union_val_decision_score": "0.6962583126352054", - "union_val_gap": "0.005582899287591814", - "union_val_ranking_score": "0.6906754133476136", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0018159806295399514", - "exact_ranking_score": "0.0018159806295399514", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.003241491085899514", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.003241491085899514", - "family": "FCN", - "model_tag": "fcn", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.00038120453362995967", - "ranking_selected_val_pr_auc": "0.0004356191579614426", - "scope": "global", - "seed": 99, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.3054145960271247", - "tolerated_ranking_score": "0.3054145960271247", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.2930313844339525", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.2930313844339525", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.3054145960271247", - "union_ranking_score": "0.3054145960271247", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.2930313844339525", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.2930313844339525", - "path": "" - }, - { - "exact_decision_head": "linear probe", - "exact_decision_head_arch": "linear", - "exact_decision_score": "0.042822140550172624", - "exact_ranking_score": "0.042822140550172624", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.043420983295568316", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.043420983295568316", - "family": "FCN", - "model_tag": "fcn", - "ranking_selected_head": "linear probe", - "ranking_selected_head_arch": "linear", - "ranking_selected_test_pr_auc": "0.029098665712304895", - "ranking_selected_val_pr_auc": "0.032221638150467385", - "scope": "fire_prone", - "seed": 99, - "tolerated_decision_head": "linear probe", - "tolerated_decision_head_arch": "linear", - "tolerated_decision_score": "0.726408418760773", - "tolerated_ranking_score": "0.726408418760773", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.6933831038489982", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.6933831038489982", - "union_decision_head": "linear probe", - "union_decision_head_arch": "linear", - "union_decision_score": "0.726408418760773", - "union_ranking_score": "0.726408418760773", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.6933831038489982", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.6933831038489982", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0", - "exact_ranking_score": "0.0", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.0031948881789137383", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.0031948881789137383", - "family": "FengWu", - "model_tag": "fengwu", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.000398077435365184", - "ranking_selected_val_pr_auc": "0.00047223676755013246", - "scope": "global", - "seed": 1, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.31071390711162444", - "tolerated_ranking_score": "0.31071390711162444", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.3130246199713141", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.3130246199713141", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.31071390711162444", - "union_ranking_score": "0.31071390711162444", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.3130246199713141", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.3130246199713141", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.04452825597664091", - "exact_ranking_score": "0.04452825597664091", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.0455414882958375", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.0455414882958375", - "family": "FengWu", - "model_tag": "fengwu", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.022736128443885992", - "ranking_selected_val_pr_auc": "0.021286665509494304", - "scope": "fire_prone", - "seed": 1, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7269980510116651", - "tolerated_ranking_score": "0.7269980510116651", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.7088464801269441", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.7088464801269441", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7269980510116651", - "union_ranking_score": "0.7269980510116651", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.7088464801269441", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.7088464801269441", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0", - "exact_ranking_score": "0.0", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.003656307129798903", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.003656307129798903", - "family": "FengWu", - "model_tag": "fengwu", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.0004116035724473925", - "ranking_selected_val_pr_auc": "0.0004377390722357372", - "scope": "global", - "seed": 123, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.3145618361221859", - "tolerated_ranking_score": "0.3145618361221859", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.3129112248106385", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.3129112248106385", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.3145618361221859", - "union_ranking_score": "0.3145618361221859", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.3129112248106385", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.3129112248106385", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.04171607200603644", - "exact_ranking_score": "0.04101254412979794", - "exact_regret": 0.0007035278762385008, - "exact_test_gap": "0.0007035278762385008", - "exact_val_decision_score": "0.043619564341543335", - "exact_val_gap": "0.0010876654173501976", - "exact_val_ranking_score": "0.04253189892419314", - "family": "FengWu", - "model_tag": "fengwu", - "ranking_selected_head": "pixel MLP head", - "ranking_selected_head_arch": "pixel_mlp", - "ranking_selected_test_pr_auc": "0.020495144103834257", - "ranking_selected_val_pr_auc": "0.023246350918909517", - "scope": "fire_prone", - "seed": 123, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7251252524331628", - "tolerated_ranking_score": "0.7185324707231184", - "tolerated_regret": 0.006592781710044404, - "tolerated_test_gap": "0.006592781710044404", - "tolerated_val_decision_score": "0.7015002586652871", - "tolerated_val_gap": "0.01082484531767347", - "tolerated_val_ranking_score": "0.6906754133476136", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7251252524331628", - "union_ranking_score": "0.7185324707231184", - "union_regret": 0.006592781710044404, - "union_test_gap": "0.006592781710044404", - "union_val_decision_score": "0.7015002586652871", - "union_val_gap": "0.01082484531767347", - "union_val_ranking_score": "0.6906754133476136", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0", - "exact_ranking_score": "0.0", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.001528350909368791", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.001528350909368791", - "family": "FengWu", - "model_tag": "fengwu", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.00036987379624400454", - "ranking_selected_val_pr_auc": "0.00039852867365630293", - "scope": "global", - "seed": 42, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.31219058559732665", - "tolerated_ranking_score": "0.31219058559732665", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.31476872634310843", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.31476872634310843", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.31219058559732665", - "union_ranking_score": "0.31219058559732665", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.31476872634310843", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.31476872634310843", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.04574838388861263", - "exact_ranking_score": "0.04574838388861263", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.04613821138211382", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.04613821138211382", - "family": "FengWu", - "model_tag": "fengwu", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.021165185967854567", - "ranking_selected_val_pr_auc": "0.020671268187642432", - "scope": "fire_prone", - "seed": 42, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7265705731122933", - "tolerated_ranking_score": "0.7265705731122933", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.7074814597750377", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.7074814597750377", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7265705731122933", - "union_ranking_score": "0.7265705731122933", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.7074814597750377", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.7074814597750377", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0", - "exact_ranking_score": "0.0", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.0027012425715829285", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.0027012425715829285", - "family": "FengWu", - "model_tag": "fengwu", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.00036963872610239895", - "ranking_selected_val_pr_auc": "0.00042054736055892957", - "scope": "global", - "seed": 7, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.30641904273669174", - "tolerated_ranking_score": "0.30641904273669174", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.30868675618673724", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.30868675618673724", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.30641904273669174", - "union_ranking_score": "0.30641904273669174", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.30868675618673724", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.30868675618673724", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.043763127370763974", - "exact_ranking_score": "0.04101254412979794", - "exact_regret": 0.0027505832409660327, - "exact_test_gap": "0.0027505832409660327", - "exact_val_decision_score": "0.04404735872043984", - "exact_val_gap": "0.0015154597962467042", - "exact_val_ranking_score": "0.04253189892419314", - "family": "FengWu", - "model_tag": "fengwu", - "ranking_selected_head": "pixel MLP head", - "ranking_selected_head_arch": "pixel_mlp", - "ranking_selected_test_pr_auc": "0.016801706970978273", - "ranking_selected_val_pr_auc": "0.022509307119807316", - "scope": "fire_prone", - "seed": 7, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.722901134194411", - "tolerated_ranking_score": "0.7185324707231184", - "tolerated_regret": 0.004368663471292611, - "tolerated_test_gap": "0.004368663471292611", - "tolerated_val_decision_score": "0.7033268834654516", - "tolerated_val_gap": "0.01265147011783796", - "tolerated_val_ranking_score": "0.6906754133476136", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.722901134194411", - "union_ranking_score": "0.7185324707231184", - "union_regret": 0.004368663471292611, - "union_test_gap": "0.004368663471292611", - "union_val_decision_score": "0.7033268834654516", - "union_val_gap": "0.01265147011783796", - "union_val_ranking_score": "0.6906754133476136", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0", - "exact_ranking_score": "0.0", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.0041356492969396195", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.0041356492969396195", - "family": "FengWu", - "model_tag": "fengwu", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.00042782651526874734", - "ranking_selected_val_pr_auc": "0.0004271990954031648", - "scope": "global", - "seed": 99, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.3111214819309595", - "tolerated_ranking_score": "0.3111214819309595", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.30674980181224504", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.30674980181224504", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.3111214819309595", - "union_ranking_score": "0.3111214819309595", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.30674980181224504", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.30674980181224504", - "path": "" - }, - { - "exact_decision_head": "linear probe", - "exact_decision_head_arch": "linear", - "exact_decision_score": "0.0", - "exact_ranking_score": "0.0", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.04882017900732303", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.04882017900732303", - "family": "FengWu", - "model_tag": "fengwu", - "ranking_selected_head": "linear probe", - "ranking_selected_head_arch": "linear", - "ranking_selected_test_pr_auc": "0.02199779031689959", - "ranking_selected_val_pr_auc": "0.030768274290330223", - "scope": "fire_prone", - "seed": 99, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7336829717908426", - "tolerated_ranking_score": "0.7185324707231184", - "tolerated_regret": 0.015150501067724198, - "tolerated_test_gap": "0.015150501067724198", - "tolerated_val_decision_score": "0.6987796297897932", - "tolerated_val_gap": "0.008104216442179624", - "tolerated_val_ranking_score": "0.6906754133476136", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7336829717908426", - "union_ranking_score": "0.7185324707231184", - "union_regret": 0.015150501067724198, - "union_test_gap": "0.015150501067724198", - "union_val_decision_score": "0.6987796297897932", - "union_val_gap": "0.008104216442179624", - "union_val_ranking_score": "0.6906754133476136", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0006866311182961118", - "exact_ranking_score": "0.0006866311182961118", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.0008811590083408097", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.0008811590083408097", - "family": "FuXi", - "model_tag": "fuxi", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.0003596048414560045", - "ranking_selected_val_pr_auc": "0.0004121169333774252", - "scope": "global", - "seed": 1, - "tolerated_decision_head": "linear probe", - "tolerated_decision_head_arch": "linear", - "tolerated_decision_score": "0.21516044416153002", - "tolerated_ranking_score": "0.3091927111996169", - "tolerated_regret": 0.0, - "tolerated_test_gap": "-0.09403226703808687", - "tolerated_val_decision_score": "0.3118144595732139", - "tolerated_val_gap": "0.001671993688703799", - "tolerated_val_ranking_score": "0.3101424658845101", - "union_decision_head": "linear probe", - "union_decision_head_arch": "linear", - "union_decision_score": "0.21516044416153002", - "union_ranking_score": "0.3091927111996169", - "union_regret": 0.0, - "union_test_gap": "-0.09403226703808687", - "union_val_decision_score": "0.3118144595732139", - "union_val_gap": "0.001671993688703799", - "union_val_ranking_score": "0.3101424658845101", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.043196160341303", - "exact_ranking_score": "0.043196160341303", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.04576053854784421", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.04576053854784421", - "family": "FuXi", - "model_tag": "fuxi", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.01904942261850253", - "ranking_selected_val_pr_auc": "0.020787588115504", - "scope": "fire_prone", - "seed": 1, - "tolerated_decision_head": "linear probe", - "tolerated_decision_head_arch": "linear", - "tolerated_decision_score": "0.7261195534617713", - "tolerated_ranking_score": "0.7246636456247585", - "tolerated_regret": 0.001455907837012771, - "tolerated_test_gap": "0.001455907837012771", - "tolerated_val_decision_score": "0.7033365957825429", - "tolerated_val_gap": "0.003706992036648815", - "tolerated_val_ranking_score": "0.6996296037458941", - "union_decision_head": "linear probe", - "union_decision_head_arch": "linear", - "union_decision_score": "0.7261195534617713", - "union_ranking_score": "0.7246636456247585", - "union_regret": 0.001455907837012771, - "union_test_gap": "0.001455907837012771", - "union_val_decision_score": "0.7033365957825429", - "union_val_gap": "0.003706992036648815", - "union_val_ranking_score": "0.6996296037458941", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0009888839378897987", - "exact_ranking_score": "0.0009888839378897987", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.0010117798317730948", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.0010117798317730948", - "family": "FuXi", - "model_tag": "fuxi", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.00039999784430104664", - "ranking_selected_val_pr_auc": "0.00043073606570656576", - "scope": "global", - "seed": 123, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.3096924811079745", - "tolerated_ranking_score": "0.3096924811079745", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.31039074285584656", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.31039074285584656", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.3096924811079745", - "union_ranking_score": "0.3096924811079745", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.31039074285584656", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.31039074285584656", - "path": "" - }, - { - "exact_decision_head": "pixel MLP head", - "exact_decision_head_arch": "pixel_mlp", - "exact_decision_score": "0.018543768748295608", - "exact_ranking_score": "0.018543768748295608", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.04600811907983762", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.04600811907983762", - "family": "FuXi", - "model_tag": "fuxi", - "ranking_selected_head": "pixel MLP head", - "ranking_selected_head_arch": "pixel_mlp", - "ranking_selected_test_pr_auc": "0.021045021724179047", - "ranking_selected_val_pr_auc": "0.025380726482200633", - "scope": "fire_prone", - "seed": 123, - "tolerated_decision_head": "linear probe", - "tolerated_decision_head_arch": "linear", - "tolerated_decision_score": "0.7224961571477053", - "tolerated_ranking_score": "0.7185324707231184", - "tolerated_regret": 0.0039636864245868875, - "tolerated_test_gap": "0.0039636864245868875", - "tolerated_val_decision_score": "0.6975587357465372", - "tolerated_val_gap": "0.00688332239892353", - "tolerated_val_ranking_score": "0.6906754133476136", - "union_decision_head": "linear probe", - "union_decision_head_arch": "linear", - "union_decision_score": "0.7224961571477053", - "union_ranking_score": "0.7185324707231184", - "union_regret": 0.0039636864245868875, - "union_test_gap": "0.0039636864245868875", - "union_val_decision_score": "0.6975587357465372", - "union_val_gap": "0.00688332239892353", - "union_val_ranking_score": "0.6906754133476136", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0003188165529554294", - "exact_ranking_score": "0.0003188165529554294", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.0011176098051633573", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.0011176098051633573", - "family": "FuXi", - "model_tag": "fuxi", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.0003830459807690152", - "ranking_selected_val_pr_auc": "0.00042901589997107405", - "scope": "global", - "seed": 42, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.316668570748256", - "tolerated_ranking_score": "0.316668570748256", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.3180456774666437", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.3180456774666437", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.316668570748256", - "union_ranking_score": "0.316668570748256", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.3180456774666437", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.3180456774666437", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.04603830266616599", - "exact_ranking_score": "0.04603830266616599", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.04428715218026559", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.04428715218026559", - "family": "FuXi", - "model_tag": "fuxi", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.020363596550511454", - "ranking_selected_val_pr_auc": "0.020674507603092498", - "scope": "fire_prone", - "seed": 42, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7054843984273774", - "tolerated_ranking_score": "0.7054843984273774", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.72648949860566", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.72648949860566", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7054843984273774", - "union_ranking_score": "0.7054843984273774", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.72648949860566", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.72648949860566", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0008450613619815959", - "exact_ranking_score": "0.0008450613619815959", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.0009152735719220856", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.0009152735719220856", - "family": "FuXi", - "model_tag": "fuxi", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.00037118462783325537", - "ranking_selected_val_pr_auc": "0.0004157684867215522", - "scope": "global", - "seed": 7, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.3062654921252336", - "tolerated_ranking_score": "0.3062654921252336", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.30677382687191324", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.30677382687191324", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.3062654921252336", - "union_ranking_score": "0.3062654921252336", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.30677382687191324", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.30677382687191324", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.044572342126298965", - "exact_ranking_score": "0.044572342126298965", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.04500911929271383", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.04500911929271383", - "family": "FuXi", - "model_tag": "fuxi", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.018717347905238046", - "ranking_selected_val_pr_auc": "0.020626878606552698", - "scope": "fire_prone", - "seed": 7, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7235687421646468", - "tolerated_ranking_score": "0.7235687421646468", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.6980281690140845", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.6980281690140845", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7235687421646468", - "union_ranking_score": "0.7235687421646468", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.6980281690140845", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.6980281690140845", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0007771032641515431", - "exact_ranking_score": "0.0007771032641515431", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.0008510302291201489", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.0008510302291201489", - "family": "FuXi", - "model_tag": "fuxi", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.0003758107890486513", - "ranking_selected_val_pr_auc": "0.0004065107971511574", - "scope": "global", - "seed": 99, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.3108110703043797", - "tolerated_ranking_score": "0.3108110703043797", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.3116979187141017", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.3116979187141017", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.3108110703043797", - "union_ranking_score": "0.3108110703043797", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.3116979187141017", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.3116979187141017", - "path": "" - }, - { - "exact_decision_head": "pixel MLP head", - "exact_decision_head_arch": "pixel_mlp", - "exact_decision_score": "0.030284377692970574", - "exact_ranking_score": "0.030284377692970574", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.051651931130758486", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.051651931130758486", - "family": "FuXi", - "model_tag": "fuxi", - "ranking_selected_head": "pixel MLP head", - "ranking_selected_head_arch": "pixel_mlp", - "ranking_selected_test_pr_auc": "0.02225455497934009", - "ranking_selected_val_pr_auc": "0.02778239852030042", - "scope": "fire_prone", - "seed": 99, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7183426482806338", - "tolerated_ranking_score": "0.7203102915557309", - "tolerated_regret": 0.0, - "tolerated_test_gap": "-0.0019676432750971307", - "tolerated_val_decision_score": "0.7124343844122053", - "tolerated_val_gap": "0.02087441927176159", - "tolerated_val_ranking_score": "0.6915599651404437", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7183426482806338", - "union_ranking_score": "0.7203102915557309", - "union_regret": 0.0, - "union_test_gap": "-0.0019676432750971307", - "union_val_decision_score": "0.7124343844122053", - "union_val_gap": "0.02087441927176159", - "union_val_ranking_score": "0.6915599651404437", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0004925501476206198", - "exact_ranking_score": "0.00048497822606044473", - "exact_regret": 7.571921560175121e-06, - "exact_test_gap": "7.571921560175121e-06", - "exact_val_decision_score": "0.0004727121748000204", - "exact_val_gap": "7.267025482891767e-06", - "exact_val_ranking_score": "0.00046544514931712863", - "family": "Pangu-Weather", - "model_tag": "pangu_weather", - "ranking_selected_head": "linear probe", - "ranking_selected_head_arch": "linear", - "ranking_selected_test_pr_auc": "0.00024254792826221397", - "ranking_selected_val_pr_auc": "0.00023277674706246134", - "scope": "global", - "seed": 1, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.240793572992212", - "tolerated_ranking_score": "0.23755358049655212", - "tolerated_regret": 0.0032399924956598714, - "tolerated_test_gap": "0.0032399924956598714", - "tolerated_val_decision_score": "0.2179368807313983", - "tolerated_val_gap": "0.00299097714352789", - "tolerated_val_ranking_score": "0.2149459035878704", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.240793572992212", - "union_ranking_score": "0.23755358049655212", - "union_regret": 0.0032399924956598714, - "union_test_gap": "0.0032399924956598714", - "union_val_decision_score": "0.2179368807313983", - "union_val_gap": "0.00299097714352789", - "union_val_ranking_score": "0.2149459035878704", - "path": "" - }, - { - "exact_decision_head": "pixel MLP head", - "exact_decision_head_arch": "pixel_mlp", - "exact_decision_score": "0.04101254412979794", - "exact_ranking_score": "0.04101254412979794", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.04253189892419314", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.04253189892419314", - "family": "Pangu-Weather", - "model_tag": "pangu_weather", - "ranking_selected_head": "pixel MLP head", - "ranking_selected_head_arch": "pixel_mlp", - "ranking_selected_test_pr_auc": "0.014813375233921266", - "ranking_selected_val_pr_auc": "0.02263545197493654", - "scope": "fire_prone", - "seed": 1, - "tolerated_decision_head": "pixel MLP head", - "tolerated_decision_head_arch": "pixel_mlp", - "tolerated_decision_score": "0.7185324707231184", - "tolerated_ranking_score": "0.7185324707231184", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.6906754133476136", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.6906754133476136", - "union_decision_head": "pixel MLP head", - "union_decision_head_arch": "pixel_mlp", - "union_decision_score": "0.7185324707231184", - "union_ranking_score": "0.7185324707231184", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.6906754133476136", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.6906754133476136", - "path": "" - }, - { - "exact_decision_head": "pixel MLP head", - "exact_decision_head_arch": "pixel_mlp", - "exact_decision_score": "0.0005266170481980583", - "exact_ranking_score": "0.0004925501476206198", - "exact_regret": 3.406690057743849e-05, - "exact_test_gap": "3.406690057743849e-05", - "exact_val_decision_score": "0.00047637431559568154", - "exact_val_gap": "3.6621407956611337e-06", - "exact_val_ranking_score": "0.0004727121748000204", - "family": "Pangu-Weather", - "model_tag": "pangu_weather", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.0003300754798792237", - "ranking_selected_val_pr_auc": "0.00024271959698351982", - "scope": "global", - "seed": 123, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.240793572992212", - "tolerated_ranking_score": "0.240793572992212", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.2179368807313983", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.2179368807313983", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.240793572992212", - "union_ranking_score": "0.240793572992212", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.2179368807313983", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.2179368807313983", - "path": "" - }, - { - "exact_decision_head": "pixel MLP head", - "exact_decision_head_arch": "pixel_mlp", - "exact_decision_score": "0.045738515901060074", - "exact_ranking_score": "0.04101254412979794", - "exact_regret": 0.004725971771262133, - "exact_test_gap": "0.004725971771262133", - "exact_val_decision_score": "0.04344902498318937", - "exact_val_gap": "0.0009171260589962357", - "exact_val_ranking_score": "0.04253189892419314", - "family": "Pangu-Weather", - "model_tag": "pangu_weather", - "ranking_selected_head": "linear probe", - "ranking_selected_head_arch": "linear", - "ranking_selected_test_pr_auc": "0.02093558282208589", - "ranking_selected_val_pr_auc": "0.0217280163599182", - "scope": "fire_prone", - "seed": 123, - "tolerated_decision_head": "linear probe", - "tolerated_decision_head_arch": "linear", - "tolerated_decision_score": "0.7185324707231184", - "tolerated_ranking_score": "0.7185324707231184", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.6906754133476136", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.6906754133476136", - "union_decision_head": "linear probe", - "union_decision_head_arch": "linear", - "union_decision_score": "0.7185324707231184", - "union_ranking_score": "0.7185324707231184", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.6906754133476136", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.6906754133476136", - "path": "" - }, - { - "exact_decision_head": "linear probe", - "exact_decision_head_arch": "linear", - "exact_decision_score": "0.0006100601402256502", - "exact_ranking_score": "0.0", - "exact_regret": 0.0006100601402256502, - "exact_test_gap": "0.0006100601402256502", - "exact_val_decision_score": "0.00048444832931275107", - "exact_val_gap": "0.00048444832931275107", - "exact_val_ranking_score": "0.0", - "family": "Pangu-Weather", - "model_tag": "pangu_weather", - "ranking_selected_head": "pixel MLP head", - "ranking_selected_head_arch": "pixel_mlp", - "ranking_selected_test_pr_auc": "0.00021817709863716954", - "ranking_selected_val_pr_auc": "0.00025116618856577995", - "scope": "global", - "seed": 42, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.240793572992212", - "tolerated_ranking_score": "0.0", - "tolerated_regret": 0.240793572992212, - "tolerated_test_gap": "0.240793572992212", - "tolerated_val_decision_score": "0.2179368807313983", - "tolerated_val_gap": "0.2179368807313983", - "tolerated_val_ranking_score": "0.0", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.240793572992212", - "union_ranking_score": "0.0", - "union_regret": 0.240793572992212, - "union_test_gap": "0.240793572992212", - "union_val_decision_score": "0.2179368807313983", - "union_val_gap": "0.2179368807313983", - "union_val_ranking_score": "0.0", - "path": "" - }, - { - "exact_decision_head": "linear probe", - "exact_decision_head_arch": "linear", - "exact_decision_score": "0.04813290893564165", - "exact_ranking_score": "0.0", - "exact_regret": 0.04813290893564165, - "exact_test_gap": "0.04813290893564165", - "exact_val_decision_score": "0.04370736036385822", - "exact_val_gap": "0.04370736036385822", - "exact_val_ranking_score": "0.0", - "family": "Pangu-Weather", - "model_tag": "pangu_weather", - "ranking_selected_head": "pixel MLP head", - "ranking_selected_head_arch": "pixel_mlp", - "ranking_selected_test_pr_auc": "0.02188826028256454", - "ranking_selected_val_pr_auc": "0.032209470774519654", - "scope": "fire_prone", - "seed": 42, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7185324707231184", - "tolerated_ranking_score": "0.0", - "tolerated_regret": 0.7185324707231184, - "tolerated_test_gap": "0.7185324707231184", - "tolerated_val_decision_score": "0.6906754133476136", - "tolerated_val_gap": "0.6906754133476136", - "tolerated_val_ranking_score": "0.0", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7185324707231184", - "union_ranking_score": "0.0", - "union_regret": 0.7185324707231184, - "union_test_gap": "0.7185324707231184", - "union_val_decision_score": "0.6906754133476136", - "union_val_gap": "0.6906754133476136", - "union_val_ranking_score": "0.0", - "path": "" - }, - { - "exact_decision_head": "linear probe", - "exact_decision_head_arch": "linear", - "exact_decision_score": "0.00035527516330545857", - "exact_ranking_score": "0.0004925501476206198", - "exact_regret": 0.0, - "exact_test_gap": "-0.00013727498431516128", - "exact_val_decision_score": "0.0005024040170237669", - "exact_val_gap": "2.9691842223746458e-05", - "exact_val_ranking_score": "0.0004727121748000204", - "family": "Pangu-Weather", - "model_tag": "pangu_weather", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.00028045576992618193", - "ranking_selected_val_pr_auc": "0.00024387937757633073", - "scope": "global", - "seed": 7, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.240793572992212", - "tolerated_ranking_score": "0.240793572992212", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.2179368807313983", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.2179368807313983", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.240793572992212", - "union_ranking_score": "0.240793572992212", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.2179368807313983", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.2179368807313983", - "path": "" - }, - { - "exact_decision_head": "linear probe", - "exact_decision_head_arch": "linear", - "exact_decision_score": "0.04550550682582603", - "exact_ranking_score": "0.0", - "exact_regret": 0.04550550682582603, - "exact_test_gap": "0.04550550682582603", - "exact_val_decision_score": "0.04319677350880917", - "exact_val_gap": "0.04319677350880917", - "exact_val_ranking_score": "0.0", - "family": "Pangu-Weather", - "model_tag": "pangu_weather", - "ranking_selected_head": "pixel MLP head", - "ranking_selected_head_arch": "pixel_mlp", - "ranking_selected_test_pr_auc": "0.029883397445312192", - "ranking_selected_val_pr_auc": "0.02789152422119005", - "scope": "fire_prone", - "seed": 7, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7185324707231184", - "tolerated_ranking_score": "0.0", - "tolerated_regret": 0.7185324707231184, - "tolerated_test_gap": "0.7185324707231184", - "tolerated_val_decision_score": "0.6906754133476136", - "tolerated_val_gap": "0.6906754133476136", - "tolerated_val_ranking_score": "0.0", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7185324707231184", - "union_ranking_score": "0.0", - "union_regret": 0.7185324707231184, - "union_test_gap": "0.7185324707231184", - "union_val_decision_score": "0.6906754133476136", - "union_val_gap": "0.6906754133476136", - "union_val_ranking_score": "0.0", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0004887421693148924", - "exact_ranking_score": "0.0004887421693148924", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.0004690575308283202", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.0004690575308283202", - "family": "Pangu-Weather", - "model_tag": "pangu_weather", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.0003408299850116487", - "ranking_selected_val_pr_auc": "0.00024250312441493891", - "scope": "global", - "seed": 99, - "tolerated_decision_head": "pixel MLP head", - "tolerated_decision_head_arch": "pixel_mlp", - "tolerated_decision_score": "0.229804399271568", - "tolerated_ranking_score": "0.23917716245227552", - "tolerated_regret": 0.0, - "tolerated_test_gap": "-0.00937276318070751", - "tolerated_val_decision_score": "0.2256312133013075", - "tolerated_val_gap": "0.009197267184311658", - "tolerated_val_ranking_score": "0.21643394611699585", - "union_decision_head": "pixel MLP head", - "union_decision_head_arch": "pixel_mlp", - "union_decision_score": "0.229804399271568", - "union_ranking_score": "0.23917716245227552", - "union_regret": 0.0, - "union_test_gap": "-0.00937276318070751", - "union_val_decision_score": "0.2256312133013075", - "union_val_gap": "0.009197267184311658", - "union_val_ranking_score": "0.21643394611699585", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.04101254412979794", - "exact_ranking_score": "0.0", - "exact_regret": 0.04101254412979794, - "exact_test_gap": "0.04101254412979794", - "exact_val_decision_score": "0.04253189892419314", - "exact_val_gap": "0.04253189892419314", - "exact_val_ranking_score": "0.0", - "family": "Pangu-Weather", - "model_tag": "pangu_weather", - "ranking_selected_head": "linear probe", - "ranking_selected_head_arch": "linear", - "ranking_selected_test_pr_auc": "0.02093558282208589", - "ranking_selected_val_pr_auc": "0.0217280163599182", - "scope": "fire_prone", - "seed": 99, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7185324707231184", - "tolerated_ranking_score": "0.0", - "tolerated_regret": 0.7185324707231184, - "tolerated_test_gap": "0.7185324707231184", - "tolerated_val_decision_score": "0.6906754133476136", - "tolerated_val_gap": "0.6906754133476136", - "tolerated_val_ranking_score": "0.0", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7185324707231184", - "union_ranking_score": "0.0", - "union_regret": 0.7185324707231184, - "union_test_gap": "0.7185324707231184", - "union_val_decision_score": "0.6906754133476136", - "union_val_gap": "0.6906754133476136", - "union_val_ranking_score": "0.0", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0", - "exact_ranking_score": "0.0", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.004253509145044662", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.004253509145044662", - "family": "Pangu-Weather", - "model_tag": "pangu6", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.00042881385578730365", - "ranking_selected_val_pr_auc": "0.0004901027948814223", - "scope": "global", - "seed": 1, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.32066974948800614", - "tolerated_ranking_score": "0.32066974948800614", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.3120952399779768", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.3120952399779768", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.32066974948800614", - "union_ranking_score": "0.32066974948800614", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.3120952399779768", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.3120952399779768", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.04418397717161179", - "exact_ranking_score": "0.04418397717161179", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.04535556523362161", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.04535556523362161", - "family": "Pangu-Weather", - "model_tag": "pangu6", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.02328829578760066", - "ranking_selected_val_pr_auc": "0.02010310006243177", - "scope": "fire_prone", - "seed": 1, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7231388400090598", - "tolerated_ranking_score": "0.7231388400090598", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.698733466652633", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.698733466652633", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7231388400090598", - "union_ranking_score": "0.7231388400090598", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.698733466652633", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.698733466652633", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0", - "exact_ranking_score": "0.0", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.0042863266180882984", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.0042863266180882984", - "family": "Pangu-Weather", - "model_tag": "pangu6", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.00043738577276574255", - "ranking_selected_val_pr_auc": "0.0004554894460132563", - "scope": "global", - "seed": 123, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.31812983009681495", - "tolerated_ranking_score": "0.31812983009681495", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.3113263265858816", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.3113263265858816", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.31812983009681495", - "union_ranking_score": "0.31812983009681495", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.3113263265858816", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.3113263265858816", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.04194314737445728", - "exact_ranking_score": "0.04101254412979794", - "exact_regret": 0.0009306032446593365, - "exact_test_gap": "0.0009306032446593365", - "exact_val_decision_score": "0.04314380355334118", - "exact_val_gap": "0.0006119046291480426", - "exact_val_ranking_score": "0.04253189892419314", - "family": "Pangu-Weather", - "model_tag": "pangu6", - "ranking_selected_head": "pixel MLP head", - "ranking_selected_head_arch": "pixel_mlp", - "ranking_selected_test_pr_auc": "0.02254610440161657", - "ranking_selected_val_pr_auc": "0.026053751384712866", - "scope": "fire_prone", - "seed": 123, - "tolerated_decision_head": "linear probe", - "tolerated_decision_head_arch": "linear", - "tolerated_decision_score": "0.7261721555350364", - "tolerated_ranking_score": "0.7185324707231184", - "tolerated_regret": 0.007639684811918013, - "tolerated_test_gap": "0.007639684811918013", - "tolerated_val_decision_score": "0.7019436364687074", - "tolerated_val_gap": "0.011268223121093768", - "tolerated_val_ranking_score": "0.6906754133476136", - "union_decision_head": "linear probe", - "union_decision_head_arch": "linear", - "union_decision_score": "0.7261721555350364", - "union_ranking_score": "0.7185324707231184", - "union_regret": 0.007639684811918013, - "union_test_gap": "0.007639684811918013", - "union_val_decision_score": "0.7019436364687074", - "union_val_gap": "0.011268223121093768", - "union_val_ranking_score": "0.6906754133476136", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0", - "exact_ranking_score": "0.0", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.0024802356223841263", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.0024802356223841263", - "family": "Pangu-Weather", - "model_tag": "pangu6", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.00040651309469793043", - "ranking_selected_val_pr_auc": "0.0004045310999550776", - "scope": "global", - "seed": 42, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.32154249424354786", - "tolerated_ranking_score": "0.32154249424354786", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.31489341288382094", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.31489341288382094", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.32154249424354786", - "union_ranking_score": "0.32154249424354786", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.31489341288382094", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.31489341288382094", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0430065972893146", - "exact_ranking_score": "0.0430065972893146", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.044111302847282145", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.044111302847282145", - "family": "Pangu-Weather", - "model_tag": "pangu6", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.021420904714594617", - "ranking_selected_val_pr_auc": "0.019667925605377917", - "scope": "fire_prone", - "seed": 42, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7241279907754397", - "tolerated_ranking_score": "0.7241279907754397", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.6965685921876124", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.6965685921876124", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7241279907754397", - "union_ranking_score": "0.7241279907754397", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.6965685921876124", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.6965685921876124", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0", - "exact_ranking_score": "0.0", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.003894080996884735", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.003894080996884735", - "family": "Pangu-Weather", - "model_tag": "pangu6", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.00038395193539280824", - "ranking_selected_val_pr_auc": "0.0005470853500668261", - "scope": "global", - "seed": 7, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.31120670593952293", - "tolerated_ranking_score": "0.31120670593952293", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.30691963220059004", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.30691963220059004", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.31120670593952293", - "union_ranking_score": "0.31120670593952293", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.30691963220059004", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.30691963220059004", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.04372215277109978", - "exact_ranking_score": "0.04101254412979794", - "exact_regret": 0.0027096086413018403, - "exact_test_gap": "0.0027096086413018403", - "exact_val_decision_score": "0.043845404352062356", - "exact_val_gap": "0.0013135054278692188", - "exact_val_ranking_score": "0.04253189892419314", - "family": "Pangu-Weather", - "model_tag": "pangu6", - "ranking_selected_head": "pixel MLP head", - "ranking_selected_head_arch": "pixel_mlp", - "ranking_selected_test_pr_auc": "0.018664183428008234", - "ranking_selected_val_pr_auc": "0.024661788918260808", - "scope": "fire_prone", - "seed": 7, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.719461376864001", - "tolerated_ranking_score": "0.7185324707231184", - "tolerated_regret": 0.0009289061408825905, - "tolerated_test_gap": "0.0009289061408825905", - "tolerated_val_decision_score": "0.6947253251451462", - "tolerated_val_gap": "0.004049911797532557", - "tolerated_val_ranking_score": "0.6906754133476136", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.719461376864001", - "union_ranking_score": "0.7185324707231184", - "union_regret": 0.0009289061408825905, - "union_test_gap": "0.0009289061408825905", - "union_val_decision_score": "0.6947253251451462", - "union_val_gap": "0.004049911797532557", - "union_val_ranking_score": "0.6906754133476136", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0", - "exact_ranking_score": "0.0", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.002408671216378964", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.002408671216378964", - "family": "Pangu-Weather", - "model_tag": "pangu6", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.0004450373086921994", - "ranking_selected_val_pr_auc": "0.000447244495874373", - "scope": "global", - "seed": 99, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.3214547875801752", - "tolerated_ranking_score": "0.3214547875801752", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.3045294255465288", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.3045294255465288", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.3214547875801752", - "union_ranking_score": "0.3214547875801752", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.3045294255465288", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.3045294255465288", - "path": "" - }, - { - "exact_decision_head": "linear probe", - "exact_decision_head_arch": "linear", - "exact_decision_score": "0.0044004400440044", - "exact_ranking_score": "0.0044004400440044", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.06142857142857142", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.06142857142857142", - "family": "Pangu-Weather", - "model_tag": "pangu6", - "ranking_selected_head": "linear probe", - "ranking_selected_head_arch": "linear", - "ranking_selected_test_pr_auc": "0.024740444457605402", - "ranking_selected_val_pr_auc": "0.036676155849720546", - "scope": "fire_prone", - "seed": 99, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.719015307962107", - "tolerated_ranking_score": "0.7183382629739177", - "tolerated_regret": 0.0006770449881893237, - "tolerated_test_gap": "0.0006770449881893237", - "tolerated_val_decision_score": "0.6913348632305398", - "tolerated_val_gap": "0.0006380914321632014", - "tolerated_val_ranking_score": "0.6906967717983766", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.719015307962107", - "union_ranking_score": "0.7183382629739177", - "union_regret": 0.0006770449881893237, - "union_test_gap": "0.0006770449881893237", - "union_val_decision_score": "0.6913348632305398", - "union_val_gap": "0.0006380914321632014", - "union_val_ranking_score": "0.6906967717983766", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0004925501476206198", - "exact_ranking_score": "0.0004925501476206198", - "exact_regret": 0.0, - "family": "Prithvi-WxC", - "model_tag": "prithvi_wxc", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_pr_auc": "0.0002463357401629007", - "scope": "global", - "seed": 1, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.240793572992212", - "tolerated_ranking_score": "0.240793572992212", - "tolerated_regret": 0.0, - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.240793572992212", - "union_ranking_score": "0.240793572992212", - "union_regret": 0.0, - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.04101254412979794", - "exact_ranking_score": "0.04101254412979794", - "exact_regret": 0.0, - "family": "Prithvi-WxC", - "model_tag": "prithvi_wxc", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_pr_auc": "0.02093558282208589", - "scope": "fire_prone", - "seed": 1, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7185324707231184", - "tolerated_ranking_score": "0.7185324707231184", - "tolerated_regret": 0.0, - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7185324707231184", - "union_ranking_score": "0.7185324707231184", - "union_regret": 0.0, - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0004925501476206198", - "exact_ranking_score": "0.0004925501476206198", - "exact_regret": 0.0, - "family": "Prithvi-WxC", - "model_tag": "prithvi_wxc", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_pr_auc": "0.0002463357401629007", - "scope": "global", - "seed": 123, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.240793572992212", - "tolerated_ranking_score": "0.240793572992212", - "tolerated_regret": 0.0, - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.240793572992212", - "union_ranking_score": "0.240793572992212", - "union_regret": 0.0, - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.04101254412979794", - "exact_ranking_score": "0.04101254412979794", - "exact_regret": 0.0, - "family": "Prithvi-WxC", - "model_tag": "prithvi_wxc", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_pr_auc": "0.02093558282208589", - "scope": "fire_prone", - "seed": 123, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7185324707231184", - "tolerated_ranking_score": "0.7185324707231184", - "tolerated_regret": 0.0, - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7185324707231184", - "union_ranking_score": "0.7185324707231184", - "union_regret": 0.0, - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0004925501476206198", - "exact_ranking_score": "0.0004925501476206198", - "exact_regret": 0.0, - "family": "Prithvi-WxC", - "model_tag": "prithvi_wxc", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_pr_auc": "0.0002463357401629007", - "scope": "global", - "seed": 42, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.240793572992212", - "tolerated_ranking_score": "0.240793572992212", - "tolerated_regret": 0.0, - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.240793572992212", - "union_ranking_score": "0.240793572992212", - "union_regret": 0.0, - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.04101254412979794", - "exact_ranking_score": "0.04101254412979794", - "exact_regret": 0.0, - "family": "Prithvi-WxC", - "model_tag": "prithvi_wxc", - "ranking_selected_head": "pixel MLP head", - "ranking_selected_head_arch": "pixel_mlp", - "ranking_selected_pr_auc": "0.02093558282208589", - "scope": "fire_prone", - "seed": 42, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7185324707231184", - "tolerated_ranking_score": "0.7185324707231184", - "tolerated_regret": 0.0, - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7185324707231184", - "union_ranking_score": "0.7185324707231184", - "union_regret": 0.0, - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0004925501476206198", - "exact_ranking_score": "0.0004925501476206198", - "exact_regret": 0.0, - "family": "Prithvi-WxC", - "model_tag": "prithvi_wxc", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_pr_auc": "0.0002463357401629007", - "scope": "global", - "seed": 7, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.240793572992212", - "tolerated_ranking_score": "0.240793572992212", - "tolerated_regret": 0.0, - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.240793572992212", - "union_ranking_score": "0.240793572992212", - "union_regret": 0.0, - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.04101254412979794", - "exact_ranking_score": "0.04101254412979794", - "exact_regret": 0.0, - "family": "Prithvi-WxC", - "model_tag": "prithvi_wxc", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_pr_auc": "0.02093558282208589", - "scope": "fire_prone", - "seed": 7, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7185324707231184", - "tolerated_ranking_score": "0.7185324707231184", - "tolerated_regret": 0.0, - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7185324707231184", - "union_ranking_score": "0.7185324707231184", - "union_regret": 0.0, - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.00048497822606044473", - "exact_ranking_score": "0.00048497822606044473", - "exact_regret": 0.0, - "family": "Prithvi-WxC", - "model_tag": "prithvi_wxc", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_pr_auc": "0.0002454330184381399", - "scope": "global", - "seed": 99, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.23755358049655212", - "tolerated_ranking_score": "0.23755358049655212", - "tolerated_regret": 0.0, - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.23755358049655212", - "union_ranking_score": "0.23755358049655212", - "union_regret": 0.0, - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.04101254412979794", - "exact_ranking_score": "0.04101254412979794", - "exact_regret": 0.0, - "family": "Prithvi-WxC", - "model_tag": "prithvi_wxc", - "ranking_selected_head": "linear probe", - "ranking_selected_head_arch": "linear", - "ranking_selected_pr_auc": "0.02093558282208589", - "scope": "fire_prone", - "seed": 99, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7185324707231184", - "tolerated_ranking_score": "0.7185324707231184", - "tolerated_regret": 0.0, - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7185324707231184", - "union_ranking_score": "0.7185324707231184", - "union_regret": 0.0, - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.017134572294714646", - "exact_ranking_score": "0.017134572294714646", - "exact_regret": 0.0, - "family": "Reference", - "model_tag": "reference", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_pr_auc": "0.002624341503354088", - "scope": "global", - "seed": 1, - "tolerated_decision_head": "linear probe", - "tolerated_decision_head_arch": "linear", - "tolerated_decision_score": "0.7286109603326707", - "tolerated_ranking_score": "0.6186566066408326", - "tolerated_regret": 0.10995435369183815, - "union_decision_head": "linear probe", - "union_decision_head_arch": "linear", - "union_decision_score": "0.7286109603326707", - "union_ranking_score": "0.6186566066408326", - "union_regret": 0.10995435369183815, - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.1611624834874505", - "exact_ranking_score": "0.1611624834874505", - "exact_regret": 0.0, - "family": "Reference", - "model_tag": "reference", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_pr_auc": "0.10204224118176683", - "scope": "fire_prone", - "seed": 1, - "tolerated_decision_head": "linear probe", - "tolerated_decision_head_arch": "linear", - "tolerated_decision_score": "0.8685988450931266", - "tolerated_ranking_score": "0.799032457577039", - "tolerated_regret": 0.06956638751608757, - "union_decision_head": "linear probe", - "union_decision_head_arch": "linear", - "union_decision_score": "0.8685988450931266", - "union_ranking_score": "0.799032457577039", - "union_regret": 0.06956638751608757, - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.019320660641944532", - "exact_ranking_score": "0.019320660641944532", - "exact_regret": 0.0, - "family": "Reference", - "model_tag": "reference", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_pr_auc": "0.0025603887793133394", - "scope": "global", - "seed": 123, - "tolerated_decision_head": "pixel MLP head", - "tolerated_decision_head_arch": "pixel_mlp", - "tolerated_decision_score": "0.735221546471909", - "tolerated_ranking_score": "0.5009707461135111", - "tolerated_regret": 0.23425080035839785, - "union_decision_head": "pixel MLP head", - "union_decision_head_arch": "pixel_mlp", - "union_decision_score": "0.735221546471909", - "union_ranking_score": "0.5009707461135111", - "union_regret": 0.23425080035839785, - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.19165378670788252", - "exact_ranking_score": "0.19165378670788252", - "exact_regret": 0.0, - "family": "Reference", - "model_tag": "reference", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_pr_auc": "0.11551882470432043", - "scope": "fire_prone", - "seed": 123, - "tolerated_decision_head": "pixel MLP head", - "tolerated_decision_head_arch": "pixel_mlp", - "tolerated_decision_score": "0.8567215417854325", - "tolerated_ranking_score": "0.8066689866810086", - "tolerated_regret": 0.05005255510442386, - "union_decision_head": "pixel MLP head", - "union_decision_head_arch": "pixel_mlp", - "union_decision_score": "0.8567215417854325", - "union_ranking_score": "0.8066689866810086", - "union_regret": 0.05005255510442386, - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.025515210991167808", - "exact_ranking_score": "0.025515210991167808", - "exact_regret": 0.0, - "family": "Reference", - "model_tag": "reference", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_pr_auc": "0.0032729435045747413", - "scope": "global", - "seed": 42, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7239943741463363", - "tolerated_ranking_score": "0.7239943741463363", - "tolerated_regret": 0.0, - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7239943741463363", - "union_ranking_score": "0.7239943741463363", - "union_regret": 0.0, - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.19225806451612906", - "exact_ranking_score": "0.19225806451612906", - "exact_regret": 0.0, - "family": "Reference", - "model_tag": "reference", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_pr_auc": "0.12048427320762313", - "scope": "fire_prone", - "seed": 42, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.8358151221553631", - "tolerated_ranking_score": "0.8358151221553631", - "tolerated_regret": 0.0, - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.8358151221553631", - "union_ranking_score": "0.8358151221553631", - "union_regret": 0.0, - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.02287675150128682", - "exact_ranking_score": "0.02287675150128682", - "exact_regret": 0.0, - "family": "Reference", - "model_tag": "reference", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_pr_auc": "0.0030317345997110698", - "scope": "global", - "seed": 7, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.5002193417313746", - "tolerated_ranking_score": "0.5002193417313746", - "tolerated_regret": 0.0, - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.5002193417313746", - "union_ranking_score": "0.5002193417313746", - "union_regret": 0.0, - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.22885572139303484", - "exact_ranking_score": "0.22885572139303484", - "exact_regret": 0.0, - "family": "Reference", - "model_tag": "reference", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_pr_auc": "0.1323902067230726", - "scope": "fire_prone", - "seed": 7, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.8027807546359551", - "tolerated_ranking_score": "0.8027807546359551", - "tolerated_regret": 0.0, - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.8027807546359551", - "union_ranking_score": "0.8027807546359551", - "union_regret": 0.0, - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.021876258220373104", - "exact_ranking_score": "0.021876258220373104", - "exact_regret": 0.0, - "family": "Reference", - "model_tag": "reference", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_pr_auc": "0.0031953098868323193", - "scope": "global", - "seed": 99, - "tolerated_decision_head": "linear probe", - "tolerated_decision_head_arch": "linear", - "tolerated_decision_score": "0.7647466876509988", - "tolerated_ranking_score": "0.6698005926442897", - "tolerated_regret": 0.09494609500670914, - "union_decision_head": "linear probe", - "union_decision_head_arch": "linear", - "union_decision_score": "0.7647466876509988", - "union_ranking_score": "0.6698005926442897", - "union_regret": 0.09494609500670914, - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.19477124183006533", - "exact_ranking_score": "0.19477124183006533", - "exact_regret": 0.0, - "family": "Reference", - "model_tag": "reference", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_pr_auc": "0.11947246238005743", - "scope": "fire_prone", - "seed": 99, - "tolerated_decision_head": "linear probe", - "tolerated_decision_head_arch": "linear", - "tolerated_decision_score": "0.9039923296574045", - "tolerated_ranking_score": "0.8534759193943048", - "tolerated_regret": 0.05051641026309972, - "union_decision_head": "linear probe", - "union_decision_head_arch": "linear", - "union_decision_score": "0.9039923296574045", - "union_ranking_score": "0.8534759193943048", - "union_regret": 0.05051641026309972, - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0004925501476206198", - "exact_ranking_score": "0.0004925501476206198", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.0004727121748000204", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.0004727121748000204", - "family": "StormCast", - "model_tag": "stormcast", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.0002463357401629007", - "ranking_selected_val_pr_auc": "0.00023641196480702655", - "scope": "global", - "seed": 1, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.240793572992212", - "tolerated_ranking_score": "0.240793572992212", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.2179368807313983", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.2179368807313983", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.240793572992212", - "union_ranking_score": "0.240793572992212", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.2179368807313983", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.2179368807313983", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.04101254412979794", - "exact_ranking_score": "0.04101254412979794", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.04253189892419314", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.04253189892419314", - "family": "StormCast", - "model_tag": "stormcast", - "ranking_selected_head": "linear probe", - "ranking_selected_head_arch": "linear", - "ranking_selected_test_pr_auc": "0.02093558282208589", - "ranking_selected_val_pr_auc": "0.0217280163599182", - "scope": "fire_prone", - "seed": 1, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7185324707231184", - "tolerated_ranking_score": "0.7185324707231184", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.6906754133476136", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.6906754133476136", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7185324707231184", - "union_ranking_score": "0.7185324707231184", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.6906754133476136", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.6906754133476136", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0004925501476206198", - "exact_ranking_score": "0.0004925501476206198", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.0004727121748000204", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.0004727121748000204", - "family": "StormCast", - "model_tag": "stormcast", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.0002463357401629007", - "ranking_selected_val_pr_auc": "0.00023641196480702655", - "scope": "global", - "seed": 123, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.240793572992212", - "tolerated_ranking_score": "0.240793572992212", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.2179368807313983", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.2179368807313983", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.240793572992212", - "union_ranking_score": "0.240793572992212", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.2179368807313983", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.2179368807313983", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.04101254412979794", - "exact_ranking_score": "0.04101254412979794", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.04253189892419314", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.04253189892419314", - "family": "StormCast", - "model_tag": "stormcast", - "ranking_selected_head": "linear probe", - "ranking_selected_head_arch": "linear", - "ranking_selected_test_pr_auc": "0.02093558282208589", - "ranking_selected_val_pr_auc": "0.0217280163599182", - "scope": "fire_prone", - "seed": 123, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7185324707231184", - "tolerated_ranking_score": "0.7185324707231184", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.6906754133476136", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.6906754133476136", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7185324707231184", - "union_ranking_score": "0.7185324707231184", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.6906754133476136", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.6906754133476136", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0004925501476206198", - "exact_ranking_score": "0.0004925501476206198", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.0004727121748000204", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.0004727121748000204", - "family": "StormCast", - "model_tag": "stormcast", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.0002463357401629007", - "ranking_selected_val_pr_auc": "0.00023641196480702655", - "scope": "global", - "seed": 42, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.240793572992212", - "tolerated_ranking_score": "0.240793572992212", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.2179368807313983", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.2179368807313983", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.240793572992212", - "union_ranking_score": "0.240793572992212", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.2179368807313983", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.2179368807313983", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.04101254412979794", - "exact_ranking_score": "0.04101254412979794", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.04253189892419314", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.04253189892419314", - "family": "StormCast", - "model_tag": "stormcast", - "ranking_selected_head": "linear probe", - "ranking_selected_head_arch": "linear", - "ranking_selected_test_pr_auc": "0.02093558282208589", - "ranking_selected_val_pr_auc": "0.0217280163599182", - "scope": "fire_prone", - "seed": 42, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7185324707231184", - "tolerated_ranking_score": "0.7185324707231184", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.6906754133476136", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.6906754133476136", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7185324707231184", - "union_ranking_score": "0.7185324707231184", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.6906754133476136", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.6906754133476136", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0004925501476206198", - "exact_ranking_score": "0.0004925501476206198", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.0004727121748000204", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.0004727121748000204", - "family": "StormCast", - "model_tag": "stormcast", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.0002463357401629007", - "ranking_selected_val_pr_auc": "0.00023641196480702655", - "scope": "global", - "seed": 7, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.240793572992212", - "tolerated_ranking_score": "0.240793572992212", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.2179368807313983", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.2179368807313983", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.240793572992212", - "union_ranking_score": "0.240793572992212", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.2179368807313983", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.2179368807313983", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.04101254412979794", - "exact_ranking_score": "0.04101254412979794", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.04253189892419314", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.04253189892419314", - "family": "StormCast", - "model_tag": "stormcast", - "ranking_selected_head": "linear probe", - "ranking_selected_head_arch": "linear", - "ranking_selected_test_pr_auc": "0.02093558282208589", - "ranking_selected_val_pr_auc": "0.0217280163599182", - "scope": "fire_prone", - "seed": 7, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7185324707231184", - "tolerated_ranking_score": "0.7185324707231184", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.6906754133476136", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.6906754133476136", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7185324707231184", - "union_ranking_score": "0.7185324707231184", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.6906754133476136", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.6906754133476136", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0004905319945803844", - "exact_ranking_score": "0.0004905319945803844", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.0004707752858483998", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.0004707752858483998", - "family": "StormCast", - "model_tag": "stormcast", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.0002463357401629007", - "ranking_selected_val_pr_auc": "0.00023641196480702655", - "scope": "global", - "seed": 99, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.2399251001974223", - "tolerated_ranking_score": "0.2399251001974223", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.21714067007331003", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.21714067007331003", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.2399251001974223", - "union_ranking_score": "0.2399251001974223", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.21714067007331003", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.21714067007331003", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.04101254412979794", - "exact_ranking_score": "0.04101254412979794", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.04253189892419314", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.04253189892419314", - "family": "StormCast", - "model_tag": "stormcast", - "ranking_selected_head": "pixel MLP head", - "ranking_selected_head_arch": "pixel_mlp", - "ranking_selected_test_pr_auc": "0.02093558282208589", - "ranking_selected_val_pr_auc": "0.0217280163599182", - "scope": "fire_prone", - "seed": 99, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7185324707231184", - "tolerated_ranking_score": "0.7185324707231184", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.6906754133476136", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.6906754133476136", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7185324707231184", - "union_ranking_score": "0.7185324707231184", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.6906754133476136", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.6906754133476136", - "path": "" - } - ], - "summary": [ - { - "model_tag": "reference", - "label": "Reference", - "scope": "global", - "scope_label": "\\(\\Omega=\\)global", - "n": 5, - "seeds": [ - 1, - 7, - 42, - 99, - 123 - ], - "paths": [ - "", - "", - "", - "", - "" - ], - "exact_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - }, - "tolerated_regret": { - "n": 5, - "mean": 0.08783024981138902, - "std": 0.09670495645481135, - "min": 0.0, - "max": 0.23425080035839785 - }, - "union_regret": { - "n": 5, - "mean": 0.08783024981138902, - "std": 0.09670495645481135, - "min": 0.0, - "max": 0.23425080035839785 - } - }, - { - "model_tag": "reference", - "label": "Reference", - "scope": "fire_prone", - "scope_label": "\\(\\Omega=\\)top 20\\%", - "n": 5, - "seeds": [ - 1, - 7, - 42, - 99, - 123 - ], - "paths": [ - "", - "", - "", - "", - "" - ], - "exact_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - }, - "tolerated_regret": { - "n": 5, - "mean": 0.03402707057672223, - "std": 0.032044658643147844, - "min": 0.0, - "max": 0.06956638751608757 - }, - "union_regret": { - "n": 5, - "mean": 0.03402707057672223, - "std": 0.032044658643147844, - "min": 0.0, - "max": 0.06956638751608757 - } - }, - { - "model_tag": "prithvi_wxc", - "label": "Prithvi-WxC", - "scope": "global", - "scope_label": "\\(\\Omega=\\)global", - "n": 5, - "seeds": [ - 1, - 7, - 42, - 99, - 123 - ], - "paths": [ - "", - "", - "", - "", - "" - ], - "exact_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - }, - "tolerated_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - }, - "union_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - } - }, - { - "model_tag": "prithvi_wxc", - "label": "Prithvi-WxC", - "scope": "fire_prone", - "scope_label": "\\(\\Omega=\\)top 20\\%", - "n": 5, - "seeds": [ - 1, - 7, - 42, - 99, - 123 - ], - "paths": [ - "", - "", - "", - "", - "" - ], - "exact_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - }, - "tolerated_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - }, - "union_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - } - }, - { - "model_tag": "aurora", - "label": "Aurora", - "scope": "global", - "scope_label": "\\(\\Omega=\\)global", - "n": 5, - "seeds": [ - 1, - 7, - 42, - 99, - 123 - ], - "paths": [ - "", - "", - "", - "", - "" - ], - "exact_regret": { - "n": 5, - "mean": 0.00020004882767231798, - "std": 0.00026703384456332115, - "min": 0.0, - "max": 0.0004925501476206198 - }, - "tolerated_regret": { - "n": 5, - "mean": 0.09851983041506818, - "std": 0.1298781980037557, - "min": 0.0032399924956598714, - "max": 0.240793572992212 - }, - "union_regret": { - "n": 5, - "mean": 0.09851983041506818, - "std": 0.1298781980037557, - "min": 0.0032399924956598714, - "max": 0.240793572992212 - } - }, - { - "model_tag": "aurora", - "label": "Aurora", - "scope": "fire_prone", - "scope_label": "\\(\\Omega=\\)top 20\\%", - "n": 5, - "seeds": [ - 1, - 7, - 42, - 99, - 123 - ], - "paths": [ - "", - "", - "", - "", - "" - ], - "exact_regret": { - "n": 5, - "mean": 0.008202508825959588, - "std": 0.01834136732088763, - "min": 0.0, - "max": 0.04101254412979794 - }, - "tolerated_regret": { - "n": 5, - "mean": 0.14391889430974364, - "std": 0.32121904665016227, - "min": 0.0, - "max": 0.7185324707231184 - }, - "union_regret": { - "n": 5, - "mean": 0.14391889430974364, - "std": 0.32121904665016227, - "min": 0.0, - "max": 0.7185324707231184 - } - }, - { - "model_tag": "climax", - "label": "ClimaX", - "scope": "global", - "scope_label": "\\(\\Omega=\\)global", - "n": 5, - "seeds": [ - 1, - 7, - 42, - 99, - 123 - ], - "paths": [ - "", - "", - "", - "", - "" - ], - "exact_regret": { - "n": 5, - "mean": 3.0287686240700486e-06, - "std": 4.147312242167625e-06, - "min": 0.0, - "max": 7.571921560175121e-06 - }, - "tolerated_regret": { - "n": 5, - "mean": 0.0012959969982639485, - "std": 0.0017746169760203706, - "min": 0.0, - "max": 0.0032399924956598714 - }, - "union_regret": { - "n": 5, - "mean": 0.0012959969982639485, - "std": 0.0017746169760203706, - "min": 0.0, - "max": 0.0032399924956598714 - } - }, - { - "model_tag": "climax", - "label": "ClimaX", - "scope": "fire_prone", - "scope_label": "\\(\\Omega=\\)top 20\\%", - "n": 5, - "seeds": [ - 1, - 7, - 42, - 99, - 123 - ], - "paths": [ - "", - "", - "", - "", - "" - ], - "exact_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - }, - "tolerated_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - }, - "union_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - } - }, - { - "model_tag": "stormcast", - "label": "StormCast", - "scope": "global", - "scope_label": "\\(\\Omega=\\)global", - "n": 5, - "seeds": [ - 1, - 7, - 42, - 99, - 123 - ], - "paths": [ - "", - "", - "", - "", - "" - ], - "exact_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - }, - "tolerated_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - }, - "union_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - } - }, - { - "model_tag": "stormcast", - "label": "StormCast", - "scope": "fire_prone", - "scope_label": "\\(\\Omega=\\)top 20\\%", - "n": 5, - "seeds": [ - 1, - 7, - 42, - 99, - 123 - ], - "paths": [ - "", - "", - "", - "", - "" - ], - "exact_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - }, - "tolerated_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - }, - "union_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - } - }, - { - "model_tag": "pangu_weather", - "label": "Pangu-Weather", - "scope": "global", - "scope_label": "\\(\\Omega=\\)global", - "n": 5, - "seeds": [ - 1, - 7, - 42, - 99, - 123 - ], - "paths": [ - "", - "", - "", - "", - "" - ], - "exact_regret": { - "n": 5, - "mean": 0.00013033979247265275, - "std": 0.0002685372203690466, - "min": 0.0, - "max": 0.0006100601402256502 - }, - "tolerated_regret": { - "n": 5, - "mean": 0.048806713097574374, - "std": 0.10733308684741971, - "min": 0.0, - "max": 0.240793572992212 - }, - "union_regret": { - "n": 5, - "mean": 0.048806713097574374, - "std": 0.10733308684741971, - "min": 0.0, - "max": 0.240793572992212 - } - }, - { - "model_tag": "pangu_weather", - "label": "Pangu-Weather", - "scope": "fire_prone", - "scope_label": "\\(\\Omega=\\)top 20\\%", - "n": 5, - "seeds": [ - 1, - 7, - 42, - 99, - 123 - ], - "paths": [ - "", - "", - "", - "", - "" - ], - "exact_regret": { - "n": 5, - "mean": 0.027875386332505546, - "std": 0.02348779386900393, - "min": 0.0, - "max": 0.04813290893564165 - }, - "tolerated_regret": { - "n": 5, - "mean": 0.43111948243387105, - "std": 0.39355644251497235, - "min": 0.0, - "max": 0.7185324707231184 - }, - "union_regret": { - "n": 5, - "mean": 0.43111948243387105, - "std": 0.39355644251497235, - "min": 0.0, - "max": 0.7185324707231184 - } - }, - { - "model_tag": "dlwp", - "label": "DLWP", - "scope": "global", - "scope_label": "\\(\\Omega=\\)global", - "n": 5, - "seeds": [ - 1, - 7, - 42, - 99, - 123 - ], - "paths": [ - "", - "", - "", - "", - "" - ], - "exact_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - }, - "tolerated_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - }, - "union_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - } - }, - { - "model_tag": "dlwp", - "label": "DLWP", - "scope": "fire_prone", - "scope_label": "\\(\\Omega=\\)top 20\\%", - "n": 5, - "seeds": [ - 1, - 7, - 42, - 99, - 123 - ], - "paths": [ - "", - "", - "", - "", - "" - ], - "exact_regret": { - "n": 5, - "mean": 0.0007702319787454587, - "std": 0.0010995336594539604, - "min": 0.0, - "max": 0.0023651634514294945 - }, - "tolerated_regret": { - "n": 5, - "mean": 0.043265915053601556, - "std": 0.04332331365579739, - "min": 0.0, - "max": 0.11215006616165157 - }, - "union_regret": { - "n": 5, - "mean": 0.043265915053601556, - "std": 0.04332331365579739, - "min": 0.0, - "max": 0.11215006616165157 - } - }, - { - "model_tag": "fcn", - "label": "FCN", - "scope": "global", - "scope_label": "\\(\\Omega=\\)global", - "n": 5, - "seeds": [ - 1, - 7, - 42, - 99, - 123 - ], - "paths": [ - "", - "", - "", - "", - "" - ], - "exact_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - }, - "tolerated_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - }, - "union_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - } - }, - { - "model_tag": "fcn", - "label": "FCN", - "scope": "fire_prone", - "scope_label": "\\(\\Omega=\\)top 20\\%", - "n": 5, - "seeds": [ - 1, - 7, - 42, - 99, - 123 - ], - "paths": [ - "", - "", - "", - "", - "" - ], - "exact_regret": { - "n": 5, - "mean": 5.960229415004348e-06, - "std": 1.3327478133443526e-05, - "min": 0.0, - "max": 2.980114707502174e-05 - }, - "tolerated_regret": { - "n": 5, - "mean": 0.011679805987441694, - "std": 0.019872372458657642, - "min": 0.0, - "max": 0.045880503237594294 - }, - "union_regret": { - "n": 5, - "mean": 0.011679805987441694, - "std": 0.019872372458657642, - "min": 0.0, - "max": 0.045880503237594294 - } - }, - { - "model_tag": "fengwu", - "label": "FengWu", - "scope": "global", - "scope_label": "\\(\\Omega=\\)global", - "n": 5, - "seeds": [ - 1, - 7, - 42, - 99, - 123 - ], - "paths": [ - "", - "", - "", - "", - "" - ], - "exact_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - }, - "tolerated_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - }, - "union_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - } - }, - { - "model_tag": "fengwu", - "label": "FengWu", - "scope": "fire_prone", - "scope_label": "\\(\\Omega=\\)top 20\\%", - "n": 5, - "seeds": [ - 1, - 7, - 42, - 99, - 123 - ], - "paths": [ - "", - "", - "", - "", - "" - ], - "exact_regret": { - "n": 5, - "mean": 0.0006908222234409067, - "std": 0.0011910586589384115, - "min": 0.0, - "max": 0.0027505832409660327 - }, - "tolerated_regret": { - "n": 5, - "mean": 0.005222389249812243, - "std": 0.0062394095558402415, - "min": 0.0, - "max": 0.015150501067724198 - }, - "union_regret": { - "n": 5, - "mean": 0.005222389249812243, - "std": 0.0062394095558402415, - "min": 0.0, - "max": 0.015150501067724198 - } - }, - { - "model_tag": "fuxi", - "label": "FuXi", - "scope": "global", - "scope_label": "\\(\\Omega=\\)global", - "n": 5, - "seeds": [ - 1, - 7, - 42, - 99, - 123 - ], - "paths": [ - "", - "", - "", - "", - "" - ], - "exact_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - }, - "tolerated_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - }, - "union_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - } - }, - { - "model_tag": "fuxi", - "label": "FuXi", - "scope": "fire_prone", - "scope_label": "\\(\\Omega=\\)top 20\\%", - "n": 5, - "seeds": [ - 1, - 7, - 42, - 99, - 123 - ], - "paths": [ - "", - "", - "", - "", - "" - ], - "exact_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - }, - "tolerated_regret": { - "n": 5, - "mean": 0.0010839188523199318, - "std": 0.0017288780545672386, - "min": 0.0, - "max": 0.0039636864245868875 - }, - "union_regret": { - "n": 5, - "mean": 0.0010839188523199318, - "std": 0.0017288780545672386, - "min": 0.0, - "max": 0.0039636864245868875 - } - }, - { - "model_tag": "pangu6", - "label": "Pangu-Weather", - "scope": "global", - "scope_label": "\\(\\Omega=\\)global", - "n": 5, - "seeds": [ - 1, - 7, - 42, - 99, - 123 - ], - "paths": [ - "", - "", - "", - "", - "" - ], - "exact_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - }, - "tolerated_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - }, - "union_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - } - }, - { - "model_tag": "pangu6", - "label": "Pangu-Weather", - "scope": "fire_prone", - "scope_label": "\\(\\Omega=\\)top 20\\%", - "n": 5, - "seeds": [ - 1, - 7, - 42, - 99, - 123 - ], - "paths": [ - "", - "", - "", - "", - "" - ], - "exact_regret": { - "n": 5, - "mean": 0.0007280423771922354, - "std": 0.001178746460551365, - "min": 0.0, - "max": 0.0027096086413018403 - }, - "tolerated_regret": { - "n": 5, - "mean": 0.0018491271881979853, - "std": 0.0032630386057089294, - "min": 0.0, - "max": 0.007639684811918013 - }, - "union_regret": { - "n": 5, - "mean": 0.0018491271881979853, - "std": 0.0032630386057089294, - "min": 0.0, - "max": 0.007639684811918013 - } - }, - { - "model_tag": "alphaearth", - "label": "AlphaEarth", - "scope": "global", - "scope_label": "\\(\\Omega=\\)global", - "n": 5, - "seeds": [ - 1, - 7, - 42, - 99, - 123 - ], - "paths": [ - "", - "", - "", - "", - "" - ], - "exact_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - }, - "tolerated_regret": { - "n": 5, - "mean": 0.1722171037486726, - "std": 0.08849214830495522, - "min": 0.050655329891645895, - "max": 0.272468682523694 - }, - "union_regret": { - "n": 5, - "mean": 0.1722171037486726, - "std": 0.08849214830495522, - "min": 0.050655329891645895, - "max": 0.272468682523694 - } - }, - { - "model_tag": "alphaearth", - "label": "AlphaEarth", - "scope": "fire_prone", - "scope_label": "\\(\\Omega=\\)top 20\\%", - "n": 5, - "seeds": [ - 1, - 7, - 42, - 99, - 123 - ], - "paths": [ - "", - "", - "", - "", - "" - ], - "exact_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - }, - "tolerated_regret": { - "n": 5, - "mean": 0.038803552655092256, - "std": 0.0594825313313219, - "min": 0.0, - "max": 0.13482302181308625 - }, - "union_regret": { - "n": 5, - "mean": 0.038803552655092256, - "std": 0.0594825313313219, - "min": 0.0, - "max": 0.13482302181308625 - } - } - ], - "issues": [], - "notes": [ - "Reference global exact_regret has true zero displayed std; table omitted", - "Reference fire_prone exact_regret has true zero displayed std; table omitted", - "Prithvi-WxC global exact_regret has true zero displayed std; table omitted", - "Prithvi-WxC global tolerated_regret has true zero displayed std; table omitted", - "Prithvi-WxC global union_regret has true zero displayed std; table omitted", - "Prithvi-WxC fire_prone exact_regret has true zero displayed std; table omitted", - "Prithvi-WxC fire_prone tolerated_regret has true zero displayed std; table omitted", - "Prithvi-WxC fire_prone union_regret has true zero displayed std; table omitted", - "ClimaX fire_prone exact_regret has true zero displayed std; table omitted", - "ClimaX fire_prone tolerated_regret has true zero displayed std; table omitted", - "ClimaX fire_prone union_regret has true zero displayed std; table omitted", - "StormCast global exact_regret has true zero displayed std; table omitted", - "StormCast global tolerated_regret has true zero displayed std; table omitted", - "StormCast global union_regret has true zero displayed std; table omitted", - "StormCast fire_prone exact_regret has true zero displayed std; table omitted", - "StormCast fire_prone tolerated_regret has true zero displayed std; table omitted", - "StormCast fire_prone union_regret has true zero displayed std; table omitted", - "DLWP global exact_regret has true zero displayed std; table omitted", - "DLWP global tolerated_regret has true zero displayed std; table omitted", - "DLWP global union_regret has true zero displayed std; table omitted", - "FCN global exact_regret has true zero displayed std; table omitted", - "FCN global tolerated_regret has true zero displayed std; table omitted", - "FCN global union_regret has true zero displayed std; table omitted", - "FengWu global exact_regret has true zero displayed std; table omitted", - "FengWu global tolerated_regret has true zero displayed std; table omitted", - "FengWu global union_regret has true zero displayed std; table omitted", - "FuXi global exact_regret has true zero displayed std; table omitted", - "FuXi global tolerated_regret has true zero displayed std; table omitted", - "FuXi global union_regret has true zero displayed std; table omitted", - "FuXi fire_prone exact_regret has true zero displayed std; table omitted", - "Pangu-Weather global exact_regret has true zero displayed std; table omitted", - "Pangu-Weather global tolerated_regret has true zero displayed std; table omitted", - "Pangu-Weather global union_regret has true zero displayed std; table omitted", - "AlphaEarth global exact_regret has true zero displayed std; table omitted", - "AlphaEarth fire_prone exact_regret has true zero displayed std; table omitted" - ] -} diff --git a/artifacts/results/selection_regret_head_metrics.csv b/artifacts/results/selection_regret_head_metrics.csv deleted file mode 100644 index 94dbb805d21454ea846dedb8eb5ed3c693660a93..0000000000000000000000000000000000000000 --- a/artifacts/results/selection_regret_head_metrics.csv +++ /dev/null @@ -1,241 +0,0 @@ -family,model_tag,scope,seed,selected_by,head_label,test_pr_auc,exact_f1,tolerated_f1,union_f1 -AlphaEarth,alphaearth,fire_prone,1,PR-AUC,shallow spatial adapter,0.04390614036305261,0.09715762273901808,0.7863702028280513,0.7863702028280513 -AlphaEarth,alphaearth,fire_prone,1,decision,shallow spatial adapter,,,,0.7863702028280513 -AlphaEarth,alphaearth,fire_prone,7,PR-AUC,shallow spatial adapter,0.05955397140893137,0.12514898688915374,0.8294499693616161,0.8294499693616161 -AlphaEarth,alphaearth,fire_prone,7,decision,shallow spatial adapter,,,,0.8294499693616161 -AlphaEarth,alphaearth,fire_prone,42,PR-AUC,shallow spatial adapter,0.038083070948941686,0.08702469619756958,0.7112901458230849,0.7112901458230849 -AlphaEarth,alphaearth,fire_prone,42,decision,pixel MLP head,,,,0.8461131676361712 -AlphaEarth,alphaearth,fire_prone,99,PR-AUC,shallow spatial adapter,0.0458102699699856,0.10365251727541955,0.7758298037709835,0.7758298037709835 -AlphaEarth,alphaearth,fire_prone,99,decision,pixel MLP head,,,,0.8350245452333586 -AlphaEarth,alphaearth,fire_prone,123,PR-AUC,shallow spatial adapter,0.045809049876129763,0.10531544957774468,0.7789089693560928,0.7789089693560928 -AlphaEarth,alphaearth,fire_prone,123,decision,shallow spatial adapter,,,,0.7789089693560928 -AlphaEarth,alphaearth,global,1,PR-AUC,shallow spatial adapter,0.0006549130347299629,0.004193290734824281,0.40561891947698747,0.40561891947698747 -AlphaEarth,alphaearth,global,1,decision,pixel MLP head,,,,0.6337627266658229 -AlphaEarth,alphaearth,global,7,PR-AUC,shallow spatial adapter,0.001005722733868245,0.010460251046025104,0.6184842128568402,0.6184842128568402 -AlphaEarth,alphaearth,global,7,decision,pixel MLP head,,,,0.6691395427484861 -AlphaEarth,alphaearth,global,42,PR-AUC,shallow spatial adapter,0.0005634701573991865,0.004809747755451047,0.4087444681515033,0.4087444681515033 -AlphaEarth,alphaearth,global,42,decision,pixel MLP head,,,,0.6812131506751973 -AlphaEarth,alphaearth,global,99,PR-AUC,shallow spatial adapter,0.0006577120081349608,0.006780481898534931,0.3921547570095426,0.3921547570095426 -AlphaEarth,alphaearth,global,99,decision,pixel MLP head,,,,0.5842714676652996 -AlphaEarth,alphaearth,global,123,PR-AUC,shallow spatial adapter,0.0007047712457371991,0.006959088991986505,0.4427625907752311,0.4427625907752311 -AlphaEarth,alphaearth,global,123,decision,pixel MLP head,,,,0.5604635792586619 -Aurora,aurora,fire_prone,1,PR-AUC,linear probe,0.02093558282208589,0.04101254412979794,0.7185324707231184,0.7185324707231184 -Aurora,aurora,fire_prone,1,decision,linear probe,,,,0.7185324707231184 -Aurora,aurora,fire_prone,7,PR-AUC,linear probe,0.024802820904513342,0.0413500618483831,0.7184857293868923,0.7184857293868923 -Aurora,aurora,fire_prone,7,decision,shallow spatial adapter,,,,0.7185324707231184 -Aurora,aurora,fire_prone,42,PR-AUC,linear probe,0.02613792907867929,0.04101254412979794,0.7185324707231184,0.7185324707231184 -Aurora,aurora,fire_prone,42,decision,linear probe,,,,0.7185324707231184 -Aurora,aurora,fire_prone,99,PR-AUC,linear probe,0.02093558282208589,0.0,0.0,0.0 -Aurora,aurora,fire_prone,99,decision,shallow spatial adapter,,,,0.7185324707231184 -Aurora,aurora,fire_prone,123,PR-AUC,pixel MLP head,0.03014151567817997,0.04541038665445361,0.7175172112337449,0.7175172112337449 -Aurora,aurora,fire_prone,123,decision,linear probe,,,,0.7185324707231184 -Aurora,aurora,global,1,PR-AUC,linear probe,0.00024254792826221397,0.00048497822606044473,0.23755358049655212,0.23755358049655212 -Aurora,aurora,global,1,decision,shallow spatial adapter,,,,0.240793572992212 -Aurora,aurora,global,7,PR-AUC,linear probe,0.00027660331739269843,0.0,0.23734400297937533,0.23734400297937533 -Aurora,aurora,global,7,decision,shallow spatial adapter,,,,0.240793572992212 -Aurora,aurora,global,42,PR-AUC,linear probe,0.0002876372030063385,0.00048497822606044473,0.0,0.0 -Aurora,aurora,global,42,decision,shallow spatial adapter,,,,0.240793572992212 -Aurora,aurora,global,99,PR-AUC,linear probe,0.00024254792826221397,0.0,0.0,0.0 -Aurora,aurora,global,99,decision,shallow spatial adapter,,,,0.240793572992212 -Aurora,aurora,global,123,PR-AUC,pixel MLP head,0.00031683315961488916,0.0005311562430574265,0.23647112940979162,0.23647112940979162 -Aurora,aurora,global,123,decision,shallow spatial adapter,,,,0.240793572992212 -ClimaX,climax,fire_prone,1,PR-AUC,linear probe,0.02281272244151735,0.04101254412979794,0.7185324707231184,0.7185324707231184 -ClimaX,climax,fire_prone,1,decision,linear probe,,,,0.7185324707231184 -ClimaX,climax,fire_prone,7,PR-AUC,linear probe,0.021317405351800135,0.04101254412979794,0.7185324707231184,0.7185324707231184 -ClimaX,climax,fire_prone,7,decision,linear probe,,,,0.7185324707231184 -ClimaX,climax,fire_prone,42,PR-AUC,linear probe,0.021516770872035896,0.04101254412979794,0.7185324707231184,0.7185324707231184 -ClimaX,climax,fire_prone,42,decision,linear probe,,,,0.7185324707231184 -ClimaX,climax,fire_prone,99,PR-AUC,shallow spatial adapter,0.02099123219536693,0.04101254412979794,0.7185324707231184,0.7185324707231184 -ClimaX,climax,fire_prone,99,decision,shallow spatial adapter,,,,0.7185324707231184 -ClimaX,climax,fire_prone,123,PR-AUC,shallow spatial adapter,0.024930358757410707,0.04101254412979794,0.7185324707231184,0.7185324707231184 -ClimaX,climax,fire_prone,123,decision,shallow spatial adapter,,,,0.7185324707231184 -ClimaX,climax,global,1,PR-AUC,linear probe,0.0002543464414550104,0.00048497822606044473,0.23755358049655212,0.23755358049655212 -ClimaX,climax,global,1,decision,shallow spatial adapter,,,,0.240793572992212 -ClimaX,climax,global,7,PR-AUC,pixel MLP head,0.00025423546642937565,0.00048497822606044473,0.23755358049655212,0.23755358049655212 -ClimaX,climax,global,7,decision,shallow spatial adapter,,,,0.240793572992212 -ClimaX,climax,global,42,PR-AUC,shallow spatial adapter,0.00023723426605001756,0.0004925501476206198,0.240793572992212,0.240793572992212 -ClimaX,climax,global,42,decision,shallow spatial adapter,,,,0.240793572992212 -ClimaX,climax,global,99,PR-AUC,shallow spatial adapter,0.0002340075376021003,0.00048696535779102983,0.2384111045733381,0.2384111045733381 -ClimaX,climax,global,99,decision,shallow spatial adapter,,,,0.2384111045733381 -ClimaX,climax,global,123,PR-AUC,shallow spatial adapter,0.00025952340213823634,0.0004925501476206198,0.240793572992212,0.240793572992212 -ClimaX,climax,global,123,decision,shallow spatial adapter,,,,0.240793572992212 -DLWP,dlwp,fire_prone,1,PR-AUC,linear probe,0.019747545663020845,0.043506471331489265,0.7280364139105968,0.7280364139105968 -DLWP,dlwp,fire_prone,1,decision,linear probe,,,,0.7280364139105968 -DLWP,dlwp,fire_prone,7,PR-AUC,pixel MLP head,0.018519739310339497,0.04101254412979794,0.7185324707231184,0.7185324707231184 -DLWP,dlwp,fire_prone,7,decision,shallow spatial adapter,,,,0.762536895087842 -DLWP,dlwp,fire_prone,42,PR-AUC,pixel MLP head,0.020762153794205103,0.04637177602565815,0.6606900017471591,0.6606900017471591 -DLWP,dlwp,fire_prone,42,decision,shallow spatial adapter,,,,0.7728400679088107 -DLWP,dlwp,fire_prone,99,PR-AUC,linear probe,0.02136633936888583,0.04819843096725701,0.7187730423241409,0.7187730423241409 -DLWP,dlwp,fire_prone,99,decision,shallow spatial adapter,,,,0.7653346239087763 -DLWP,dlwp,fire_prone,123,PR-AUC,pixel MLP head,0.021118517500793188,0.04101254412979794,0.7185324707231184,0.7185324707231184 -DLWP,dlwp,fire_prone,123,decision,linear probe,,,,0.7321459738801157 -DLWP,dlwp,global,1,PR-AUC,shallow spatial adapter,0.0006257446338466172,0.00487022180273714,0.38023285660836226,0.38023285660836226 -DLWP,dlwp,global,1,decision,shallow spatial adapter,,,,0.38023285660836226 -DLWP,dlwp,global,7,PR-AUC,shallow spatial adapter,0.0005264872646085452,0.0,0.3432315705541329,0.3432315705541329 -DLWP,dlwp,global,7,decision,shallow spatial adapter,,,,0.3432315705541329 -DLWP,dlwp,global,42,PR-AUC,shallow spatial adapter,0.0006203713852571992,0.0,0.3405125814370199,0.3405125814370199 -DLWP,dlwp,global,42,decision,shallow spatial adapter,,,,0.3405125814370199 -DLWP,dlwp,global,99,PR-AUC,shallow spatial adapter,0.0007477128447471452,0.0,0.3979559626836394,0.3979559626836394 -DLWP,dlwp,global,99,decision,shallow spatial adapter,,,,0.3979559626836394 -DLWP,dlwp,global,123,PR-AUC,shallow spatial adapter,0.0007129763973023342,0.0,0.3797689460796109,0.3797689460796109 -DLWP,dlwp,global,123,decision,shallow spatial adapter,,,,0.3797689460796109 -FCN,fcn,fire_prone,1,PR-AUC,shallow spatial adapter,0.01844011667219625,0.042068766252528166,0.7185324707231184,0.7185324707231184 -FCN,fcn,fire_prone,1,decision,linear probe,,,,0.7182175622542595 -FCN,fcn,fire_prone,7,PR-AUC,pixel MLP head,0.02050876208409485,0.04101254412979794,0.7185324707231184,0.7185324707231184 -FCN,fcn,fire_prone,7,decision,linear probe,,,,0.7644129739607127 -FCN,fcn,fire_prone,42,PR-AUC,shallow spatial adapter,0.018030815062946615,0.0414596444738876,0.7197180735022655,0.7197180735022655 -FCN,fcn,fire_prone,42,decision,shallow spatial adapter,,,,0.7197180735022655 -FCN,fcn,fire_prone,99,PR-AUC,linear probe,0.029098665712304895,0.042822140550172624,0.726408418760773,0.726408418760773 -FCN,fcn,fire_prone,99,decision,linear probe,,,,0.726408418760773 -FCN,fcn,fire_prone,123,PR-AUC,pixel MLP head,0.019943278646881796,0.04101254412979794,0.7185324707231184,0.7185324707231184 -FCN,fcn,fire_prone,123,decision,linear probe,,,,0.7310509974227326 -FCN,fcn,global,1,PR-AUC,shallow spatial adapter,0.00037256097806901117,0.0009319664492078285,0.31167484413093016,0.31167484413093016 -FCN,fcn,global,1,decision,shallow spatial adapter,,,,0.31167484413093016 -FCN,fcn,global,7,PR-AUC,shallow spatial adapter,0.0003268363416054406,0.001086071137659517,0.3051941376005135,0.3051941376005135 -FCN,fcn,global,7,decision,shallow spatial adapter,,,,0.3051941376005135 -FCN,fcn,global,42,PR-AUC,shallow spatial adapter,0.00041063897933390575,0.0007027406886858749,0.31987973649439366,0.31987973649439366 -FCN,fcn,global,42,decision,pixel MLP head,,,,0.2870596305028149 -FCN,fcn,global,99,PR-AUC,shallow spatial adapter,0.00038120453362995967,0.0018159806295399514,0.3054145960271247,0.3054145960271247 -FCN,fcn,global,99,decision,shallow spatial adapter,,,,0.3054145960271247 -FCN,fcn,global,123,PR-AUC,shallow spatial adapter,0.000387412312932838,0.0006535947712418301,0.3096850885545486,0.3096850885545486 -FCN,fcn,global,123,decision,shallow spatial adapter,,,,0.3096850885545486 -FengWu,fengwu,fire_prone,1,PR-AUC,shallow spatial adapter,0.022736128443885992,0.04452825597664091,0.7269980510116651,0.7269980510116651 -FengWu,fengwu,fire_prone,1,decision,shallow spatial adapter,,,,0.7269980510116651 -FengWu,fengwu,fire_prone,7,PR-AUC,pixel MLP head,0.016801706970978273,0.04101254412979794,0.7185324707231184,0.7185324707231184 -FengWu,fengwu,fire_prone,7,decision,shallow spatial adapter,,,,0.722901134194411 -FengWu,fengwu,fire_prone,42,PR-AUC,shallow spatial adapter,0.021165185967854567,0.04574838388861263,0.7265705731122933,0.7265705731122933 -FengWu,fengwu,fire_prone,42,decision,shallow spatial adapter,,,,0.7265705731122933 -FengWu,fengwu,fire_prone,99,PR-AUC,linear probe,0.02199779031689959,0.0,0.7185324707231184,0.7185324707231184 -FengWu,fengwu,fire_prone,99,decision,shallow spatial adapter,,,,0.7336829717908426 -FengWu,fengwu,fire_prone,123,PR-AUC,pixel MLP head,0.020495144103834257,0.04101254412979794,0.7185324707231184,0.7185324707231184 -FengWu,fengwu,fire_prone,123,decision,shallow spatial adapter,,,,0.7251252524331628 -FengWu,fengwu,global,1,PR-AUC,shallow spatial adapter,0.000398077435365184,0.0,0.31071390711162444,0.31071390711162444 -FengWu,fengwu,global,1,decision,shallow spatial adapter,,,,0.31071390711162444 -FengWu,fengwu,global,7,PR-AUC,shallow spatial adapter,0.00036963872610239895,0.0,0.30641904273669174,0.30641904273669174 -FengWu,fengwu,global,7,decision,shallow spatial adapter,,,,0.30641904273669174 -FengWu,fengwu,global,42,PR-AUC,shallow spatial adapter,0.00036987379624400454,0.0,0.31219058559732665,0.31219058559732665 -FengWu,fengwu,global,42,decision,shallow spatial adapter,,,,0.31219058559732665 -FengWu,fengwu,global,99,PR-AUC,shallow spatial adapter,0.00042782651526874734,0.0,0.3111214819309595,0.3111214819309595 -FengWu,fengwu,global,99,decision,shallow spatial adapter,,,,0.3111214819309595 -FengWu,fengwu,global,123,PR-AUC,shallow spatial adapter,0.0004116035724473925,0.0,0.3145618361221859,0.3145618361221859 -FengWu,fengwu,global,123,decision,shallow spatial adapter,,,,0.3145618361221859 -FuXi,fuxi,fire_prone,1,PR-AUC,shallow spatial adapter,0.01904942261850253,0.043196160341303,0.7246636456247585,0.7246636456247585 -FuXi,fuxi,fire_prone,1,decision,linear probe,,,,0.7261195534617713 -FuXi,fuxi,fire_prone,7,PR-AUC,shallow spatial adapter,0.018717347905238046,0.044572342126298965,0.7235687421646468,0.7235687421646468 -FuXi,fuxi,fire_prone,7,decision,shallow spatial adapter,,,,0.7235687421646468 -FuXi,fuxi,fire_prone,42,PR-AUC,shallow spatial adapter,0.020363596550511454,0.04603830266616599,0.7054843984273774,0.7054843984273774 -FuXi,fuxi,fire_prone,42,decision,shallow spatial adapter,,,,0.7054843984273774 -FuXi,fuxi,fire_prone,99,PR-AUC,pixel MLP head,0.02225455497934009,0.030284377692970574,0.7203102915557309,0.7203102915557309 -FuXi,fuxi,fire_prone,99,decision,shallow spatial adapter,,,,0.7183426482806338 -FuXi,fuxi,fire_prone,123,PR-AUC,pixel MLP head,0.021045021724179047,0.018543768748295608,0.7185324707231184,0.7185324707231184 -FuXi,fuxi,fire_prone,123,decision,linear probe,,,,0.7224961571477053 -FuXi,fuxi,global,1,PR-AUC,shallow spatial adapter,0.0003596048414560045,0.0006866311182961118,0.3091927111996169,0.3091927111996169 -FuXi,fuxi,global,1,decision,linear probe,,,,0.21516044416153002 -FuXi,fuxi,global,7,PR-AUC,shallow spatial adapter,0.00037118462783325537,0.0008450613619815959,0.3062654921252336,0.3062654921252336 -FuXi,fuxi,global,7,decision,shallow spatial adapter,,,,0.3062654921252336 -FuXi,fuxi,global,42,PR-AUC,shallow spatial adapter,0.0003830459807690152,0.0003188165529554294,0.316668570748256,0.316668570748256 -FuXi,fuxi,global,42,decision,shallow spatial adapter,,,,0.316668570748256 -FuXi,fuxi,global,99,PR-AUC,shallow spatial adapter,0.0003758107890486513,0.0007771032641515431,0.3108110703043797,0.3108110703043797 -FuXi,fuxi,global,99,decision,shallow spatial adapter,,,,0.3108110703043797 -FuXi,fuxi,global,123,PR-AUC,shallow spatial adapter,0.00039999784430104664,0.0009888839378897987,0.3096924811079745,0.3096924811079745 -FuXi,fuxi,global,123,decision,shallow spatial adapter,,,,0.3096924811079745 -Pangu-Weather,pangu_weather,fire_prone,1,PR-AUC,pixel MLP head,0.014813375233921266,0.04101254412979794,0.7185324707231184,0.7185324707231184 -Pangu-Weather,pangu_weather,fire_prone,1,decision,pixel MLP head,,,,0.7185324707231184 -Pangu-Weather,pangu_weather,fire_prone,7,PR-AUC,pixel MLP head,0.029883397445312192,0.0,0.0,0.0 -Pangu-Weather,pangu_weather,fire_prone,7,decision,shallow spatial adapter,,,,0.7185324707231184 -Pangu-Weather,pangu_weather,fire_prone,42,PR-AUC,pixel MLP head,0.02188826028256454,0.0,0.0,0.0 -Pangu-Weather,pangu_weather,fire_prone,42,decision,shallow spatial adapter,,,,0.7185324707231184 -Pangu-Weather,pangu_weather,fire_prone,99,PR-AUC,linear probe,0.02093558282208589,0.0,0.0,0.0 -Pangu-Weather,pangu_weather,fire_prone,99,decision,shallow spatial adapter,,,,0.7185324707231184 -Pangu-Weather,pangu_weather,fire_prone,123,PR-AUC,linear probe,0.02093558282208589,0.04101254412979794,0.7185324707231184,0.7185324707231184 -Pangu-Weather,pangu_weather,fire_prone,123,decision,linear probe,,,,0.7185324707231184 -Pangu-Weather,pangu_weather,global,1,PR-AUC,linear probe,0.00024254792826221397,0.00048497822606044473,0.23755358049655212,0.23755358049655212 -Pangu-Weather,pangu_weather,global,1,decision,shallow spatial adapter,,,,0.240793572992212 -Pangu-Weather,pangu_weather,global,7,PR-AUC,shallow spatial adapter,0.00028045576992618193,0.0004925501476206198,0.240793572992212,0.240793572992212 -Pangu-Weather,pangu_weather,global,7,decision,shallow spatial adapter,,,,0.240793572992212 -Pangu-Weather,pangu_weather,global,42,PR-AUC,pixel MLP head,0.00021817709863716954,0.0,0.0,0.0 -Pangu-Weather,pangu_weather,global,42,decision,shallow spatial adapter,,,,0.240793572992212 -Pangu-Weather,pangu_weather,global,99,PR-AUC,shallow spatial adapter,0.0003408299850116487,0.0004887421693148924,0.23917716245227552,0.23917716245227552 -Pangu-Weather,pangu_weather,global,99,decision,pixel MLP head,,,,0.229804399271568 -Pangu-Weather,pangu_weather,global,123,PR-AUC,shallow spatial adapter,0.0003300754798792237,0.0004925501476206198,0.240793572992212,0.240793572992212 -Pangu-Weather,pangu_weather,global,123,decision,shallow spatial adapter,,,,0.240793572992212 -Pangu-Weather,pangu6,fire_prone,1,PR-AUC,shallow spatial adapter,0.02328829578760066,0.04418397717161179,0.7231388400090598,0.7231388400090598 -Pangu-Weather,pangu6,fire_prone,1,decision,shallow spatial adapter,,,,0.7231388400090598 -Pangu-Weather,pangu6,fire_prone,7,PR-AUC,pixel MLP head,0.018664183428008234,0.04101254412979794,0.7185324707231184,0.7185324707231184 -Pangu-Weather,pangu6,fire_prone,7,decision,shallow spatial adapter,,,,0.719461376864001 -Pangu-Weather,pangu6,fire_prone,42,PR-AUC,shallow spatial adapter,0.021420904714594617,0.0430065972893146,0.7241279907754397,0.7241279907754397 -Pangu-Weather,pangu6,fire_prone,42,decision,shallow spatial adapter,,,,0.7241279907754397 -Pangu-Weather,pangu6,fire_prone,99,PR-AUC,linear probe,0.024740444457605402,0.0044004400440044,0.7183382629739177,0.7183382629739177 -Pangu-Weather,pangu6,fire_prone,99,decision,shallow spatial adapter,,,,0.719015307962107 -Pangu-Weather,pangu6,fire_prone,123,PR-AUC,pixel MLP head,0.02254610440161657,0.04101254412979794,0.7185324707231184,0.7185324707231184 -Pangu-Weather,pangu6,fire_prone,123,decision,linear probe,,,,0.7261721555350364 -Pangu-Weather,pangu6,global,1,PR-AUC,shallow spatial adapter,0.00042881385578730365,0.0,0.32066974948800614,0.32066974948800614 -Pangu-Weather,pangu6,global,1,decision,shallow spatial adapter,,,,0.32066974948800614 -Pangu-Weather,pangu6,global,7,PR-AUC,shallow spatial adapter,0.00038395193539280824,0.0,0.31120670593952293,0.31120670593952293 -Pangu-Weather,pangu6,global,7,decision,shallow spatial adapter,,,,0.31120670593952293 -Pangu-Weather,pangu6,global,42,PR-AUC,shallow spatial adapter,0.00040651309469793043,0.0,0.32154249424354786,0.32154249424354786 -Pangu-Weather,pangu6,global,42,decision,shallow spatial adapter,,,,0.32154249424354786 -Pangu-Weather,pangu6,global,99,PR-AUC,shallow spatial adapter,0.0004450373086921994,0.0,0.3214547875801752,0.3214547875801752 -Pangu-Weather,pangu6,global,99,decision,shallow spatial adapter,,,,0.3214547875801752 -Pangu-Weather,pangu6,global,123,PR-AUC,shallow spatial adapter,0.00043738577276574255,0.0,0.31812983009681495,0.31812983009681495 -Pangu-Weather,pangu6,global,123,decision,shallow spatial adapter,,,,0.31812983009681495 -Prithvi-WxC,prithvi_wxc,fire_prone,1,PR-AUC,shallow spatial adapter,0.02093558282208589,0.04101254412979794,0.7185324707231184,0.7185324707231184 -Prithvi-WxC,prithvi_wxc,fire_prone,1,decision,shallow spatial adapter,,,,0.7185324707231184 -Prithvi-WxC,prithvi_wxc,fire_prone,7,PR-AUC,shallow spatial adapter,0.02093558282208589,0.04101254412979794,0.7185324707231184,0.7185324707231184 -Prithvi-WxC,prithvi_wxc,fire_prone,7,decision,shallow spatial adapter,,,,0.7185324707231184 -Prithvi-WxC,prithvi_wxc,fire_prone,42,PR-AUC,pixel MLP head,0.02093558282208589,0.04101254412979794,0.7185324707231184,0.7185324707231184 -Prithvi-WxC,prithvi_wxc,fire_prone,42,decision,shallow spatial adapter,,,,0.7185324707231184 -Prithvi-WxC,prithvi_wxc,fire_prone,99,PR-AUC,linear probe,0.02093558282208589,0.04101254412979794,0.7185324707231184,0.7185324707231184 -Prithvi-WxC,prithvi_wxc,fire_prone,99,decision,shallow spatial adapter,,,,0.7185324707231184 -Prithvi-WxC,prithvi_wxc,fire_prone,123,PR-AUC,shallow spatial adapter,0.02093558282208589,0.04101254412979794,0.7185324707231184,0.7185324707231184 -Prithvi-WxC,prithvi_wxc,fire_prone,123,decision,shallow spatial adapter,,,,0.7185324707231184 -Prithvi-WxC,prithvi_wxc,global,1,PR-AUC,shallow spatial adapter,0.0002463357401629007,0.0004925501476206198,0.240793572992212,0.240793572992212 -Prithvi-WxC,prithvi_wxc,global,1,decision,shallow spatial adapter,,,,0.240793572992212 -Prithvi-WxC,prithvi_wxc,global,7,PR-AUC,shallow spatial adapter,0.0002463357401629007,0.0004925501476206198,0.240793572992212,0.240793572992212 -Prithvi-WxC,prithvi_wxc,global,7,decision,shallow spatial adapter,,,,0.240793572992212 -Prithvi-WxC,prithvi_wxc,global,42,PR-AUC,shallow spatial adapter,0.0002463357401629007,0.0004925501476206198,0.240793572992212,0.240793572992212 -Prithvi-WxC,prithvi_wxc,global,42,decision,shallow spatial adapter,,,,0.240793572992212 -Prithvi-WxC,prithvi_wxc,global,99,PR-AUC,shallow spatial adapter,0.0002454330184381399,0.00048497822606044473,0.23755358049655212,0.23755358049655212 -Prithvi-WxC,prithvi_wxc,global,99,decision,shallow spatial adapter,,,,0.23755358049655212 -Prithvi-WxC,prithvi_wxc,global,123,PR-AUC,shallow spatial adapter,0.0002463357401629007,0.0004925501476206198,0.240793572992212,0.240793572992212 -Prithvi-WxC,prithvi_wxc,global,123,decision,shallow spatial adapter,,,,0.240793572992212 -Reference,reference,fire_prone,1,PR-AUC,shallow spatial adapter,0.10204224118176683,0.1611624834874505,0.799032457577039,0.799032457577039 -Reference,reference,fire_prone,1,decision,linear probe,,,,0.8685988450931266 -Reference,reference,fire_prone,7,PR-AUC,shallow spatial adapter,0.1323902067230726,0.22885572139303484,0.8027807546359551,0.8027807546359551 -Reference,reference,fire_prone,7,decision,shallow spatial adapter,,,,0.8027807546359551 -Reference,reference,fire_prone,42,PR-AUC,shallow spatial adapter,0.12048427320762313,0.19225806451612906,0.8358151221553631,0.8358151221553631 -Reference,reference,fire_prone,42,decision,shallow spatial adapter,,,,0.8358151221553631 -Reference,reference,fire_prone,99,PR-AUC,shallow spatial adapter,0.11947246238005743,0.19477124183006533,0.8534759193943048,0.8534759193943048 -Reference,reference,fire_prone,99,decision,linear probe,,,,0.9039923296574045 -Reference,reference,fire_prone,123,PR-AUC,shallow spatial adapter,0.11551882470432043,0.19165378670788252,0.8066689866810086,0.8066689866810086 -Reference,reference,fire_prone,123,decision,pixel MLP head,,,,0.8567215417854325 -Reference,reference,global,1,PR-AUC,shallow spatial adapter,0.002624341503354088,0.017134572294714646,0.6186566066408326,0.6186566066408326 -Reference,reference,global,1,decision,linear probe,,,,0.7286109603326707 -Reference,reference,global,7,PR-AUC,shallow spatial adapter,0.0030317345997110698,0.02287675150128682,0.5002193417313746,0.5002193417313746 -Reference,reference,global,7,decision,shallow spatial adapter,,,,0.5002193417313746 -Reference,reference,global,42,PR-AUC,shallow spatial adapter,0.0032729435045747413,0.025515210991167808,0.7239943741463363,0.7239943741463363 -Reference,reference,global,42,decision,shallow spatial adapter,,,,0.7239943741463363 -Reference,reference,global,99,PR-AUC,shallow spatial adapter,0.0031953098868323193,0.021876258220373104,0.6698005926442897,0.6698005926442897 -Reference,reference,global,99,decision,linear probe,,,,0.7647466876509988 -Reference,reference,global,123,PR-AUC,shallow spatial adapter,0.0025603887793133394,0.019320660641944532,0.5009707461135111,0.5009707461135111 -Reference,reference,global,123,decision,pixel MLP head,,,,0.735221546471909 -StormCast,stormcast,fire_prone,1,PR-AUC,linear probe,0.02093558282208589,0.04101254412979794,0.7185324707231184,0.7185324707231184 -StormCast,stormcast,fire_prone,1,decision,shallow spatial adapter,,,,0.7185324707231184 -StormCast,stormcast,fire_prone,7,PR-AUC,linear probe,0.02093558282208589,0.04101254412979794,0.7185324707231184,0.7185324707231184 -StormCast,stormcast,fire_prone,7,decision,shallow spatial adapter,,,,0.7185324707231184 -StormCast,stormcast,fire_prone,42,PR-AUC,linear probe,0.02093558282208589,0.04101254412979794,0.7185324707231184,0.7185324707231184 -StormCast,stormcast,fire_prone,42,decision,shallow spatial adapter,,,,0.7185324707231184 -StormCast,stormcast,fire_prone,99,PR-AUC,pixel MLP head,0.02093558282208589,0.04101254412979794,0.7185324707231184,0.7185324707231184 -StormCast,stormcast,fire_prone,99,decision,shallow spatial adapter,,,,0.7185324707231184 -StormCast,stormcast,fire_prone,123,PR-AUC,linear probe,0.02093558282208589,0.04101254412979794,0.7185324707231184,0.7185324707231184 -StormCast,stormcast,fire_prone,123,decision,shallow spatial adapter,,,,0.7185324707231184 -StormCast,stormcast,global,1,PR-AUC,shallow spatial adapter,0.0002463357401629007,0.0004925501476206198,0.240793572992212,0.240793572992212 -StormCast,stormcast,global,1,decision,shallow spatial adapter,,,,0.240793572992212 -StormCast,stormcast,global,7,PR-AUC,shallow spatial adapter,0.0002463357401629007,0.0004925501476206198,0.240793572992212,0.240793572992212 -StormCast,stormcast,global,7,decision,shallow spatial adapter,,,,0.240793572992212 -StormCast,stormcast,global,42,PR-AUC,shallow spatial adapter,0.0002463357401629007,0.0004925501476206198,0.240793572992212,0.240793572992212 -StormCast,stormcast,global,42,decision,shallow spatial adapter,,,,0.240793572992212 -StormCast,stormcast,global,99,PR-AUC,shallow spatial adapter,0.0002463357401629007,0.0004905319945803844,0.2399251001974223,0.2399251001974223 -StormCast,stormcast,global,99,decision,shallow spatial adapter,,,,0.2399251001974223 -StormCast,stormcast,global,123,PR-AUC,shallow spatial adapter,0.0002463357401629007,0.0004925501476206198,0.240793572992212,0.240793572992212 -StormCast,stormcast,global,123,decision,shallow spatial adapter,,,,0.240793572992212 diff --git a/artifacts/results/selection_regret_per_seed.csv b/artifacts/results/selection_regret_per_seed.csv deleted file mode 100644 index c6166c5bb3e2d4d3511716055b7615ee11e4e624..0000000000000000000000000000000000000000 --- a/artifacts/results/selection_regret_per_seed.csv +++ /dev/null @@ -1,121 +0,0 @@ -family,model_tag,scope,seed,ranking_head,decision_head,ranking_pr_auc,ranking_union_f1,decision_union_f1,regret,top1_agreement -AlphaEarth,alphaearth,fire_prone,1,shallow spatial adapter,shallow spatial adapter,0.04390614036305261,0.7863702028280513,0.7863702028280513,0.0,True -AlphaEarth,alphaearth,fire_prone,7,shallow spatial adapter,shallow spatial adapter,0.05955397140893137,0.8294499693616161,0.8294499693616161,0.0,True -AlphaEarth,alphaearth,fire_prone,42,shallow spatial adapter,pixel MLP head,0.038083070948941686,0.7112901458230849,0.8461131676361712,0.13482302181308625,False -AlphaEarth,alphaearth,fire_prone,99,shallow spatial adapter,pixel MLP head,0.0458102699699856,0.7758298037709835,0.8350245452333586,0.05919474146237502,False -AlphaEarth,alphaearth,fire_prone,123,shallow spatial adapter,shallow spatial adapter,0.045809049876129763,0.7789089693560928,0.7789089693560928,0.0,True -AlphaEarth,alphaearth,global,1,shallow spatial adapter,pixel MLP head,0.0006549130347299629,0.40561891947698747,0.6337627266658229,0.22814380718883542,False -AlphaEarth,alphaearth,global,7,shallow spatial adapter,pixel MLP head,0.001005722733868245,0.6184842128568402,0.6691395427484861,0.050655329891645895,False -AlphaEarth,alphaearth,global,42,shallow spatial adapter,pixel MLP head,0.0005634701573991865,0.4087444681515033,0.6812131506751973,0.272468682523694,False -AlphaEarth,alphaearth,global,99,shallow spatial adapter,pixel MLP head,0.0006577120081349608,0.3921547570095426,0.5842714676652996,0.19211671065575697,False -AlphaEarth,alphaearth,global,123,shallow spatial adapter,pixel MLP head,0.0007047712457371991,0.4427625907752311,0.5604635792586619,0.11770098848343075,False -Aurora,aurora,fire_prone,1,linear probe,linear probe,0.02093558282208589,0.7185324707231184,0.7185324707231184,0.0,True -Aurora,aurora,fire_prone,7,linear probe,shallow spatial adapter,0.024802820904513342,0.7184857293868923,0.7185324707231184,4.674133622617482e-05,False -Aurora,aurora,fire_prone,42,linear probe,linear probe,0.02613792907867929,0.7185324707231184,0.7185324707231184,0.0,True -Aurora,aurora,fire_prone,99,linear probe,shallow spatial adapter,0.02093558282208589,0.0,0.7185324707231184,0.7185324707231184,False -Aurora,aurora,fire_prone,123,pixel MLP head,linear probe,0.03014151567817997,0.7175172112337449,0.7185324707231184,0.0010152594893735323,False -Aurora,aurora,global,1,linear probe,shallow spatial adapter,0.00024254792826221397,0.23755358049655212,0.240793572992212,0.0032399924956598714,False -Aurora,aurora,global,7,linear probe,shallow spatial adapter,0.00027660331739269843,0.23734400297937533,0.240793572992212,0.0034495700128366613,False -Aurora,aurora,global,42,linear probe,shallow spatial adapter,0.0002876372030063385,0.0,0.240793572992212,0.240793572992212,False -Aurora,aurora,global,99,linear probe,shallow spatial adapter,0.00024254792826221397,0.0,0.240793572992212,0.240793572992212,False -Aurora,aurora,global,123,pixel MLP head,shallow spatial adapter,0.00031683315961488916,0.23647112940979162,0.240793572992212,0.004322443582420371,False -ClimaX,climax,fire_prone,1,linear probe,linear probe,0.02281272244151735,0.7185324707231184,0.7185324707231184,0.0,True -ClimaX,climax,fire_prone,7,linear probe,linear probe,0.021317405351800135,0.7185324707231184,0.7185324707231184,0.0,True -ClimaX,climax,fire_prone,42,linear probe,linear probe,0.021516770872035896,0.7185324707231184,0.7185324707231184,0.0,True -ClimaX,climax,fire_prone,99,shallow spatial adapter,shallow spatial adapter,0.02099123219536693,0.7185324707231184,0.7185324707231184,0.0,True -ClimaX,climax,fire_prone,123,shallow spatial adapter,shallow spatial adapter,0.024930358757410707,0.7185324707231184,0.7185324707231184,0.0,True -ClimaX,climax,global,1,linear probe,shallow spatial adapter,0.0002543464414550104,0.23755358049655212,0.240793572992212,0.0032399924956598714,False -ClimaX,climax,global,7,pixel MLP head,shallow spatial adapter,0.00025423546642937565,0.23755358049655212,0.240793572992212,0.0032399924956598714,False -ClimaX,climax,global,42,shallow spatial adapter,shallow spatial adapter,0.00023723426605001756,0.240793572992212,0.240793572992212,0.0,True -ClimaX,climax,global,99,shallow spatial adapter,shallow spatial adapter,0.0002340075376021003,0.2384111045733381,0.2384111045733381,0.0,True -ClimaX,climax,global,123,shallow spatial adapter,shallow spatial adapter,0.00025952340213823634,0.240793572992212,0.240793572992212,0.0,True -DLWP,dlwp,fire_prone,1,linear probe,linear probe,0.019747545663020845,0.7280364139105968,0.7280364139105968,0.0,True -DLWP,dlwp,fire_prone,7,pixel MLP head,shallow spatial adapter,0.018519739310339497,0.7185324707231184,0.762536895087842,0.044004424364723516,False -DLWP,dlwp,fire_prone,42,pixel MLP head,shallow spatial adapter,0.020762153794205103,0.6606900017471591,0.7728400679088107,0.11215006616165157,False -DLWP,dlwp,fire_prone,99,linear probe,shallow spatial adapter,0.02136633936888583,0.7187730423241409,0.7653346239087763,0.046561581584635414,False -DLWP,dlwp,fire_prone,123,pixel MLP head,linear probe,0.021118517500793188,0.7185324707231184,0.7321459738801157,0.013613503156997275,False -DLWP,dlwp,global,1,shallow spatial adapter,shallow spatial adapter,0.0006257446338466172,0.38023285660836226,0.38023285660836226,0.0,True -DLWP,dlwp,global,7,shallow spatial adapter,shallow spatial adapter,0.0005264872646085452,0.3432315705541329,0.3432315705541329,0.0,True -DLWP,dlwp,global,42,shallow spatial adapter,shallow spatial adapter,0.0006203713852571992,0.3405125814370199,0.3405125814370199,0.0,True -DLWP,dlwp,global,99,shallow spatial adapter,shallow spatial adapter,0.0007477128447471452,0.3979559626836394,0.3979559626836394,0.0,True -DLWP,dlwp,global,123,shallow spatial adapter,shallow spatial adapter,0.0007129763973023342,0.3797689460796109,0.3797689460796109,0.0,True -FCN,fcn,fire_prone,1,shallow spatial adapter,linear probe,0.01844011667219625,0.7185324707231184,0.7182175622542595,0.0,False -FCN,fcn,fire_prone,7,pixel MLP head,linear probe,0.02050876208409485,0.7185324707231184,0.7644129739607127,0.045880503237594294,False -FCN,fcn,fire_prone,42,shallow spatial adapter,shallow spatial adapter,0.018030815062946615,0.7197180735022655,0.7197180735022655,0.0,True -FCN,fcn,fire_prone,99,linear probe,linear probe,0.029098665712304895,0.726408418760773,0.726408418760773,0.0,True -FCN,fcn,fire_prone,123,pixel MLP head,linear probe,0.019943278646881796,0.7185324707231184,0.7310509974227326,0.012518526699614174,False -FCN,fcn,global,1,shallow spatial adapter,shallow spatial adapter,0.00037256097806901117,0.31167484413093016,0.31167484413093016,0.0,True -FCN,fcn,global,7,shallow spatial adapter,shallow spatial adapter,0.0003268363416054406,0.3051941376005135,0.3051941376005135,0.0,True -FCN,fcn,global,42,shallow spatial adapter,pixel MLP head,0.00041063897933390575,0.31987973649439366,0.2870596305028149,0.0,False -FCN,fcn,global,99,shallow spatial adapter,shallow spatial adapter,0.00038120453362995967,0.3054145960271247,0.3054145960271247,0.0,True -FCN,fcn,global,123,shallow spatial adapter,shallow spatial adapter,0.000387412312932838,0.3096850885545486,0.3096850885545486,0.0,True -FengWu,fengwu,fire_prone,1,shallow spatial adapter,shallow spatial adapter,0.022736128443885992,0.7269980510116651,0.7269980510116651,0.0,True -FengWu,fengwu,fire_prone,7,pixel MLP head,shallow spatial adapter,0.016801706970978273,0.7185324707231184,0.722901134194411,0.004368663471292611,False -FengWu,fengwu,fire_prone,42,shallow spatial adapter,shallow spatial adapter,0.021165185967854567,0.7265705731122933,0.7265705731122933,0.0,True -FengWu,fengwu,fire_prone,99,linear probe,shallow spatial adapter,0.02199779031689959,0.7185324707231184,0.7336829717908426,0.015150501067724198,False -FengWu,fengwu,fire_prone,123,pixel MLP head,shallow spatial adapter,0.020495144103834257,0.7185324707231184,0.7251252524331628,0.006592781710044404,False -FengWu,fengwu,global,1,shallow spatial adapter,shallow spatial adapter,0.000398077435365184,0.31071390711162444,0.31071390711162444,0.0,True -FengWu,fengwu,global,7,shallow spatial adapter,shallow spatial adapter,0.00036963872610239895,0.30641904273669174,0.30641904273669174,0.0,True -FengWu,fengwu,global,42,shallow spatial adapter,shallow spatial adapter,0.00036987379624400454,0.31219058559732665,0.31219058559732665,0.0,True -FengWu,fengwu,global,99,shallow spatial adapter,shallow spatial adapter,0.00042782651526874734,0.3111214819309595,0.3111214819309595,0.0,True -FengWu,fengwu,global,123,shallow spatial adapter,shallow spatial adapter,0.0004116035724473925,0.3145618361221859,0.3145618361221859,0.0,True -FuXi,fuxi,fire_prone,1,shallow spatial adapter,linear probe,0.01904942261850253,0.7246636456247585,0.7261195534617713,0.001455907837012771,False -FuXi,fuxi,fire_prone,7,shallow spatial adapter,shallow spatial adapter,0.018717347905238046,0.7235687421646468,0.7235687421646468,0.0,True -FuXi,fuxi,fire_prone,42,shallow spatial adapter,shallow spatial adapter,0.020363596550511454,0.7054843984273774,0.7054843984273774,0.0,True -FuXi,fuxi,fire_prone,99,pixel MLP head,shallow spatial adapter,0.02225455497934009,0.7203102915557309,0.7183426482806338,0.0,False -FuXi,fuxi,fire_prone,123,pixel MLP head,linear probe,0.021045021724179047,0.7185324707231184,0.7224961571477053,0.0039636864245868875,False -FuXi,fuxi,global,1,shallow spatial adapter,linear probe,0.0003596048414560045,0.3091927111996169,0.21516044416153002,0.0,False -FuXi,fuxi,global,7,shallow spatial adapter,shallow spatial adapter,0.00037118462783325537,0.3062654921252336,0.3062654921252336,0.0,True -FuXi,fuxi,global,42,shallow spatial adapter,shallow spatial adapter,0.0003830459807690152,0.316668570748256,0.316668570748256,0.0,True -FuXi,fuxi,global,99,shallow spatial adapter,shallow spatial adapter,0.0003758107890486513,0.3108110703043797,0.3108110703043797,0.0,True -FuXi,fuxi,global,123,shallow spatial adapter,shallow spatial adapter,0.00039999784430104664,0.3096924811079745,0.3096924811079745,0.0,True -Pangu-Weather,pangu_weather,fire_prone,1,pixel MLP head,pixel MLP head,0.014813375233921266,0.7185324707231184,0.7185324707231184,0.0,True -Pangu-Weather,pangu_weather,fire_prone,7,pixel MLP head,shallow spatial adapter,0.029883397445312192,0.0,0.7185324707231184,0.7185324707231184,False -Pangu-Weather,pangu_weather,fire_prone,42,pixel MLP head,shallow spatial adapter,0.02188826028256454,0.0,0.7185324707231184,0.7185324707231184,False -Pangu-Weather,pangu_weather,fire_prone,99,linear probe,shallow spatial adapter,0.02093558282208589,0.0,0.7185324707231184,0.7185324707231184,False -Pangu-Weather,pangu_weather,fire_prone,123,linear probe,linear probe,0.02093558282208589,0.7185324707231184,0.7185324707231184,0.0,True -Pangu-Weather,pangu_weather,global,1,linear probe,shallow spatial adapter,0.00024254792826221397,0.23755358049655212,0.240793572992212,0.0032399924956598714,False -Pangu-Weather,pangu_weather,global,7,shallow spatial adapter,shallow spatial adapter,0.00028045576992618193,0.240793572992212,0.240793572992212,0.0,True -Pangu-Weather,pangu_weather,global,42,pixel MLP head,shallow spatial adapter,0.00021817709863716954,0.0,0.240793572992212,0.240793572992212,False -Pangu-Weather,pangu_weather,global,99,shallow spatial adapter,pixel MLP head,0.0003408299850116487,0.23917716245227552,0.229804399271568,0.0,False -Pangu-Weather,pangu_weather,global,123,shallow spatial adapter,shallow spatial adapter,0.0003300754798792237,0.240793572992212,0.240793572992212,0.0,True -Pangu-Weather,pangu6,fire_prone,1,shallow spatial adapter,shallow spatial adapter,0.02328829578760066,0.7231388400090598,0.7231388400090598,0.0,True -Pangu-Weather,pangu6,fire_prone,7,pixel MLP head,shallow spatial adapter,0.018664183428008234,0.7185324707231184,0.719461376864001,0.0009289061408825905,False -Pangu-Weather,pangu6,fire_prone,42,shallow spatial adapter,shallow spatial adapter,0.021420904714594617,0.7241279907754397,0.7241279907754397,0.0,True -Pangu-Weather,pangu6,fire_prone,99,linear probe,shallow spatial adapter,0.024740444457605402,0.7183382629739177,0.719015307962107,0.0006770449881893237,False -Pangu-Weather,pangu6,fire_prone,123,pixel MLP head,linear probe,0.02254610440161657,0.7185324707231184,0.7261721555350364,0.007639684811918013,False -Pangu-Weather,pangu6,global,1,shallow spatial adapter,shallow spatial adapter,0.00042881385578730365,0.32066974948800614,0.32066974948800614,0.0,True -Pangu-Weather,pangu6,global,7,shallow spatial adapter,shallow spatial adapter,0.00038395193539280824,0.31120670593952293,0.31120670593952293,0.0,True -Pangu-Weather,pangu6,global,42,shallow spatial adapter,shallow spatial adapter,0.00040651309469793043,0.32154249424354786,0.32154249424354786,0.0,True -Pangu-Weather,pangu6,global,99,shallow spatial adapter,shallow spatial adapter,0.0004450373086921994,0.3214547875801752,0.3214547875801752,0.0,True -Pangu-Weather,pangu6,global,123,shallow spatial adapter,shallow spatial adapter,0.00043738577276574255,0.31812983009681495,0.31812983009681495,0.0,True -Prithvi-WxC,prithvi_wxc,fire_prone,1,shallow spatial adapter,shallow spatial adapter,0.02093558282208589,0.7185324707231184,0.7185324707231184,0.0,True -Prithvi-WxC,prithvi_wxc,fire_prone,7,shallow spatial adapter,shallow spatial adapter,0.02093558282208589,0.7185324707231184,0.7185324707231184,0.0,True -Prithvi-WxC,prithvi_wxc,fire_prone,42,pixel MLP head,shallow spatial adapter,0.02093558282208589,0.7185324707231184,0.7185324707231184,0.0,False -Prithvi-WxC,prithvi_wxc,fire_prone,99,linear probe,shallow spatial adapter,0.02093558282208589,0.7185324707231184,0.7185324707231184,0.0,False -Prithvi-WxC,prithvi_wxc,fire_prone,123,shallow spatial adapter,shallow spatial adapter,0.02093558282208589,0.7185324707231184,0.7185324707231184,0.0,True -Prithvi-WxC,prithvi_wxc,global,1,shallow spatial adapter,shallow spatial adapter,0.0002463357401629007,0.240793572992212,0.240793572992212,0.0,True -Prithvi-WxC,prithvi_wxc,global,7,shallow spatial adapter,shallow spatial adapter,0.0002463357401629007,0.240793572992212,0.240793572992212,0.0,True -Prithvi-WxC,prithvi_wxc,global,42,shallow spatial adapter,shallow spatial adapter,0.0002463357401629007,0.240793572992212,0.240793572992212,0.0,True -Prithvi-WxC,prithvi_wxc,global,99,shallow spatial adapter,shallow spatial adapter,0.0002454330184381399,0.23755358049655212,0.23755358049655212,0.0,True -Prithvi-WxC,prithvi_wxc,global,123,shallow spatial adapter,shallow spatial adapter,0.0002463357401629007,0.240793572992212,0.240793572992212,0.0,True -Reference,reference,fire_prone,1,shallow spatial adapter,linear probe,0.10204224118176683,0.799032457577039,0.8685988450931266,0.06956638751608757,False -Reference,reference,fire_prone,7,shallow spatial adapter,shallow spatial adapter,0.1323902067230726,0.8027807546359551,0.8027807546359551,0.0,True -Reference,reference,fire_prone,42,shallow spatial adapter,shallow spatial adapter,0.12048427320762313,0.8358151221553631,0.8358151221553631,0.0,True -Reference,reference,fire_prone,99,shallow spatial adapter,linear probe,0.11947246238005743,0.8534759193943048,0.9039923296574045,0.05051641026309972,False -Reference,reference,fire_prone,123,shallow spatial adapter,pixel MLP head,0.11551882470432043,0.8066689866810086,0.8567215417854325,0.05005255510442386,False -Reference,reference,global,1,shallow spatial adapter,linear probe,0.002624341503354088,0.6186566066408326,0.7286109603326707,0.10995435369183815,False -Reference,reference,global,7,shallow spatial adapter,shallow spatial adapter,0.0030317345997110698,0.5002193417313746,0.5002193417313746,0.0,True -Reference,reference,global,42,shallow spatial adapter,shallow spatial adapter,0.0032729435045747413,0.7239943741463363,0.7239943741463363,0.0,True -Reference,reference,global,99,shallow spatial adapter,linear probe,0.0031953098868323193,0.6698005926442897,0.7647466876509988,0.09494609500670914,False -Reference,reference,global,123,shallow spatial adapter,pixel MLP head,0.0025603887793133394,0.5009707461135111,0.735221546471909,0.23425080035839785,False -StormCast,stormcast,fire_prone,1,linear probe,shallow spatial adapter,0.02093558282208589,0.7185324707231184,0.7185324707231184,0.0,False -StormCast,stormcast,fire_prone,7,linear probe,shallow spatial adapter,0.02093558282208589,0.7185324707231184,0.7185324707231184,0.0,False -StormCast,stormcast,fire_prone,42,linear probe,shallow spatial adapter,0.02093558282208589,0.7185324707231184,0.7185324707231184,0.0,False -StormCast,stormcast,fire_prone,99,pixel MLP head,shallow spatial adapter,0.02093558282208589,0.7185324707231184,0.7185324707231184,0.0,False -StormCast,stormcast,fire_prone,123,linear probe,shallow spatial adapter,0.02093558282208589,0.7185324707231184,0.7185324707231184,0.0,False -StormCast,stormcast,global,1,shallow spatial adapter,shallow spatial adapter,0.0002463357401629007,0.240793572992212,0.240793572992212,0.0,True -StormCast,stormcast,global,7,shallow spatial adapter,shallow spatial adapter,0.0002463357401629007,0.240793572992212,0.240793572992212,0.0,True -StormCast,stormcast,global,42,shallow spatial adapter,shallow spatial adapter,0.0002463357401629007,0.240793572992212,0.240793572992212,0.0,True -StormCast,stormcast,global,99,shallow spatial adapter,shallow spatial adapter,0.0002463357401629007,0.2399251001974223,0.2399251001974223,0.0,True -StormCast,stormcast,global,123,shallow spatial adapter,shallow spatial adapter,0.0002463357401629007,0.240793572992212,0.240793572992212,0.0,True diff --git a/artifacts/results/selection_regret_rq2_figure_values.csv b/artifacts/results/selection_regret_rq2_figure_values.csv deleted file mode 100644 index 66605affbdfa7fdc9e132b4749da594f89863727..0000000000000000000000000000000000000000 --- a/artifacts/results/selection_regret_rq2_figure_values.csv +++ /dev/null @@ -1,12 +0,0 @@ -feature_source,global_mean_pp,global_std_pp,top20_mean_pp,top20_std_pp -FireWx-FM ref.,7.3831,7.4536,2.9385,2.7513 -Prithvi-WxC,0.0000,0.0000,0.0000,0.0000 -Aurora,4.9455,10.6974,14.3706,32.1337 -ClimaX,0.1296,0.1775,0.0000,0.0000 -StormCast,0.0000,0.0000,0.0000,0.0000 -DLWP,0.0000,0.0000,4.4634,4.3561 -FCN,0.0000,0.0000,1.1680,1.9872 -FengWu,0.0000,0.0000,0.5222,0.6239 -FuXi,0.0000,0.0000,0.2833,0.3681 -Pangu-Weather,0.0000,0.0000,0.1868,0.3255 -AlphaEarth,17.2217,8.8492,3.8804,5.9483 diff --git a/artifacts/results/selection_regret_scope_sweep_20260505.csv b/artifacts/results/selection_regret_scope_sweep_20260505.csv deleted file mode 100644 index b2d3d4305cf14014566b7a7ab80542c3846b3fba..0000000000000000000000000000000000000000 --- a/artifacts/results/selection_regret_scope_sweep_20260505.csv +++ /dev/null @@ -1,45 +0,0 @@ -model_tag,label,scope,scope_label,n,seeds,exact_regret_mean,exact_regret_std,exact_regret_min,exact_regret_max,tolerated_regret_mean,tolerated_regret_std,tolerated_regret_min,tolerated_regret_max,union_regret_mean,union_regret_std,union_regret_min,union_regret_max -reference,FireWx-FM ref.,global,\(\Omega=\)global,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.07383089600948442,0.07453636071372995,0.0,0.17497107865629713,0.07383089600948442,0.07453636071372995,0.0,0.17497107865629713 -reference,FireWx-FM ref.,top5,\(\Omega=\)top 5\%,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.003663718115532055,0.006812231244812292,0.0,0.015676762201120686,0.003663718115532055,0.006812231244812292,0.0,0.015676762201120686 -reference,FireWx-FM ref.,top10,\(\Omega=\)top 10\%,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.012275489592085752,0.012665162001740834,0.0,0.02670880922526031,0.012275489592085752,0.012665162001740834,0.0,0.02670880922526031 -reference,FireWx-FM ref.,top20,\(\Omega=\)top 20\%,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.029384646387840017,0.02751315335001922,0.0,0.05675140203555318,0.029384646387840017,0.02751315335001922,0.0,0.05675140203555318 -prithvi_wxc,Prithvi-WxC,global,\(\Omega=\)global,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 -prithvi_wxc,Prithvi-WxC,top5,\(\Omega=\)top 5\%,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 -prithvi_wxc,Prithvi-WxC,top10,\(\Omega=\)top 10\%,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 -prithvi_wxc,Prithvi-WxC,top20,\(\Omega=\)top 20\%,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 -aurora,Aurora,global,\(\Omega=\)global,5,1 7 42 99 123,0.00010153879814819402,0.00021861477435572763,0.0,0.0004925501476206198,0.04945471159670635,0.10697394238964528,0.0,0.240793572992212,0.04945471159670635,0.10697394238964528,0.0,0.240793572992212 -aurora,Aurora,top5,\(\Omega=\)top 5\%,5,1 7 42 99 123,0.023667154505540456,0.05292136630837888,0.0,0.11833577252770228,0.1542829840966487,0.34498724021162547,0.0,0.7714149204832434,0.1542829840966487,0.34498724021162547,0.0,0.7714149204832434 -aurora,Aurora,top10,\(\Omega=\)top 10\%,5,1 7 42 99 123,0.014651279478173606,0.032761256870543834,0.0,0.07325639739086803,0.1399343063221699,0.31290262132065055,0.0,0.6996715316108496,0.1399343063221699,0.31290262132065055,0.0,0.6996715316108496 -aurora,Aurora,top20,\(\Omega=\)top 20\%,5,1 7 42 99 123,0.008202508825959588,0.01834136732088763,0.0,0.04101254412979794,0.1437064941446237,0.32133748971555404,0.0,0.7185324707231184,0.1437064941446237,0.32133748971555404,0.0,0.7185324707231184 -climax,ClimaX,global,\(\Omega=\)global,5,1 7 42 99 123,3.0287686240700486e-06,4.147312242167625e-06,0.0,7.571921560175121e-06,0.0012959969982639485,0.0017746169760203706,0.0,0.0032399924956598714,0.0012959969982639485,0.0017746169760203706,0.0,0.0032399924956598714 -climax,ClimaX,top5,\(\Omega=\)top 5\%,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 -climax,ClimaX,top10,\(\Omega=\)top 10\%,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 -climax,ClimaX,top20,\(\Omega=\)top 20\%,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 -stormcast,StormCast,global,\(\Omega=\)global,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 -stormcast,StormCast,top5,\(\Omega=\)top 5\%,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 -stormcast,StormCast,top10,\(\Omega=\)top 10\%,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 -stormcast,StormCast,top20,\(\Omega=\)top 20\%,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 -dlwp,DLWP,global,\(\Omega=\)global,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 -dlwp,DLWP,top5,\(\Omega=\)top 5\%,5,1 7 42 99 123,0.0048037709215293075,0.006217185202152866,0.0,0.015203005078871956,0.016716228534155796,0.016079313546074458,0.0,0.03305057342744666,0.016716228534155796,0.016079313546074458,0.0,0.03305057342744666 -dlwp,DLWP,top10,\(\Omega=\)top 10\%,5,1 7 42 99 123,0.0017281632798742507,0.002514722758075371,0.0,0.005523780499856246,0.02846514801700826,0.026938012702643194,0.0,0.053927677500854476,0.02846514801700826,0.026938012702643194,0.0,0.053927677500854476 -dlwp,DLWP,top20,\(\Omega=\)top 20\%,5,1 7 42 99 123,0.0007702319787454587,0.0010995336594539604,0.0,0.0023651634514294945,0.04463354681768479,0.04356064433532197,0.0,0.11215006616165157,0.04463354681768479,0.04356064433532197,0.0,0.11215006616165157 -fcn,FCN,global,\(\Omega=\)global,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 -fcn,FCN,top5,\(\Omega=\)top 5\%,5,1 7 42 99 123,0.0006342898232943345,0.0009899554165032742,0.0,0.002257520679520411,0.004509624980300697,0.010070611656609236,0.0,0.022524473456150496,0.004509624980300697,0.010070611656609236,0.0,0.022524473456150496 -fcn,FCN,top10,\(\Omega=\)top 10\%,5,1 7 42 99 123,0.00021156854817603877,0.0004730816556225618,0.0,0.0010578427408801938,0.004199537050817615,0.009390450319657174,0.0,0.020997685254088072,0.004199537050817615,0.009390450319657174,0.0,0.020997685254088072 -fcn,FCN,top20,\(\Omega=\)top 20\%,5,1 7 42 99 123,5.754560074337778e-06,1.2867587506825515e-05,0.0,2.877280037168889e-05,0.011679805987441694,0.019872372458657642,0.0,0.045880503237594294,0.011679805987441694,0.019872372458657642,0.0,0.045880503237594294 -fengwu,FengWu,global,\(\Omega=\)global,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 -fengwu,FengWu,top5,\(\Omega=\)top 5\%,5,1 7 42 99 123,0.0005029843170376968,0.0008109166521114917,0.0,0.0018628094907809783,0.008795951947678215,0.005532321338017505,0.0,0.01484136735033148,0.008795951947678215,0.005532321338017505,0.0,0.01484136735033148 -fengwu,FengWu,top10,\(\Omega=\)top 10\%,5,1 7 42 99 123,0.000495228089292582,0.0007349190216431337,0.0,0.0016387212062008855,0.00402300475984525,0.005510851442075993,0.0,0.010273937098576491,0.00402300475984525,0.005510851442075993,0.0,0.010273937098576491 -fengwu,FengWu,top20,\(\Omega=\)top 20\%,5,1 7 42 99 123,0.0006908222234409067,0.0011910586589384115,0.0,0.0027505832409660327,0.005222389249812243,0.0062394095558402415,0.0,0.015150501067724198,0.005222389249812243,0.0062394095558402415,0.0,0.015150501067724198 -fuxi,FuXi,global,\(\Omega=\)global,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 -fuxi,FuXi,top5,\(\Omega=\)top 5\%,5,1 7 42 99 123,0.002973545331200933,0.0023946274991058026,0.0010927807990139538,0.007024214143542151,0.013545122545609134,0.02097023683418404,0.0,0.050156261654859424,0.013545122545609134,0.02097023683418404,0.0,0.050156261654859424 -fuxi,FuXi,top10,\(\Omega=\)top 10\%,5,1 7 42 99 123,0.001383793743586542,0.0019248128430711165,0.0,0.003938013087198336,0.0016559834970027332,0.0037028916689159307,0.0,0.008279917485013666,0.0016559834970027332,0.0037028916689159307,0.0,0.008279917485013666 -fuxi,FuXi,top20,\(\Omega=\)top 20\%,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.00283318355751887,0.0036808289681375247,0.0,0.008746323525994693,0.00283318355751887,0.0036808289681375247,0.0,0.008746323525994693 -pangu6,Pangu-Weather,global,\(\Omega=\)global,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 -pangu6,Pangu-Weather,top5,\(\Omega=\)top 5\%,5,1 7 42 99 123,0.003154674487241463,0.002546125713211599,0.0,0.005711594157251587,0.007592888149777122,0.00897418737588444,0.0,0.019790633919317124,0.007592888149777122,0.00897418737588444,0.0,0.019790633919317124 -pangu6,Pangu-Weather,top10,\(\Omega=\)top 10\%,5,1 7 42 99 123,0.0017345627303725214,0.0019305189318827886,0.0,0.004535321555179647,0.003047840737004992,0.005053805614558161,0.0,0.011660780793438352,0.003047840737004992,0.005053805614558161,0.0,0.011660780793438352 -pangu6,Pangu-Weather,top20,\(\Omega=\)top 20\%,5,1 7 42 99 123,0.0007280423771922354,0.001178746460551365,0.0,0.0027096086413018403,0.0018679847512695024,0.0032548337047755126,0.0,0.007639684811918013,0.0018679847512695024,0.0032548337047755126,0.0,0.007639684811918013 -alphaearth,AlphaEarth,global,\(\Omega=\)global,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.1722171037486726,0.08849214830495522,0.050655329891645895,0.272468682523694,0.1722171037486726,0.08849214830495522,0.050655329891645895,0.272468682523694 -alphaearth,AlphaEarth,top5,\(\Omega=\)top 5\%,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.06384618090125256,0.04965276403138872,0.0,0.1365277562230962,0.06384618090125256,0.04965276403138872,0.0,0.1365277562230962 -alphaearth,AlphaEarth,top10,\(\Omega=\)top 10\%,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.06573776411084173,0.06897015340160571,0.0,0.1615566566666954,0.06573776411084173,0.06897015340160571,0.0,0.1615566566666954 -alphaearth,AlphaEarth,top20,\(\Omega=\)top 20\%,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.038803552655092256,0.0594825313313219,0.0,0.13482302181308625,0.038803552655092256,0.0594825313313219,0.0,0.13482302181308625 diff --git a/artifacts/results/selection_regret_scope_sweep_20260505.json b/artifacts/results/selection_regret_scope_sweep_20260505.json deleted file mode 100644 index 6892a59cdf49a0dfc4a7ef7483950aab498863dc..0000000000000000000000000000000000000000 --- a/artifacts/results/selection_regret_scope_sweep_20260505.json +++ /dev/null @@ -1,10274 +0,0 @@ -{ - "rows": [ - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.004193290734824281", - "exact_ranking_score": "0.004193290734824281", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.0056438127090301", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.0056438127090301", - "family": "AlphaEarth", - "model_tag": "alphaearth", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.0006549130347299629", - "ranking_selected_val_pr_auc": "0.0007883068263073185", - "scope": "global", - "seed": 1, - "tolerated_decision_head": "pixel MLP head", - "tolerated_decision_head_arch": "pixel_mlp", - "tolerated_decision_score": "0.6337627266658229", - "tolerated_ranking_score": "0.40561891947698747", - "tolerated_regret": 0.22814380718883542, - "tolerated_test_gap": "0.22814380718883542", - "tolerated_val_decision_score": "0.6238973446702573", - "tolerated_val_gap": "0.1715500493543587", - "tolerated_val_ranking_score": "0.45234729531589857", - "union_decision_head": "pixel MLP head", - "union_decision_head_arch": "pixel_mlp", - "union_decision_score": "0.6337627266658229", - "union_ranking_score": "0.40561891947698747", - "union_regret": 0.22814380718883542, - "union_test_gap": "0.22814380718883542", - "union_val_decision_score": "0.6238973446702573", - "union_val_gap": "0.1715500493543587", - "union_val_ranking_score": "0.45234729531589857", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.2053654024051804", - "exact_ranking_score": "0.2053654024051804", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.24264049955396969", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.24264049955396969", - "family": "AlphaEarth", - "model_tag": "alphaearth", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.13521999470870394", - "ranking_selected_val_pr_auc": "0.1530074531653213", - "scope": "top5", - "seed": 1, - "tolerated_decision_head": "pixel MLP head", - "tolerated_decision_head_arch": "pixel_mlp", - "tolerated_decision_score": "0.9218545130329889", - "tolerated_ranking_score": "0.8564974369370675", - "tolerated_regret": 0.06535707609592145, - "tolerated_test_gap": "0.06535707609592145", - "tolerated_val_decision_score": "0.8921006948278831", - "tolerated_val_gap": "0.06576085909428331", - "tolerated_val_ranking_score": "0.8263398357335998", - "union_decision_head": "pixel MLP head", - "union_decision_head_arch": "pixel_mlp", - "union_decision_score": "0.9218545130329889", - "union_ranking_score": "0.8564974369370675", - "union_regret": 0.06535707609592145, - "union_test_gap": "0.06535707609592145", - "union_val_decision_score": "0.8921006948278831", - "union_val_gap": "0.06576085909428331", - "union_val_ranking_score": "0.8263398357335998", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.16019417475728157", - "exact_ranking_score": "0.16019417475728157", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.18440523619806493", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.18440523619806493", - "family": "AlphaEarth", - "model_tag": "alphaearth", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.08541592232377082", - "ranking_selected_val_pr_auc": "0.09854032446284237", - "scope": "top10", - "seed": 1, - "tolerated_decision_head": "pixel MLP head", - "tolerated_decision_head_arch": "pixel_mlp", - "tolerated_decision_score": "0.8506993653336619", - "tolerated_ranking_score": "0.7839805825242719", - "tolerated_regret": 0.06671878280938992, - "tolerated_test_gap": "0.06671878280938992", - "tolerated_val_decision_score": "0.8034698049616", - "tolerated_val_gap": "0.04257704982233934", - "tolerated_val_ranking_score": "0.7608927551392607", - "union_decision_head": "pixel MLP head", - "union_decision_head_arch": "pixel_mlp", - "union_decision_score": "0.8506993653336619", - "union_ranking_score": "0.7839805825242719", - "union_regret": 0.06671878280938992, - "union_test_gap": "0.06671878280938992", - "union_val_decision_score": "0.8034698049616", - "union_val_gap": "0.04257704982233934", - "union_val_ranking_score": "0.7608927551392607", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.09715762273901808", - "exact_ranking_score": "0.09715762273901808", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.13462469733656174", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.13462469733656174", - "family": "AlphaEarth", - "model_tag": "alphaearth", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.04390614036305261", - "ranking_selected_val_pr_auc": "0.05734457583552234", - "scope": "top20", - "seed": 1, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7863702028280513", - "tolerated_ranking_score": "0.7863702028280513", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.758785818734687", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.758785818734687", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7863702028280513", - "union_ranking_score": "0.7863702028280513", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.758785818734687", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.758785818734687", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.006959088991986505", - "exact_ranking_score": "0.006959088991986505", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.006399069226294357", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.006399069226294357", - "family": "AlphaEarth", - "model_tag": "alphaearth", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.0007047712457371991", - "ranking_selected_val_pr_auc": "0.0007434343102653092", - "scope": "global", - "seed": 123, - "tolerated_decision_head": "pixel MLP head", - "tolerated_decision_head_arch": "pixel_mlp", - "tolerated_decision_score": "0.5604635792586619", - "tolerated_ranking_score": "0.4427625907752311", - "tolerated_regret": 0.11770098848343075, - "tolerated_test_gap": "0.11770098848343075", - "tolerated_val_decision_score": "0.5441785824390972", - "tolerated_val_gap": "0.056405166058839995", - "tolerated_val_ranking_score": "0.48777341638025723", - "union_decision_head": "pixel MLP head", - "union_decision_head_arch": "pixel_mlp", - "union_decision_score": "0.5604635792586619", - "union_ranking_score": "0.4427625907752311", - "union_regret": 0.11770098848343075, - "union_test_gap": "0.11770098848343075", - "union_val_decision_score": "0.5441785824390972", - "union_val_gap": "0.056405166058839995", - "union_val_ranking_score": "0.48777341638025723", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.1978021978021978", - "exact_ranking_score": "0.1978021978021978", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.2180376610505451", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.2180376610505451", - "family": "AlphaEarth", - "model_tag": "alphaearth", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.13463750657454468", - "ranking_selected_val_pr_auc": "0.14286946914705156", - "scope": "top5", - "seed": 123, - "tolerated_decision_head": "pixel MLP head", - "tolerated_decision_head_arch": "pixel_mlp", - "tolerated_decision_score": "0.8743168479997309", - "tolerated_ranking_score": "0.8305909861908366", - "tolerated_regret": 0.04372586180889426, - "tolerated_test_gap": "0.04372586180889426", - "tolerated_val_decision_score": "0.8625937780701644", - "tolerated_val_gap": "0.05250190173735958", - "tolerated_val_ranking_score": "0.8100918763328048", - "union_decision_head": "pixel MLP head", - "union_decision_head_arch": "pixel_mlp", - "union_decision_score": "0.8743168479997309", - "union_ranking_score": "0.8305909861908366", - "union_regret": 0.04372586180889426, - "union_test_gap": "0.04372586180889426", - "union_val_decision_score": "0.8625937780701644", - "union_val_gap": "0.05250190173735958", - "union_val_ranking_score": "0.8100918763328048", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.15615615615615616", - "exact_ranking_score": "0.15615615615615616", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.1708413615928067", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.1708413615928067", - "family": "AlphaEarth", - "model_tag": "alphaearth", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.0877757525096481", - "ranking_selected_val_pr_auc": "0.09333059447315038", - "scope": "top10", - "seed": 123, - "tolerated_decision_head": "linear probe", - "tolerated_decision_head_arch": "linear", - "tolerated_decision_score": "0.6977392921485441", - "tolerated_ranking_score": "0.7665340693262388", - "tolerated_regret": 0.0, - "tolerated_test_gap": "-0.06879477717769467", - "tolerated_val_decision_score": "0.7559010025052879", - "tolerated_val_gap": "0.0006607651392849245", - "tolerated_val_ranking_score": "0.755240237366003", - "union_decision_head": "linear probe", - "union_decision_head_arch": "linear", - "union_decision_score": "0.6977392921485441", - "union_ranking_score": "0.7665340693262388", - "union_regret": 0.0, - "union_test_gap": "-0.06879477717769467", - "union_val_decision_score": "0.7559010025052879", - "union_val_gap": "0.0006607651392849245", - "union_val_ranking_score": "0.755240237366003", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.10531544957774468", - "exact_ranking_score": "0.10531544957774468", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.12267343485617596", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.12267343485617596", - "family": "AlphaEarth", - "model_tag": "alphaearth", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.045809049876129763", - "ranking_selected_val_pr_auc": "0.05470980065861018", - "scope": "top20", - "seed": 123, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7789089693560928", - "tolerated_ranking_score": "0.7789089693560928", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.749825256630312", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.749825256630312", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7789089693560928", - "union_ranking_score": "0.7789089693560928", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.749825256630312", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.749825256630312", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.004809747755451047", - "exact_ranking_score": "0.004809747755451047", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.004359099620457705", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.004359099620457705", - "family": "AlphaEarth", - "model_tag": "alphaearth", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.0005634701573991865", - "ranking_selected_val_pr_auc": "0.0005920938943373892", - "scope": "global", - "seed": 42, - "tolerated_decision_head": "pixel MLP head", - "tolerated_decision_head_arch": "pixel_mlp", - "tolerated_decision_score": "0.6812131506751973", - "tolerated_ranking_score": "0.4087444681515033", - "tolerated_regret": 0.272468682523694, - "tolerated_test_gap": "0.272468682523694", - "tolerated_val_decision_score": "0.7447325774946227", - "tolerated_val_gap": "0.3021550064919567", - "tolerated_val_ranking_score": "0.44257757100266604", - "union_decision_head": "pixel MLP head", - "union_decision_head_arch": "pixel_mlp", - "union_decision_score": "0.6812131506751973", - "union_ranking_score": "0.4087444681515033", - "union_regret": 0.272468682523694, - "union_test_gap": "0.272468682523694", - "union_val_decision_score": "0.7447325774946227", - "union_val_gap": "0.3021550064919567", - "union_val_ranking_score": "0.44257757100266604", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.1818181818181818", - "exact_ranking_score": "0.1818181818181818", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.20273972602739726", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.20273972602739726", - "family": "AlphaEarth", - "model_tag": "alphaearth", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.11776218604745192", - "ranking_selected_val_pr_auc": "0.12790468093676954", - "scope": "top5", - "seed": 42, - "tolerated_decision_head": "pixel MLP head", - "tolerated_decision_head_arch": "pixel_mlp", - "tolerated_decision_score": "0.9079426767063397", - "tolerated_ranking_score": "0.7714149204832434", - "tolerated_regret": 0.1365277562230962, - "tolerated_test_gap": "0.1365277562230962", - "tolerated_val_decision_score": "0.8560076563904758", - "tolerated_val_gap": "0.058814205003907216", - "tolerated_val_ranking_score": "0.7971934513865686", - "union_decision_head": "pixel MLP head", - "union_decision_head_arch": "pixel_mlp", - "union_decision_score": "0.9079426767063397", - "union_ranking_score": "0.7714149204832434", - "union_regret": 0.1365277562230962, - "union_test_gap": "0.1365277562230962", - "union_val_decision_score": "0.8560076563904758", - "union_val_gap": "0.058814205003907216", - "union_val_ranking_score": "0.7971934513865686", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.14064516129032256", - "exact_ranking_score": "0.14064516129032256", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.1536748329621381", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.1536748329621381", - "family": "AlphaEarth", - "model_tag": "alphaearth", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.07431134992726657", - "ranking_selected_val_pr_auc": "0.07868604702405124", - "scope": "top10", - "seed": 42, - "tolerated_decision_head": "pixel MLP head", - "tolerated_decision_head_arch": "pixel_mlp", - "tolerated_decision_score": "0.860840238051614", - "tolerated_ranking_score": "0.6992835813849186", - "tolerated_regret": 0.1615566566666954, - "tolerated_test_gap": "0.1615566566666954", - "tolerated_val_decision_score": "0.8130323993938946", - "tolerated_val_gap": "0.0827924722936264", - "tolerated_val_ranking_score": "0.7302399271002682", - "union_decision_head": "pixel MLP head", - "union_decision_head_arch": "pixel_mlp", - "union_decision_score": "0.860840238051614", - "union_ranking_score": "0.6992835813849186", - "union_regret": 0.1615566566666954, - "union_test_gap": "0.1615566566666954", - "union_val_decision_score": "0.8130323993938946", - "union_val_gap": "0.0827924722936264", - "union_val_ranking_score": "0.7302399271002682", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.08702469619756958", - "exact_ranking_score": "0.08702469619756958", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.10515392597717052", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.10515392597717052", - "family": "AlphaEarth", - "model_tag": "alphaearth", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.038083070948941686", - "ranking_selected_val_pr_auc": "0.04505214599515043", - "scope": "top20", - "seed": 42, - "tolerated_decision_head": "pixel MLP head", - "tolerated_decision_head_arch": "pixel_mlp", - "tolerated_decision_score": "0.8461131676361712", - "tolerated_ranking_score": "0.7112901458230849", - "tolerated_regret": 0.13482302181308625, - "tolerated_test_gap": "0.13482302181308625", - "tolerated_val_decision_score": "0.806760850261024", - "tolerated_val_gap": "0.07815518654517495", - "tolerated_val_ranking_score": "0.728605663715849", - "union_decision_head": "pixel MLP head", - "union_decision_head_arch": "pixel_mlp", - "union_decision_score": "0.8461131676361712", - "union_ranking_score": "0.7112901458230849", - "union_regret": 0.13482302181308625, - "union_test_gap": "0.13482302181308625", - "union_val_decision_score": "0.806760850261024", - "union_val_gap": "0.07815518654517495", - "union_val_ranking_score": "0.728605663715849", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.010460251046025104", - "exact_ranking_score": "0.010460251046025104", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.009547591069330199", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.009547591069330199", - "family": "AlphaEarth", - "model_tag": "alphaearth", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.001005722733868245", - "ranking_selected_val_pr_auc": "0.0009777524224237678", - "scope": "global", - "seed": 7, - "tolerated_decision_head": "pixel MLP head", - "tolerated_decision_head_arch": "pixel_mlp", - "tolerated_decision_score": "0.6691395427484861", - "tolerated_ranking_score": "0.6184842128568402", - "tolerated_regret": 0.050655329891645895, - "tolerated_test_gap": "0.050655329891645895", - "tolerated_val_decision_score": "0.677404587086621", - "tolerated_val_gap": "0.011527156971746999", - "tolerated_val_ranking_score": "0.665877430114874", - "union_decision_head": "pixel MLP head", - "union_decision_head_arch": "pixel_mlp", - "union_decision_score": "0.6691395427484861", - "union_ranking_score": "0.6184842128568402", - "union_regret": 0.050655329891645895, - "union_test_gap": "0.050655329891645895", - "union_val_decision_score": "0.677404587086621", - "union_val_gap": "0.011527156971746999", - "union_val_ranking_score": "0.665877430114874", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.18705957719410635", - "exact_ranking_score": "0.18705957719410635", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.22265122265122267", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.22265122265122267", - "family": "AlphaEarth", - "model_tag": "alphaearth", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.1415020173172751", - "ranking_selected_val_pr_auc": "0.13899023094495766", - "scope": "top5", - "seed": 7, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.8671946832706918", - "tolerated_ranking_score": "0.8671946832706918", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.8765714285714286", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.8765714285714286", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.8671946832706918", - "union_ranking_score": "0.8671946832706918", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.8765714285714286", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.8765714285714286", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.15882806476484196", - "exact_ranking_score": "0.15882806476484196", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.17872968980797638", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.17872968980797638", - "family": "AlphaEarth", - "model_tag": "alphaearth", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.0993207020173454", - "ranking_selected_val_pr_auc": "0.09562560899249391", - "scope": "top10", - "seed": 7, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.8345528858733409", - "tolerated_ranking_score": "0.8345528858733409", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.8241729549074388", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.8241729549074388", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.8345528858733409", - "union_ranking_score": "0.8345528858733409", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.8241729549074388", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.8241729549074388", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.12514898688915374", - "exact_ranking_score": "0.12514898688915374", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.14325685506435365", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.14325685506435365", - "family": "AlphaEarth", - "model_tag": "alphaearth", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.05955397140893137", - "ranking_selected_val_pr_auc": "0.06058697421048699", - "scope": "top20", - "seed": 7, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.8294499693616161", - "tolerated_ranking_score": "0.8294499693616161", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.8107581951083846", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.8107581951083846", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.8294499693616161", - "union_ranking_score": "0.8294499693616161", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.8107581951083846", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.8107581951083846", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.006780481898534931", - "exact_ranking_score": "0.006780481898534931", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.005964994898359627", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.005964994898359627", - "family": "AlphaEarth", - "model_tag": "alphaearth", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.0006577120081349608", - "ranking_selected_val_pr_auc": "0.0007340310681749519", - "scope": "global", - "seed": 99, - "tolerated_decision_head": "pixel MLP head", - "tolerated_decision_head_arch": "pixel_mlp", - "tolerated_decision_score": "0.5842714676652996", - "tolerated_ranking_score": "0.3921547570095426", - "tolerated_regret": 0.19211671065575697, - "tolerated_test_gap": "0.19211671065575697", - "tolerated_val_decision_score": "0.5663548190132099", - "tolerated_val_gap": "0.10549666519652506", - "tolerated_val_ranking_score": "0.4608581538166849", - "union_decision_head": "pixel MLP head", - "union_decision_head_arch": "pixel_mlp", - "union_decision_score": "0.5842714676652996", - "union_ranking_score": "0.3921547570095426", - "union_regret": 0.19211671065575697, - "union_test_gap": "0.19211671065575697", - "union_val_decision_score": "0.5663548190132099", - "union_val_gap": "0.10549666519652506", - "union_val_ranking_score": "0.4608581538166849", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.1867517956903432", - "exact_ranking_score": "0.1867517956903432", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.2253944402704733", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.2253944402704733", - "family": "AlphaEarth", - "model_tag": "alphaearth", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.13302539422913043", - "ranking_selected_val_pr_auc": "0.14641451099665195", - "scope": "top5", - "seed": 99, - "tolerated_decision_head": "pixel MLP head", - "tolerated_decision_head_arch": "pixel_mlp", - "tolerated_decision_score": "0.898839683642517", - "tolerated_ranking_score": "0.8252194732641661", - "tolerated_regret": 0.07362021037835087, - "tolerated_test_gap": "0.07362021037835087", - "tolerated_val_decision_score": "0.8737527215993107", - "tolerated_val_gap": "0.06839730093782548", - "tolerated_val_ranking_score": "0.8053554206614852", - "union_decision_head": "pixel MLP head", - "union_decision_head_arch": "pixel_mlp", - "union_decision_score": "0.898839683642517", - "union_ranking_score": "0.8252194732641661", - "union_regret": 0.07362021037835087, - "union_test_gap": "0.07362021037835087", - "union_val_decision_score": "0.8737527215993107", - "union_val_gap": "0.06839730093782548", - "union_val_ranking_score": "0.8053554206614852", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.15477084898572502", - "exact_ranking_score": "0.15477084898572502", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.16870573084352863", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.16870573084352863", - "family": "AlphaEarth", - "model_tag": "alphaearth", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.08656943374833374", - "ranking_selected_val_pr_auc": "0.09321280420976051", - "scope": "top10", - "seed": 99, - "tolerated_decision_head": "pixel MLP head", - "tolerated_decision_head_arch": "pixel_mlp", - "tolerated_decision_score": "0.8655328563033635", - "tolerated_ranking_score": "0.7651194752252402", - "tolerated_regret": 0.10041338107812336, - "tolerated_test_gap": "0.10041338107812336", - "tolerated_val_decision_score": "0.8092827685342129", - "tolerated_val_gap": "0.06382807430932869", - "tolerated_val_ranking_score": "0.7454546942248842", - "union_decision_head": "pixel MLP head", - "union_decision_head_arch": "pixel_mlp", - "union_decision_score": "0.8655328563033635", - "union_ranking_score": "0.7651194752252402", - "union_regret": 0.10041338107812336, - "union_test_gap": "0.10041338107812336", - "union_val_decision_score": "0.8092827685342129", - "union_val_gap": "0.06382807430932869", - "union_val_ranking_score": "0.7454546942248842", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.10365251727541955", - "exact_ranking_score": "0.10365251727541955", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.12251941328731666", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.12251941328731666", - "family": "AlphaEarth", - "model_tag": "alphaearth", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.0458102699699856", - "ranking_selected_val_pr_auc": "0.0554648890254577", - "scope": "top20", - "seed": 99, - "tolerated_decision_head": "pixel MLP head", - "tolerated_decision_head_arch": "pixel_mlp", - "tolerated_decision_score": "0.8350245452333586", - "tolerated_ranking_score": "0.7758298037709835", - "tolerated_regret": 0.05919474146237502, - "tolerated_test_gap": "0.05919474146237502", - "tolerated_val_decision_score": "0.7736506356603177", - "tolerated_val_gap": "0.027586681737492236", - "tolerated_val_ranking_score": "0.7460639539228254", - "union_decision_head": "pixel MLP head", - "union_decision_head_arch": "pixel_mlp", - "union_decision_score": "0.8350245452333586", - "union_ranking_score": "0.7758298037709835", - "union_regret": 0.05919474146237502, - "union_test_gap": "0.05919474146237502", - "union_val_decision_score": "0.7736506356603177", - "union_val_gap": "0.027586681737492236", - "union_val_ranking_score": "0.7460639539228254", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0004925501476206198", - "exact_ranking_score": "0.00048497822606044473", - "exact_regret": 7.571921560175121e-06, - "exact_test_gap": "7.571921560175121e-06", - "exact_val_decision_score": "0.0004727121748000204", - "exact_val_gap": "7.267025482891767e-06", - "exact_val_ranking_score": "0.00046544514931712863", - "family": "Aurora", - "model_tag": "aurora", - "ranking_selected_head": "pixel MLP head", - "ranking_selected_head_arch": "pixel_mlp", - "ranking_selected_test_pr_auc": "0.00017433024575177084", - "ranking_selected_val_pr_auc": "0.00039761796201498743", - "scope": "global", - "seed": 1, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.240793572992212", - "tolerated_ranking_score": "0.23755358049655212", - "tolerated_regret": 0.0032399924956598714, - "tolerated_test_gap": "0.0032399924956598714", - "tolerated_val_decision_score": "0.2179368807313983", - "tolerated_val_gap": "0.00299097714352789", - "tolerated_val_ranking_score": "0.2149459035878704", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.240793572992212", - "union_ranking_score": "0.23755358049655212", - "union_regret": 0.0032399924956598714, - "union_test_gap": "0.0032399924956598714", - "union_val_decision_score": "0.2179368807313983", - "union_val_gap": "0.00299097714352789", - "union_val_ranking_score": "0.2149459035878704", - "path": "" - }, - { - "exact_decision_head": "pixel MLP head", - "exact_decision_head_arch": "pixel_mlp", - "exact_decision_score": "0.11833577252770228", - "exact_ranking_score": "0.11833577252770228", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.11931877546755824", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.11931877546755824", - "family": "Aurora", - "model_tag": "aurora", - "ranking_selected_head": "pixel MLP head", - "ranking_selected_head_arch": "pixel_mlp", - "ranking_selected_test_pr_auc": "0.043017434088706286", - "ranking_selected_val_pr_auc": "0.18453515297332748", - "scope": "top5", - "seed": 1, - "tolerated_decision_head": "pixel MLP head", - "tolerated_decision_head_arch": "pixel_mlp", - "tolerated_decision_score": "0.7714149204832434", - "tolerated_ranking_score": "0.7714149204832434", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.7971934513865686", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.7971934513865686", - "union_decision_head": "pixel MLP head", - "union_decision_head_arch": "pixel_mlp", - "union_decision_score": "0.7714149204832434", - "union_ranking_score": "0.7714149204832434", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.7971934513865686", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.7971934513865686", - "path": "" - }, - { - "exact_decision_head": "pixel MLP head", - "exact_decision_head_arch": "pixel_mlp", - "exact_decision_score": "0.07325639739086803", - "exact_ranking_score": "0.07325639739086803", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.07460890493381468", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.07460890493381468", - "family": "Aurora", - "model_tag": "aurora", - "ranking_selected_head": "pixel MLP head", - "ranking_selected_head_arch": "pixel_mlp", - "ranking_selected_test_pr_auc": "0.025652132600196966", - "ranking_selected_val_pr_auc": "0.11087183021305622", - "scope": "top10", - "seed": 1, - "tolerated_decision_head": "pixel MLP head", - "tolerated_decision_head_arch": "pixel_mlp", - "tolerated_decision_score": "0.6996715316108496", - "tolerated_ranking_score": "0.6996715316108496", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.7015182768065465", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.7015182768065465", - "union_decision_head": "pixel MLP head", - "union_decision_head_arch": "pixel_mlp", - "union_decision_score": "0.6996715316108496", - "union_ranking_score": "0.6996715316108496", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.7015182768065465", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.7015182768065465", - "path": "" - }, - { - "exact_decision_head": "pixel MLP head", - "exact_decision_head_arch": "pixel_mlp", - "exact_decision_score": "0.04101254412979794", - "exact_ranking_score": "0.04101254412979794", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.04253189892419314", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.04253189892419314", - "family": "Aurora", - "model_tag": "aurora", - "ranking_selected_head": "pixel MLP head", - "ranking_selected_head_arch": "pixel_mlp", - "ranking_selected_test_pr_auc": "0.014185996780547955", - "ranking_selected_val_pr_auc": "0.059710932216298356", - "scope": "top20", - "seed": 1, - "tolerated_decision_head": "pixel MLP head", - "tolerated_decision_head_arch": "pixel_mlp", - "tolerated_decision_score": "0.7185324707231184", - "tolerated_ranking_score": "0.7185324707231184", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.6906754133476136", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.6906754133476136", - "union_decision_head": "pixel MLP head", - "union_decision_head_arch": "pixel_mlp", - "union_decision_score": "0.7185324707231184", - "union_ranking_score": "0.7185324707231184", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.6906754133476136", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.6906754133476136", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0004925501476206198", - "exact_ranking_score": "0.0004925501476206198", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.0004727121748000204", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.0004727121748000204", - "family": "Aurora", - "model_tag": "aurora", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.0001870694786133523", - "ranking_selected_val_pr_auc": "0.00044346166657711726", - "scope": "global", - "seed": 123, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.240793572992212", - "tolerated_ranking_score": "0.240793572992212", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.2179368807313983", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.2179368807313983", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.240793572992212", - "union_ranking_score": "0.240793572992212", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.2179368807313983", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.2179368807313983", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.11833577252770228", - "exact_ranking_score": "0.11833577252770228", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.11931877546755824", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.11931877546755824", - "family": "Aurora", - "model_tag": "aurora", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.0476637592736729", - "ranking_selected_val_pr_auc": "0.19978759776073984", - "scope": "top5", - "seed": 123, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7714149204832434", - "tolerated_ranking_score": "0.7714149204832434", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.7971934513865686", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.7971934513865686", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7714149204832434", - "union_ranking_score": "0.7714149204832434", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.7971934513865686", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.7971934513865686", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.07325639739086803", - "exact_ranking_score": "0.07325639739086803", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.07460890493381468", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.07460890493381468", - "family": "Aurora", - "model_tag": "aurora", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.028202726028592855", - "ranking_selected_val_pr_auc": "0.12040472446461332", - "scope": "top10", - "seed": 123, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.6996715316108496", - "tolerated_ranking_score": "0.6996715316108496", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.7015182768065465", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.7015182768065465", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.6996715316108496", - "union_ranking_score": "0.6996715316108496", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.7015182768065465", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.7015182768065465", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.04101254412979794", - "exact_ranking_score": "0.04101254412979794", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.04253189892419314", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.04253189892419314", - "family": "Aurora", - "model_tag": "aurora", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.015530707998309533", - "ranking_selected_val_pr_auc": "0.06432554845990393", - "scope": "top20", - "seed": 123, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7185324707231184", - "tolerated_ranking_score": "0.7185324707231184", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.6906754133476136", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.6906754133476136", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7185324707231184", - "union_ranking_score": "0.7185324707231184", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.6906754133476136", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.6906754133476136", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0004925501476206198", - "exact_ranking_score": "0.0", - "exact_regret": 0.0004925501476206198, - "exact_test_gap": "0.0004925501476206198", - "exact_val_decision_score": "0.0004727121748000204", - "exact_val_gap": "0.0004727121748000204", - "exact_val_ranking_score": "0.0", - "family": "Aurora", - "model_tag": "aurora", - "ranking_selected_head": "pixel MLP head", - "ranking_selected_head_arch": "pixel_mlp", - "ranking_selected_test_pr_auc": "0.000198182392518492", - "ranking_selected_val_pr_auc": "0.0002767362365663907", - "scope": "global", - "seed": 42, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.240793572992212", - "tolerated_ranking_score": "0.0", - "tolerated_regret": 0.240793572992212, - "tolerated_test_gap": "0.240793572992212", - "tolerated_val_decision_score": "0.2179368807313983", - "tolerated_val_gap": "0.2179368807313983", - "tolerated_val_ranking_score": "0.0", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.240793572992212", - "union_ranking_score": "0.0", - "union_regret": 0.240793572992212, - "union_test_gap": "0.240793572992212", - "union_val_decision_score": "0.2179368807313983", - "union_val_gap": "0.2179368807313983", - "union_val_ranking_score": "0.0", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.11833577252770228", - "exact_ranking_score": "0.0", - "exact_regret": 0.11833577252770228, - "exact_test_gap": "0.11833577252770228", - "exact_val_decision_score": "0.11931877546755824", - "exact_val_gap": "0.11931877546755824", - "exact_val_ranking_score": "0.0", - "family": "Aurora", - "model_tag": "aurora", - "ranking_selected_head": "pixel MLP head", - "ranking_selected_head_arch": "pixel_mlp", - "ranking_selected_test_pr_auc": "0.047074707804185205", - "ranking_selected_val_pr_auc": "0.11187203617000477", - "scope": "top5", - "seed": 42, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7714149204832434", - "tolerated_ranking_score": "0.0", - "tolerated_regret": 0.7714149204832434, - "tolerated_test_gap": "0.7714149204832434", - "tolerated_val_decision_score": "0.7971934513865686", - "tolerated_val_gap": "0.7971934513865686", - "tolerated_val_ranking_score": "0.0", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7714149204832434", - "union_ranking_score": "0.0", - "union_regret": 0.7714149204832434, - "union_test_gap": "0.7714149204832434", - "union_val_decision_score": "0.7971934513865686", - "union_val_gap": "0.7971934513865686", - "union_val_ranking_score": "0.0", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.07325639739086803", - "exact_ranking_score": "0.0", - "exact_regret": 0.07325639739086803, - "exact_test_gap": "0.07325639739086803", - "exact_val_decision_score": "0.07460890493381468", - "exact_val_gap": "0.07460890493381468", - "exact_val_ranking_score": "0.0", - "family": "Aurora", - "model_tag": "aurora", - "ranking_selected_head": "pixel MLP head", - "ranking_selected_head_arch": "pixel_mlp", - "ranking_selected_test_pr_auc": "0.027753810085825968", - "ranking_selected_val_pr_auc": "0.06589784404428278", - "scope": "top10", - "seed": 42, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.6996715316108496", - "tolerated_ranking_score": "0.0", - "tolerated_regret": 0.6996715316108496, - "tolerated_test_gap": "0.6996715316108496", - "tolerated_val_decision_score": "0.7015182768065465", - "tolerated_val_gap": "0.7015182768065465", - "tolerated_val_ranking_score": "0.0", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.6996715316108496", - "union_ranking_score": "0.0", - "union_regret": 0.6996715316108496, - "union_test_gap": "0.6996715316108496", - "union_val_decision_score": "0.7015182768065465", - "union_val_gap": "0.7015182768065465", - "union_val_ranking_score": "0.0", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.04101254412979794", - "exact_ranking_score": "0.0", - "exact_regret": 0.04101254412979794, - "exact_test_gap": "0.04101254412979794", - "exact_val_decision_score": "0.04253189892419314", - "exact_val_gap": "0.04253189892419314", - "exact_val_ranking_score": "0.0", - "family": "Aurora", - "model_tag": "aurora", - "ranking_selected_head": "pixel MLP head", - "ranking_selected_head_arch": "pixel_mlp", - "ranking_selected_test_pr_auc": "0.015401540261000917", - "ranking_selected_val_pr_auc": "0.03525031596119595", - "scope": "top20", - "seed": 42, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7185324707231184", - "tolerated_ranking_score": "0.0", - "tolerated_regret": 0.7185324707231184, - "tolerated_test_gap": "0.7185324707231184", - "tolerated_val_decision_score": "0.6906754133476136", - "tolerated_val_gap": "0.6906754133476136", - "tolerated_val_ranking_score": "0.0", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7185324707231184", - "union_ranking_score": "0.0", - "union_regret": 0.7185324707231184, - "union_test_gap": "0.7185324707231184", - "union_val_decision_score": "0.6906754133476136", - "union_val_gap": "0.6906754133476136", - "union_val_ranking_score": "0.0", - "path": "" - }, - { - "exact_decision_head": "linear probe", - "exact_decision_head_arch": "linear", - "exact_decision_score": "0.0", - "exact_ranking_score": "0.0004925501476206198", - "exact_regret": 0.0, - "exact_test_gap": "-0.0004925501476206198", - "exact_val_decision_score": "0.0005036333549176199", - "exact_val_gap": "3.092118011759949e-05", - "exact_val_ranking_score": "0.0004727121748000204", - "family": "Aurora", - "model_tag": "aurora", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.0002120945228028488", - "ranking_selected_val_pr_auc": "0.000380637430664089", - "scope": "global", - "seed": 7, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.240793572992212", - "tolerated_ranking_score": "0.240793572992212", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.2179368807313983", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.2179368807313983", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.240793572992212", - "union_ranking_score": "0.240793572992212", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.2179368807313983", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.2179368807313983", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.11833577252770228", - "exact_ranking_score": "0.11833577252770228", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.11931877546755824", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.11931877546755824", - "family": "Aurora", - "model_tag": "aurora", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.05983564956422643", - "ranking_selected_val_pr_auc": "0.15910946905394902", - "scope": "top5", - "seed": 7, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7714149204832434", - "tolerated_ranking_score": "0.7714149204832434", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.7971934513865686", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.7971934513865686", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7714149204832434", - "union_ranking_score": "0.7714149204832434", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.7971934513865686", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.7971934513865686", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.07325639739086803", - "exact_ranking_score": "0.07325639739086803", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.07460890493381468", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.07460890493381468", - "family": "Aurora", - "model_tag": "aurora", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.03479712897675945", - "ranking_selected_val_pr_auc": "0.0949024678484547", - "scope": "top10", - "seed": 7, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.6996715316108496", - "tolerated_ranking_score": "0.6996715316108496", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.7015182768065465", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.7015182768065465", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.6996715316108496", - "union_ranking_score": "0.6996715316108496", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.7015182768065465", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.7015182768065465", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.04101254412979794", - "exact_ranking_score": "0.04101254412979794", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.04253189892419314", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.04253189892419314", - "family": "Aurora", - "model_tag": "aurora", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.01908165402331067", - "ranking_selected_val_pr_auc": "0.05014988402480292", - "scope": "top20", - "seed": 7, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7185324707231184", - "tolerated_ranking_score": "0.7185324707231184", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.6906754133476136", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.6906754133476136", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7185324707231184", - "union_ranking_score": "0.7185324707231184", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.6906754133476136", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.6906754133476136", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0004925501476206198", - "exact_ranking_score": "0.00048497822606044473", - "exact_regret": 7.571921560175121e-06, - "exact_test_gap": "7.571921560175121e-06", - "exact_val_decision_score": "0.0004727121748000204", - "exact_val_gap": "7.267025482891767e-06", - "exact_val_ranking_score": "0.00046544514931712863", - "family": "Aurora", - "model_tag": "aurora", - "ranking_selected_head": "pixel MLP head", - "ranking_selected_head_arch": "pixel_mlp", - "ranking_selected_test_pr_auc": "0.00016818557842652088", - "ranking_selected_val_pr_auc": "0.0004365856110391047", - "scope": "global", - "seed": 99, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.240793572992212", - "tolerated_ranking_score": "0.23755358049655212", - "tolerated_regret": 0.0032399924956598714, - "tolerated_test_gap": "0.0032399924956598714", - "tolerated_val_decision_score": "0.2179368807313983", - "tolerated_val_gap": "0.0025778293487095094", - "tolerated_val_ranking_score": "0.2153590513826888", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.240793572992212", - "union_ranking_score": "0.23755358049655212", - "union_regret": 0.0032399924956598714, - "union_test_gap": "0.0032399924956598714", - "union_val_decision_score": "0.2179368807313983", - "union_val_gap": "0.0025778293487095094", - "union_val_ranking_score": "0.2153590513826888", - "path": "" - }, - { - "exact_decision_head": "pixel MLP head", - "exact_decision_head_arch": "pixel_mlp", - "exact_decision_score": "0.11833577252770228", - "exact_ranking_score": "0.11833577252770228", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.1250696689332293", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.1250696689332293", - "family": "Aurora", - "model_tag": "aurora", - "ranking_selected_head": "pixel MLP head", - "ranking_selected_head_arch": "pixel_mlp", - "ranking_selected_test_pr_auc": "0.04280276014558355", - "ranking_selected_val_pr_auc": "0.20552291444972312", - "scope": "top5", - "seed": 99, - "tolerated_decision_head": "pixel MLP head", - "tolerated_decision_head_arch": "pixel_mlp", - "tolerated_decision_score": "0.7714149204832434", - "tolerated_ranking_score": "0.7714149204832434", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.7985410856039477", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.7985410856039477", - "union_decision_head": "pixel MLP head", - "union_decision_head_arch": "pixel_mlp", - "union_decision_score": "0.7714149204832434", - "union_ranking_score": "0.7714149204832434", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.7985410856039477", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.7985410856039477", - "path": "" - }, - { - "exact_decision_head": "pixel MLP head", - "exact_decision_head_arch": "pixel_mlp", - "exact_decision_score": "0.07325639739086803", - "exact_ranking_score": "0.07325639739086803", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.07801114444920705", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.07801114444920705", - "family": "Aurora", - "model_tag": "aurora", - "ranking_selected_head": "pixel MLP head", - "ranking_selected_head_arch": "pixel_mlp", - "ranking_selected_test_pr_auc": "0.025614133200453124", - "ranking_selected_val_pr_auc": "0.12487452965433544", - "scope": "top10", - "seed": 99, - "tolerated_decision_head": "pixel MLP head", - "tolerated_decision_head_arch": "pixel_mlp", - "tolerated_decision_score": "0.6996715316108496", - "tolerated_ranking_score": "0.6996715316108496", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.7022486149835246", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.7022486149835246", - "union_decision_head": "pixel MLP head", - "union_decision_head_arch": "pixel_mlp", - "union_decision_score": "0.6996715316108496", - "union_ranking_score": "0.6996715316108496", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.7022486149835246", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.7022486149835246", - "path": "" - }, - { - "exact_decision_head": "pixel MLP head", - "exact_decision_head_arch": "pixel_mlp", - "exact_decision_score": "0.04101254412979794", - "exact_ranking_score": "0.04101254412979794", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.04437663936620095", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.04437663936620095", - "family": "Aurora", - "model_tag": "aurora", - "ranking_selected_head": "pixel MLP head", - "ranking_selected_head_arch": "pixel_mlp", - "ranking_selected_test_pr_auc": "0.01412932353396452", - "ranking_selected_val_pr_auc": "0.06804297819080397", - "scope": "top20", - "seed": 99, - "tolerated_decision_head": "pixel MLP head", - "tolerated_decision_head_arch": "pixel_mlp", - "tolerated_decision_score": "0.7185324707231184", - "tolerated_ranking_score": "0.7185324707231184", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.692216769941617", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.692216769941617", - "union_decision_head": "pixel MLP head", - "union_decision_head_arch": "pixel_mlp", - "union_decision_score": "0.7185324707231184", - "union_ranking_score": "0.7185324707231184", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.692216769941617", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.692216769941617", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0004925501476206198", - "exact_ranking_score": "0.0004925501476206198", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.0004727121748000204", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.0004727121748000204", - "family": "ClimaX", - "model_tag": "climax", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.00023234964096770515", - "ranking_selected_val_pr_auc": "0.0002601351527039687", - "scope": "global", - "seed": 1, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.240793572992212", - "tolerated_ranking_score": "0.240793572992212", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.2179368807313983", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.2179368807313983", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.240793572992212", - "union_ranking_score": "0.240793572992212", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.2179368807313983", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.2179368807313983", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.11833577252770228", - "exact_ranking_score": "0.11833577252770228", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.11931877546755824", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.11931877546755824", - "family": "ClimaX", - "model_tag": "climax", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.055890306674778144", - "ranking_selected_val_pr_auc": "0.0694777315217738", - "scope": "top5", - "seed": 1, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7714149204832434", - "tolerated_ranking_score": "0.7714149204832434", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.7971934513865686", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.7971934513865686", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7714149204832434", - "union_ranking_score": "0.7714149204832434", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.7971934513865686", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.7971934513865686", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.07325639739086803", - "exact_ranking_score": "0.07325639739086803", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.07460890493381468", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.07460890493381468", - "family": "ClimaX", - "model_tag": "climax", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.03345772284683149", - "ranking_selected_val_pr_auc": "0.042554440111266106", - "scope": "top10", - "seed": 1, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.6996715316108496", - "tolerated_ranking_score": "0.6996715316108496", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.7015182768065465", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.7015182768065465", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.6996715316108496", - "union_ranking_score": "0.6996715316108496", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.7015182768065465", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.7015182768065465", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.04101254412979794", - "exact_ranking_score": "0.04101254412979794", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.04253189892419314", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.04253189892419314", - "family": "ClimaX", - "model_tag": "climax", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.01871363925580095", - "ranking_selected_val_pr_auc": "0.024212450415123626", - "scope": "top20", - "seed": 1, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7185324707231184", - "tolerated_ranking_score": "0.7185324707231184", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.6906754133476136", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.6906754133476136", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7185324707231184", - "union_ranking_score": "0.7185324707231184", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.6906754133476136", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.6906754133476136", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0004925501476206198", - "exact_ranking_score": "0.0004925501476206198", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.0004727121748000204", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.0004727121748000204", - "family": "ClimaX", - "model_tag": "climax", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.00023488826232608934", - "ranking_selected_val_pr_auc": "0.00022945923295652787", - "scope": "global", - "seed": 123, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.240793572992212", - "tolerated_ranking_score": "0.240793572992212", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.2179368807313983", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.2179368807313983", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.240793572992212", - "union_ranking_score": "0.240793572992212", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.2179368807313983", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.2179368807313983", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.11833577252770228", - "exact_ranking_score": "0.11833577252770228", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.11931877546755824", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.11931877546755824", - "family": "ClimaX", - "model_tag": "climax", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.06647102886669706", - "ranking_selected_val_pr_auc": "0.07019151291101249", - "scope": "top5", - "seed": 123, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7714149204832434", - "tolerated_ranking_score": "0.7714149204832434", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.7971934513865686", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.7971934513865686", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7714149204832434", - "union_ranking_score": "0.7714149204832434", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.7971934513865686", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.7971934513865686", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.07325639739086803", - "exact_ranking_score": "0.07325639739086803", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.07460890493381468", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.07460890493381468", - "family": "ClimaX", - "model_tag": "climax", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.03955291143531272", - "ranking_selected_val_pr_auc": "0.04218583124525834", - "scope": "top10", - "seed": 123, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.6996715316108496", - "tolerated_ranking_score": "0.6996715316108496", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.7015182768065465", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.7015182768065465", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.6996715316108496", - "union_ranking_score": "0.6996715316108496", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.7015182768065465", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.7015182768065465", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.04101254412979794", - "exact_ranking_score": "0.04101254412979794", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.04253189892419314", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.04253189892419314", - "family": "ClimaX", - "model_tag": "climax", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.021395640000026753", - "ranking_selected_val_pr_auc": "0.02312815465123813", - "scope": "top20", - "seed": 123, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7185324707231184", - "tolerated_ranking_score": "0.7185324707231184", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.6906754133476136", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.6906754133476136", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7185324707231184", - "union_ranking_score": "0.7185324707231184", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.6906754133476136", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.6906754133476136", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0004925501476206198", - "exact_ranking_score": "0.00048497822606044473", - "exact_regret": 7.571921560175121e-06, - "exact_test_gap": "7.571921560175121e-06", - "exact_val_decision_score": "0.0004727121748000204", - "exact_val_gap": "7.267025482891767e-06", - "exact_val_ranking_score": "0.00046544514931712863", - "family": "ClimaX", - "model_tag": "climax", - "ranking_selected_head": "linear probe", - "ranking_selected_head_arch": "linear", - "ranking_selected_test_pr_auc": "0.00023439386477689915", - "ranking_selected_val_pr_auc": "0.0002283896136939234", - "scope": "global", - "seed": 42, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.240793572992212", - "tolerated_ranking_score": "0.23755358049655212", - "tolerated_regret": 0.0032399924956598714, - "tolerated_test_gap": "0.0032399924956598714", - "tolerated_val_decision_score": "0.2179368807313983", - "tolerated_val_gap": "0.00299097714352789", - "tolerated_val_ranking_score": "0.2149459035878704", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.240793572992212", - "union_ranking_score": "0.23755358049655212", - "union_regret": 0.0032399924956598714, - "union_test_gap": "0.0032399924956598714", - "union_val_decision_score": "0.2179368807313983", - "union_val_gap": "0.00299097714352789", - "union_val_ranking_score": "0.2149459035878704", - "path": "" - }, - { - "exact_decision_head": "linear probe", - "exact_decision_head_arch": "linear", - "exact_decision_score": "0.11833577252770228", - "exact_ranking_score": "0.11833577252770228", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.11931877546755824", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.11931877546755824", - "family": "ClimaX", - "model_tag": "climax", - "ranking_selected_head": "linear probe", - "ranking_selected_head_arch": "linear", - "ranking_selected_test_pr_auc": "0.06646486382553912", - "ranking_selected_val_pr_auc": "0.07019337939369524", - "scope": "top5", - "seed": 42, - "tolerated_decision_head": "linear probe", - "tolerated_decision_head_arch": "linear", - "tolerated_decision_score": "0.7714149204832434", - "tolerated_ranking_score": "0.7714149204832434", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.7971934513865686", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.7971934513865686", - "union_decision_head": "linear probe", - "union_decision_head_arch": "linear", - "union_decision_score": "0.7714149204832434", - "union_ranking_score": "0.7714149204832434", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.7971934513865686", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.7971934513865686", - "path": "" - }, - { - "exact_decision_head": "linear probe", - "exact_decision_head_arch": "linear", - "exact_decision_score": "0.07325639739086803", - "exact_ranking_score": "0.07325639739086803", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.07460890493381468", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.07460890493381468", - "family": "ClimaX", - "model_tag": "climax", - "ranking_selected_head": "linear probe", - "ranking_selected_head_arch": "linear", - "ranking_selected_test_pr_auc": "0.039777194719281345", - "ranking_selected_val_pr_auc": "0.042300836914518636", - "scope": "top10", - "seed": 42, - "tolerated_decision_head": "linear probe", - "tolerated_decision_head_arch": "linear", - "tolerated_decision_score": "0.6996715316108496", - "tolerated_ranking_score": "0.6996715316108496", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.7015182768065465", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.7015182768065465", - "union_decision_head": "linear probe", - "union_decision_head_arch": "linear", - "union_decision_score": "0.6996715316108496", - "union_ranking_score": "0.6996715316108496", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.7015182768065465", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.7015182768065465", - "path": "" - }, - { - "exact_decision_head": "linear probe", - "exact_decision_head_arch": "linear", - "exact_decision_score": "0.04101254412979794", - "exact_ranking_score": "0.04101254412979794", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.04253189892419314", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.04253189892419314", - "family": "ClimaX", - "model_tag": "climax", - "ranking_selected_head": "linear probe", - "ranking_selected_head_arch": "linear", - "ranking_selected_test_pr_auc": "0.021516770872035896", - "ranking_selected_val_pr_auc": "0.023234418719582723", - "scope": "top20", - "seed": 42, - "tolerated_decision_head": "linear probe", - "tolerated_decision_head_arch": "linear", - "tolerated_decision_score": "0.7185324707231184", - "tolerated_ranking_score": "0.7185324707231184", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.6906754133476136", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.6906754133476136", - "union_decision_head": "linear probe", - "union_decision_head_arch": "linear", - "union_decision_score": "0.7185324707231184", - "union_ranking_score": "0.7185324707231184", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.6906754133476136", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.6906754133476136", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0004925501476206198", - "exact_ranking_score": "0.00048497822606044473", - "exact_regret": 7.571921560175121e-06, - "exact_test_gap": "7.571921560175121e-06", - "exact_val_decision_score": "0.0004727121748000204", - "exact_val_gap": "7.267025482891767e-06", - "exact_val_ranking_score": "0.00046544514931712863", - "family": "ClimaX", - "model_tag": "climax", - "ranking_selected_head": "pixel MLP head", - "ranking_selected_head_arch": "pixel_mlp", - "ranking_selected_test_pr_auc": "0.00025423546642937565", - "ranking_selected_val_pr_auc": "0.00023120715224267713", - "scope": "global", - "seed": 7, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.240793572992212", - "tolerated_ranking_score": "0.23755358049655212", - "tolerated_regret": 0.0032399924956598714, - "tolerated_test_gap": "0.0032399924956598714", - "tolerated_val_decision_score": "0.2179368807313983", - "tolerated_val_gap": "0.00299097714352789", - "tolerated_val_ranking_score": "0.2149459035878704", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.240793572992212", - "union_ranking_score": "0.23755358049655212", - "union_regret": 0.0032399924956598714, - "union_test_gap": "0.0032399924956598714", - "union_val_decision_score": "0.2179368807313983", - "union_val_gap": "0.00299097714352789", - "union_val_ranking_score": "0.2149459035878704", - "path": "" - }, - { - "exact_decision_head": "linear probe", - "exact_decision_head_arch": "linear", - "exact_decision_score": "0.11833577252770228", - "exact_ranking_score": "0.11833577252770228", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.11931877546755824", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.11931877546755824", - "family": "ClimaX", - "model_tag": "climax", - "ranking_selected_head": "linear probe", - "ranking_selected_head_arch": "linear", - "ranking_selected_test_pr_auc": "0.06582645251185222", - "ranking_selected_val_pr_auc": "0.07011813439504147", - "scope": "top5", - "seed": 7, - "tolerated_decision_head": "linear probe", - "tolerated_decision_head_arch": "linear", - "tolerated_decision_score": "0.7714149204832434", - "tolerated_ranking_score": "0.7714149204832434", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.7971934513865686", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.7971934513865686", - "union_decision_head": "linear probe", - "union_decision_head_arch": "linear", - "union_decision_score": "0.7714149204832434", - "union_ranking_score": "0.7714149204832434", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.7971934513865686", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.7971934513865686", - "path": "" - }, - { - "exact_decision_head": "linear probe", - "exact_decision_head_arch": "linear", - "exact_decision_score": "0.07325639739086803", - "exact_ranking_score": "0.07325639739086803", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.07460890493381468", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.07460890493381468", - "family": "ClimaX", - "model_tag": "climax", - "ranking_selected_head": "linear probe", - "ranking_selected_head_arch": "linear", - "ranking_selected_test_pr_auc": "0.03938368107690723", - "ranking_selected_val_pr_auc": "0.042249808498221526", - "scope": "top10", - "seed": 7, - "tolerated_decision_head": "linear probe", - "tolerated_decision_head_arch": "linear", - "tolerated_decision_score": "0.6996715316108496", - "tolerated_ranking_score": "0.6996715316108496", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.7015182768065465", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.7015182768065465", - "union_decision_head": "linear probe", - "union_decision_head_arch": "linear", - "union_decision_score": "0.6996715316108496", - "union_ranking_score": "0.6996715316108496", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.7015182768065465", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.7015182768065465", - "path": "" - }, - { - "exact_decision_head": "linear probe", - "exact_decision_head_arch": "linear", - "exact_decision_score": "0.04101254412979794", - "exact_ranking_score": "0.04101254412979794", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.04253189892419314", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.04253189892419314", - "family": "ClimaX", - "model_tag": "climax", - "ranking_selected_head": "linear probe", - "ranking_selected_head_arch": "linear", - "ranking_selected_test_pr_auc": "0.021317405351800135", - "ranking_selected_val_pr_auc": "0.02320587721933335", - "scope": "top20", - "seed": 7, - "tolerated_decision_head": "linear probe", - "tolerated_decision_head_arch": "linear", - "tolerated_decision_score": "0.7185324707231184", - "tolerated_ranking_score": "0.7185324707231184", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.6906754133476136", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.6906754133476136", - "union_decision_head": "linear probe", - "union_decision_head_arch": "linear", - "union_decision_score": "0.7185324707231184", - "union_ranking_score": "0.7185324707231184", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.6906754133476136", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.6906754133476136", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.00048696535779102983", - "exact_ranking_score": "0.00048696535779102983", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.00046735226565276225", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.00046735226565276225", - "family": "ClimaX", - "model_tag": "climax", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.0002540711776626832", - "ranking_selected_val_pr_auc": "0.00022678877479338636", - "scope": "global", - "seed": 99, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.2384111045733381", - "tolerated_ranking_score": "0.2384111045733381", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.21573180756095836", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.21573180756095836", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.2384111045733381", - "union_ranking_score": "0.2384111045733381", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.21573180756095836", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.21573180756095836", - "path": "" - }, - { - "exact_decision_head": "linear probe", - "exact_decision_head_arch": "linear", - "exact_decision_score": "0.11833577252770228", - "exact_ranking_score": "0.11833577252770228", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.11931877546755824", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.11931877546755824", - "family": "ClimaX", - "model_tag": "climax", - "ranking_selected_head": "linear probe", - "ranking_selected_head_arch": "linear", - "ranking_selected_test_pr_auc": "0.06558789102558008", - "ranking_selected_val_pr_auc": "0.0697713050234347", - "scope": "top5", - "seed": 99, - "tolerated_decision_head": "linear probe", - "tolerated_decision_head_arch": "linear", - "tolerated_decision_score": "0.7714149204832434", - "tolerated_ranking_score": "0.7714149204832434", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.7971934513865686", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.7971934513865686", - "union_decision_head": "linear probe", - "union_decision_head_arch": "linear", - "union_decision_score": "0.7714149204832434", - "union_ranking_score": "0.7714149204832434", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.7971934513865686", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.7971934513865686", - "path": "" - }, - { - "exact_decision_head": "linear probe", - "exact_decision_head_arch": "linear", - "exact_decision_score": "0.07325639739086803", - "exact_ranking_score": "0.07325639739086803", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.07460890493381468", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.07460890493381468", - "family": "ClimaX", - "model_tag": "climax", - "ranking_selected_head": "linear probe", - "ranking_selected_head_arch": "linear", - "ranking_selected_test_pr_auc": "0.039026227487447884", - "ranking_selected_val_pr_auc": "0.04197164598362404", - "scope": "top10", - "seed": 99, - "tolerated_decision_head": "linear probe", - "tolerated_decision_head_arch": "linear", - "tolerated_decision_score": "0.6996715316108496", - "tolerated_ranking_score": "0.6996715316108496", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.7015182768065465", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.7015182768065465", - "union_decision_head": "linear probe", - "union_decision_head_arch": "linear", - "union_decision_score": "0.6996715316108496", - "union_ranking_score": "0.6996715316108496", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.7015182768065465", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.7015182768065465", - "path": "" - }, - { - "exact_decision_head": "linear probe", - "exact_decision_head_arch": "linear", - "exact_decision_score": "0.04101254412979794", - "exact_ranking_score": "0.04101254412979794", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.04253189892419314", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.04253189892419314", - "family": "ClimaX", - "model_tag": "climax", - "ranking_selected_head": "linear probe", - "ranking_selected_head_arch": "linear", - "ranking_selected_test_pr_auc": "0.021159149554047907", - "ranking_selected_val_pr_auc": "0.023032008375736605", - "scope": "top20", - "seed": 99, - "tolerated_decision_head": "linear probe", - "tolerated_decision_head_arch": "linear", - "tolerated_decision_score": "0.7185324707231184", - "tolerated_ranking_score": "0.7185324707231184", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.6906754133476136", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.6906754133476136", - "union_decision_head": "linear probe", - "union_decision_head_arch": "linear", - "union_decision_score": "0.7185324707231184", - "union_ranking_score": "0.7185324707231184", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.6906754133476136", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.6906754133476136", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0053407639614536165", - "exact_ranking_score": "0.0053407639614536165", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.0016902678135446794", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.0016902678135446794", - "family": "DLWP", - "model_tag": "dlwp", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.0006824870760177812", - "ranking_selected_val_pr_auc": "0.00042302773922457535", - "scope": "global", - "seed": 1, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.3560602558354403", - "tolerated_ranking_score": "0.3560602558354403", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.3983991584038598", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.3983991584038598", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.3560602558354403", - "union_ranking_score": "0.3560602558354403", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.3983991584038598", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.3983991584038598", - "path": "" - }, - { - "exact_decision_head": "linear probe", - "exact_decision_head_arch": "linear", - "exact_decision_score": "0.12001093493712411", - "exact_ranking_score": "0.12001093493712411", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.1225081551286698", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.1225081551286698", - "family": "DLWP", - "model_tag": "dlwp", - "ranking_selected_head": "linear probe", - "ranking_selected_head_arch": "linear", - "ranking_selected_test_pr_auc": "0.05689342648649434", - "ranking_selected_val_pr_auc": "0.05377805405038012", - "scope": "top5", - "seed": 1, - "tolerated_decision_head": "linear probe", - "tolerated_decision_head_arch": "linear", - "tolerated_decision_score": "0.7748819895562846", - "tolerated_ranking_score": "0.7748819895562846", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.8195894738438443", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.8195894738438443", - "union_decision_head": "linear probe", - "union_decision_head_arch": "linear", - "union_decision_score": "0.7748819895562846", - "union_ranking_score": "0.7748819895562846", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.8195894738438443", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.8195894738438443", - "path": "" - }, - { - "exact_decision_head": "linear probe", - "exact_decision_head_arch": "linear", - "exact_decision_score": "0.078002052685597", - "exact_ranking_score": "0.078002052685597", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.07851948205522379", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.07851948205522379", - "family": "DLWP", - "model_tag": "dlwp", - "ranking_selected_head": "linear probe", - "ranking_selected_head_arch": "linear", - "ranking_selected_test_pr_auc": "0.03678683052582872", - "ranking_selected_val_pr_auc": "0.03453391215275296", - "scope": "top10", - "seed": 1, - "tolerated_decision_head": "linear probe", - "tolerated_decision_head_arch": "linear", - "tolerated_decision_score": "0.723747005823518", - "tolerated_ranking_score": "0.723747005823518", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.7278456297612906", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.7278456297612906", - "union_decision_head": "linear probe", - "union_decision_head_arch": "linear", - "union_decision_score": "0.723747005823518", - "union_ranking_score": "0.723747005823518", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.7278456297612906", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.7278456297612906", - "path": "" - }, - { - "exact_decision_head": "linear probe", - "exact_decision_head_arch": "linear", - "exact_decision_score": "0.043506471331489265", - "exact_ranking_score": "0.043506471331489265", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.04461624715984538", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.04461624715984538", - "family": "DLWP", - "model_tag": "dlwp", - "ranking_selected_head": "linear probe", - "ranking_selected_head_arch": "linear", - "ranking_selected_test_pr_auc": "0.019747545663020845", - "ranking_selected_val_pr_auc": "0.019740435565559492", - "scope": "top20", - "seed": 1, - "tolerated_decision_head": "linear probe", - "tolerated_decision_head_arch": "linear", - "tolerated_decision_score": "0.7280364139105968", - "tolerated_ranking_score": "0.7280364139105968", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.7120828721033882", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.7120828721033882", - "union_decision_head": "linear probe", - "union_decision_head_arch": "linear", - "union_decision_score": "0.7280364139105968", - "union_ranking_score": "0.7280364139105968", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.7120828721033882", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.7120828721033882", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0", - "exact_ranking_score": "0.0", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.0020376974019358125", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.0020376974019358125", - "family": "DLWP", - "model_tag": "dlwp", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.0007448529317390826", - "ranking_selected_val_pr_auc": "0.00044761016558069356", - "scope": "global", - "seed": 123, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.3859134431294053", - "tolerated_ranking_score": "0.3859134431294053", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.4280727908627396", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.4280727908627396", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.3859134431294053", - "union_ranking_score": "0.3859134431294053", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.4280727908627396", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.4280727908627396", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.13353877760657423", - "exact_ranking_score": "0.11833577252770228", - "exact_regret": 0.015203005078871956, - "exact_test_gap": "0.015203005078871956", - "exact_val_decision_score": "0.12187299550994227", - "exact_val_gap": "0.0025542200423840228", - "exact_val_ranking_score": "0.11931877546755824", - "family": "DLWP", - "model_tag": "dlwp", - "ranking_selected_head": "pixel MLP head", - "ranking_selected_head_arch": "pixel_mlp", - "ranking_selected_test_pr_auc": "0.06589923575225648", - "ranking_selected_val_pr_auc": "0.06980014500652997", - "scope": "top5", - "seed": 123, - "tolerated_decision_head": "linear probe", - "tolerated_decision_head_arch": "linear", - "tolerated_decision_score": "0.7910976805607687", - "tolerated_ranking_score": "0.7714149204832434", - "tolerated_regret": 0.01968276007752523, - "tolerated_test_gap": "0.01968276007752523", - "tolerated_val_decision_score": "0.8156849318846232", - "tolerated_val_gap": "0.018491480498054602", - "tolerated_val_ranking_score": "0.7971934513865686", - "union_decision_head": "linear probe", - "union_decision_head_arch": "linear", - "union_decision_score": "0.7910976805607687", - "union_ranking_score": "0.7714149204832434", - "union_regret": 0.01968276007752523, - "union_test_gap": "0.01968276007752523", - "union_val_decision_score": "0.8156849318846232", - "union_val_gap": "0.018491480498054602", - "union_val_ranking_score": "0.7971934513865686", - "path": "" - }, - { - "exact_decision_head": "linear probe", - "exact_decision_head_arch": "linear", - "exact_decision_score": "0.07637343329038304", - "exact_ranking_score": "0.07325639739086803", - "exact_regret": 0.0031170358995150083, - "exact_test_gap": "0.0031170358995150083", - "exact_val_decision_score": "0.07697248688577239", - "exact_val_gap": "0.0023635819519577106", - "exact_val_ranking_score": "0.07460890493381468", - "family": "DLWP", - "model_tag": "dlwp", - "ranking_selected_head": "pixel MLP head", - "ranking_selected_head_arch": "pixel_mlp", - "ranking_selected_test_pr_auc": "0.03734242132263064", - "ranking_selected_val_pr_auc": "0.04123208357155757", - "scope": "top10", - "seed": 123, - "tolerated_decision_head": "linear probe", - "tolerated_decision_head_arch": "linear", - "tolerated_decision_score": "0.7355445123185741", - "tolerated_ranking_score": "0.6996715316108496", - "tolerated_regret": 0.03587298070772449, - "tolerated_test_gap": "0.03587298070772449", - "tolerated_val_decision_score": "0.7311417346046021", - "tolerated_val_gap": "0.02962345779805553", - "tolerated_val_ranking_score": "0.7015182768065465", - "union_decision_head": "linear probe", - "union_decision_head_arch": "linear", - "union_decision_score": "0.7355445123185741", - "union_ranking_score": "0.6996715316108496", - "union_regret": 0.03587298070772449, - "union_test_gap": "0.03587298070772449", - "union_val_decision_score": "0.7311417346046021", - "union_val_gap": "0.02962345779805553", - "union_val_ranking_score": "0.7015182768065465", - "path": "" - }, - { - "exact_decision_head": "linear probe", - "exact_decision_head_arch": "linear", - "exact_decision_score": "0.04249854057209574", - "exact_ranking_score": "0.04101254412979794", - "exact_regret": 0.001485996442297799, - "exact_test_gap": "0.001485996442297799", - "exact_val_decision_score": "0.04371409485637943", - "exact_val_gap": "0.0011821959321862904", - "exact_val_ranking_score": "0.04253189892419314", - "family": "DLWP", - "model_tag": "dlwp", - "ranking_selected_head": "pixel MLP head", - "ranking_selected_head_arch": "pixel_mlp", - "ranking_selected_test_pr_auc": "0.021118517500793188", - "ranking_selected_val_pr_auc": "0.02303236785318257", - "scope": "top20", - "seed": 123, - "tolerated_decision_head": "linear probe", - "tolerated_decision_head_arch": "linear", - "tolerated_decision_score": "0.7321459738801157", - "tolerated_ranking_score": "0.7185324707231184", - "tolerated_regret": 0.013613503156997275, - "tolerated_test_gap": "0.013613503156997275", - "tolerated_val_decision_score": "0.7128995538153613", - "tolerated_val_gap": "0.022224140467747677", - "tolerated_val_ranking_score": "0.6906754133476136", - "union_decision_head": "linear probe", - "union_decision_head_arch": "linear", - "union_decision_score": "0.7321459738801157", - "union_ranking_score": "0.7185324707231184", - "union_regret": 0.013613503156997275, - "union_test_gap": "0.013613503156997275", - "union_val_decision_score": "0.7128995538153613", - "union_val_gap": "0.022224140467747677", - "union_val_ranking_score": "0.6906754133476136", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0", - "exact_ranking_score": "0.0", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.0020120724346076456", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.0020120724346076456", - "family": "DLWP", - "model_tag": "dlwp", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.0006203713852571992", - "ranking_selected_val_pr_auc": "0.0004707444097300293", - "scope": "global", - "seed": 42, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.3405125814370199", - "tolerated_ranking_score": "0.3405125814370199", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.474350305376108", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.474350305376108", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.3405125814370199", - "union_ranking_score": "0.3405125814370199", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.474350305376108", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.474350305376108", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.1227865796831314", - "exact_ranking_score": "0.11833577252770228", - "exact_regret": 0.004450807155429129, - "exact_test_gap": "0.004450807155429129", - "exact_val_decision_score": "0.1210664835793569", - "exact_val_gap": "0.0017477081117986543", - "exact_val_ranking_score": "0.11931877546755824", - "family": "DLWP", - "model_tag": "dlwp", - "ranking_selected_head": "pixel MLP head", - "ranking_selected_head_arch": "pixel_mlp", - "ranking_selected_test_pr_auc": "0.058186924071015014", - "ranking_selected_val_pr_auc": "0.055593579382965747", - "scope": "top5", - "seed": 42, - "tolerated_decision_head": "pixel MLP head", - "tolerated_decision_head_arch": "pixel_mlp", - "tolerated_decision_score": "0.7049597657509027", - "tolerated_ranking_score": "0.7049597657509027", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.8131649530621368", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.8131649530621368", - "union_decision_head": "pixel MLP head", - "union_decision_head_arch": "pixel_mlp", - "union_decision_score": "0.7049597657509027", - "union_ranking_score": "0.7049597657509027", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.8131649530621368", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.8131649530621368", - "path": "" - }, - { - "exact_decision_head": "pixel MLP head", - "exact_decision_head_arch": "pixel_mlp", - "exact_decision_score": "0.0811232449297972", - "exact_ranking_score": "0.0811232449297972", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.07607950651130911", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.07607950651130911", - "family": "DLWP", - "model_tag": "dlwp", - "ranking_selected_head": "pixel MLP head", - "ranking_selected_head_arch": "pixel_mlp", - "ranking_selected_test_pr_auc": "0.03845833667153789", - "ranking_selected_val_pr_auc": "0.036365711264814124", - "scope": "top10", - "seed": 42, - "tolerated_decision_head": "pixel MLP head", - "tolerated_decision_head_arch": "pixel_mlp", - "tolerated_decision_score": "0.6763081901295341", - "tolerated_ranking_score": "0.6763081901295341", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.7265137957010752", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.7265137957010752", - "union_decision_head": "pixel MLP head", - "union_decision_head_arch": "pixel_mlp", - "union_decision_score": "0.6763081901295341", - "union_ranking_score": "0.6763081901295341", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.7265137957010752", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.7265137957010752", - "path": "" - }, - { - "exact_decision_head": "pixel MLP head", - "exact_decision_head_arch": "pixel_mlp", - "exact_decision_score": "0.04637177602565815", - "exact_ranking_score": "0.04637177602565815", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.044021164021164016", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.044021164021164016", - "family": "DLWP", - "model_tag": "dlwp", - "ranking_selected_head": "pixel MLP head", - "ranking_selected_head_arch": "pixel_mlp", - "ranking_selected_test_pr_auc": "0.020762153794205103", - "ranking_selected_val_pr_auc": "0.020894537350314525", - "scope": "top20", - "seed": 42, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7728400679088107", - "tolerated_ranking_score": "0.6606900017471591", - "tolerated_regret": 0.11215006616165157, - "tolerated_test_gap": "0.11215006616165157", - "tolerated_val_decision_score": "0.7122584178150692", - "tolerated_val_gap": "0.009110393702543962", - "tolerated_val_ranking_score": "0.7031480241125252", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7728400679088107", - "union_ranking_score": "0.6606900017471591", - "union_regret": 0.11215006616165157, - "union_test_gap": "0.11215006616165157", - "union_val_decision_score": "0.7122584178150692", - "union_val_gap": "0.009110393702543962", - "union_val_ranking_score": "0.7031480241125252", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0", - "exact_ranking_score": "0.0", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.0010056989607777405", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.0010056989607777405", - "family": "DLWP", - "model_tag": "dlwp", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.0005264872646085452", - "ranking_selected_val_pr_auc": "0.00035284338324857376", - "scope": "global", - "seed": 7, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.3432315705541329", - "tolerated_ranking_score": "0.3432315705541329", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.3469162743670964", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.3469162743670964", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.3432315705541329", - "union_ranking_score": "0.3432315705541329", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.3469162743670964", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.3469162743670964", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.12270081490104773", - "exact_ranking_score": "0.11833577252770228", - "exact_regret": 0.004365042373345451, - "exact_test_gap": "0.004365042373345451", - "exact_val_decision_score": "0.12128657583203037", - "exact_val_gap": "0.001967800364472125", - "exact_val_ranking_score": "0.11931877546755824", - "family": "DLWP", - "model_tag": "dlwp", - "ranking_selected_head": "pixel MLP head", - "ranking_selected_head_arch": "pixel_mlp", - "ranking_selected_test_pr_auc": "0.05890031091946385", - "ranking_selected_val_pr_auc": "0.05786838028004154", - "scope": "top5", - "seed": 7, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.8044654939106901", - "tolerated_ranking_score": "0.7714149204832434", - "tolerated_regret": 0.03305057342744666, - "tolerated_test_gap": "0.03305057342744666", - "tolerated_val_decision_score": "0.8071267256214067", - "tolerated_val_gap": "0.009933274234838141", - "tolerated_val_ranking_score": "0.7971934513865686", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.8044654939106901", - "union_ranking_score": "0.7714149204832434", - "union_regret": 0.03305057342744666, - "union_test_gap": "0.03305057342744666", - "union_val_decision_score": "0.8071267256214067", - "union_val_gap": "0.009933274234838141", - "union_val_ranking_score": "0.7971934513865686", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.07878017789072428", - "exact_ranking_score": "0.07325639739086803", - "exact_regret": 0.005523780499856246, - "exact_test_gap": "0.005523780499856246", - "exact_val_decision_score": "0.07624569272001312", - "exact_val_gap": "0.0016367877861984403", - "exact_val_ranking_score": "0.07460890493381468", - "family": "DLWP", - "model_tag": "dlwp", - "ranking_selected_head": "pixel MLP head", - "ranking_selected_head_arch": "pixel_mlp", - "ranking_selected_test_pr_auc": "0.03308736705588518", - "ranking_selected_val_pr_auc": "0.03446421323907367", - "scope": "top10", - "seed": 7, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7521966134873119", - "tolerated_ranking_score": "0.6996715316108496", - "tolerated_regret": 0.05252508187646232, - "tolerated_test_gap": "0.05252508187646232", - "tolerated_val_decision_score": "0.7227897891452092", - "tolerated_val_gap": "0.021271512338662646", - "tolerated_val_ranking_score": "0.7015182768065465", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7521966134873119", - "union_ranking_score": "0.6996715316108496", - "union_regret": 0.05252508187646232, - "union_test_gap": "0.05252508187646232", - "union_val_decision_score": "0.7227897891452092", - "union_val_gap": "0.021271512338662646", - "union_val_ranking_score": "0.7015182768065465", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.043377707581227436", - "exact_ranking_score": "0.04101254412979794", - "exact_regret": 0.0023651634514294945, - "exact_test_gap": "0.0023651634514294945", - "exact_val_decision_score": "0.04303681559882578", - "exact_val_gap": "0.0005049166746326425", - "exact_val_ranking_score": "0.04253189892419314", - "family": "DLWP", - "model_tag": "dlwp", - "ranking_selected_head": "pixel MLP head", - "ranking_selected_head_arch": "pixel_mlp", - "ranking_selected_test_pr_auc": "0.018519739310339497", - "ranking_selected_val_pr_auc": "0.01953230111607746", - "scope": "top20", - "seed": 7, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.762536895087842", - "tolerated_ranking_score": "0.7185324707231184", - "tolerated_regret": 0.044004424364723516, - "tolerated_test_gap": "0.044004424364723516", - "tolerated_val_decision_score": "0.7274867592832499", - "tolerated_val_gap": "0.036811345935636286", - "tolerated_val_ranking_score": "0.6906754133476136", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.762536895087842", - "union_ranking_score": "0.7185324707231184", - "union_regret": 0.044004424364723516, - "union_test_gap": "0.044004424364723516", - "union_val_decision_score": "0.7274867592832499", - "union_val_gap": "0.036811345935636286", - "union_val_ranking_score": "0.6906754133476136", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0", - "exact_ranking_score": "0.0", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.0016250253910217347", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.0016250253910217347", - "family": "DLWP", - "model_tag": "dlwp", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.0007269780629045868", - "ranking_selected_val_pr_auc": "0.0004442608756800871", - "scope": "global", - "seed": 99, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.3494091553610714", - "tolerated_ranking_score": "0.3494091553610714", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.41347824096521096", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.41347824096521096", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.3494091553610714", - "union_ranking_score": "0.3494091553610714", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.41347824096521096", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.41347824096521096", - "path": "" - }, - { - "exact_decision_head": "linear probe", - "exact_decision_head_arch": "linear", - "exact_decision_score": "0.14089481946624804", - "exact_ranking_score": "0.14089481946624804", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.15619694397283532", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.15619694397283532", - "family": "DLWP", - "model_tag": "dlwp", - "ranking_selected_head": "linear probe", - "ranking_selected_head_arch": "linear", - "ranking_selected_test_pr_auc": "0.0672622238447139", - "ranking_selected_val_pr_auc": "0.07740571441869959", - "scope": "top5", - "seed": 99, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.8022627296490505", - "tolerated_ranking_score": "0.7714149204832434", - "tolerated_regret": 0.030847809165807094, - "tolerated_test_gap": "0.030847809165807094", - "tolerated_val_decision_score": "0.8078282610569877", - "tolerated_val_gap": "0.010634809670419076", - "tolerated_val_ranking_score": "0.7971934513865686", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.8022627296490505", - "union_ranking_score": "0.7714149204832434", - "union_regret": 0.030847809165807094, - "union_test_gap": "0.030847809165807094", - "union_val_decision_score": "0.8078282610569877", - "union_val_gap": "0.010634809670419076", - "union_val_ranking_score": "0.7971934513865686", - "path": "" - }, - { - "exact_decision_head": "linear probe", - "exact_decision_head_arch": "linear", - "exact_decision_score": "0.08289870985451551", - "exact_ranking_score": "0.08289870985451551", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.09763863093658795", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.09763863093658795", - "family": "DLWP", - "model_tag": "dlwp", - "ranking_selected_head": "linear probe", - "ranking_selected_head_arch": "linear", - "ranking_selected_test_pr_auc": "0.037689588394968596", - "ranking_selected_val_pr_auc": "0.0445507563162313", - "scope": "top10", - "seed": 99, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7536939930764294", - "tolerated_ranking_score": "0.6997663155755749", - "tolerated_regret": 0.053927677500854476, - "tolerated_test_gap": "0.053927677500854476", - "tolerated_val_decision_score": "0.7243805445090241", - "tolerated_val_gap": "0.02259882240347566", - "tolerated_val_ranking_score": "0.7017817221055485", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7536939930764294", - "union_ranking_score": "0.6997663155755749", - "union_regret": 0.053927677500854476, - "union_test_gap": "0.053927677500854476", - "union_val_decision_score": "0.7243805445090241", - "union_val_gap": "0.02259882240347566", - "union_val_ranking_score": "0.7017817221055485", - "path": "" - }, - { - "exact_decision_head": "linear probe", - "exact_decision_head_arch": "linear", - "exact_decision_score": "0.04819843096725701", - "exact_ranking_score": "0.04819843096725701", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.05605272285813389", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.05605272285813389", - "family": "DLWP", - "model_tag": "dlwp", - "ranking_selected_head": "linear probe", - "ranking_selected_head_arch": "linear", - "ranking_selected_test_pr_auc": "0.02136633936888583", - "ranking_selected_val_pr_auc": "0.024917289620763165", - "scope": "top20", - "seed": 99, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7721727827291924", - "tolerated_ranking_score": "0.7187730423241409", - "tolerated_regret": 0.05339974040505158, - "tolerated_test_gap": "0.05339974040505158", - "tolerated_val_decision_score": "0.7238375070567806", - "tolerated_val_gap": "0.032337052985272896", - "tolerated_val_ranking_score": "0.6915004540715077", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7721727827291924", - "union_ranking_score": "0.7187730423241409", - "union_regret": 0.05339974040505158, - "union_test_gap": "0.05339974040505158", - "union_val_decision_score": "0.7238375070567806", - "union_val_gap": "0.032337052985272896", - "union_val_ranking_score": "0.6915004540715077", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.000946969696969697", - "exact_ranking_score": "0.000946969696969697", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.004184100418410042", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.004184100418410042", - "family": "FCN", - "model_tag": "fcn", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.0003658682156259175", - "ranking_selected_val_pr_auc": "0.0004302667397755488", - "scope": "global", - "seed": 1, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.31130402819686975", - "tolerated_ranking_score": "0.31130402819686975", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.3016823150717557", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.3016823150717557", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.31130402819686975", - "union_ranking_score": "0.31130402819686975", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.3016823150717557", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.3016823150717557", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.12059329320722269", - "exact_ranking_score": "0.11833577252770228", - "exact_regret": 0.002257520679520411, - "exact_test_gap": "0.002257520679520411", - "exact_val_decision_score": "0.12220272049144364", - "exact_val_gap": "0.0028839450238853925", - "exact_val_ranking_score": "0.11931877546755824", - "family": "FCN", - "model_tag": "fcn", - "ranking_selected_head": "pixel MLP head", - "ranking_selected_head_arch": "pixel_mlp", - "ranking_selected_test_pr_auc": "0.0689177522952707", - "ranking_selected_val_pr_auc": "0.06437104063472687", - "scope": "top5", - "seed": 1, - "tolerated_decision_head": "linear probe", - "tolerated_decision_head_arch": "linear", - "tolerated_decision_score": "0.7714385719285964", - "tolerated_ranking_score": "0.7714149204832434", - "tolerated_regret": 2.365144535299013e-05, - "tolerated_test_gap": "2.365144535299013e-05", - "tolerated_val_decision_score": "0.7976559342583861", - "tolerated_val_gap": "0.00046248287181749337", - "tolerated_val_ranking_score": "0.7971934513865686", - "union_decision_head": "linear probe", - "union_decision_head_arch": "linear", - "union_decision_score": "0.7714385719285964", - "union_ranking_score": "0.7714149204832434", - "union_regret": 2.365144535299013e-05, - "union_test_gap": "2.365144535299013e-05", - "union_val_decision_score": "0.7976559342583861", - "union_val_gap": "0.00046248287181749337", - "union_val_ranking_score": "0.7971934513865686", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.07431424013174823", - "exact_ranking_score": "0.07325639739086803", - "exact_regret": 0.0010578427408801938, - "exact_test_gap": "0.0010578427408801938", - "exact_val_decision_score": "0.07578837633567892", - "exact_val_gap": "0.0011794714018642377", - "exact_val_ranking_score": "0.07460890493381468", - "family": "FCN", - "model_tag": "fcn", - "ranking_selected_head": "pixel MLP head", - "ranking_selected_head_arch": "pixel_mlp", - "ranking_selected_test_pr_auc": "0.03927901160149114", - "ranking_selected_val_pr_auc": "0.03716454445172032", - "scope": "top10", - "seed": 1, - "tolerated_decision_head": "linear probe", - "tolerated_decision_head_arch": "linear", - "tolerated_decision_score": "0.6756347251105248", - "tolerated_ranking_score": "0.6996715316108496", - "tolerated_regret": 0.0, - "tolerated_test_gap": "-0.024036806500324803", - "tolerated_val_decision_score": "0.7162675437027928", - "tolerated_val_gap": "0.014749266896246227", - "tolerated_val_ranking_score": "0.7015182768065465", - "union_decision_head": "linear probe", - "union_decision_head_arch": "linear", - "union_decision_score": "0.6756347251105248", - "union_ranking_score": "0.6996715316108496", - "union_regret": 0.0, - "union_test_gap": "-0.024036806500324803", - "union_val_decision_score": "0.7162675437027928", - "union_val_gap": "0.014749266896246227", - "union_val_ranking_score": "0.7015182768065465", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.042069180430040515", - "exact_ranking_score": "0.042069180430040515", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.04316055094101566", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.04316055094101566", - "family": "FCN", - "model_tag": "fcn", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.017947733209238474", - "ranking_selected_val_pr_auc": "0.019698135675792587", - "scope": "top20", - "seed": 1, - "tolerated_decision_head": "linear probe", - "tolerated_decision_head_arch": "linear", - "tolerated_decision_score": "0.7182175622542595", - "tolerated_ranking_score": "0.7185324707231184", - "tolerated_regret": 0.0, - "tolerated_test_gap": "-0.0003149084688589543", - "tolerated_val_decision_score": "0.6906967717983766", - "tolerated_val_gap": "2.135845076300935e-05", - "tolerated_val_ranking_score": "0.6906754133476136", - "union_decision_head": "linear probe", - "union_decision_head_arch": "linear", - "union_decision_score": "0.7182175622542595", - "union_ranking_score": "0.7185324707231184", - "union_regret": 0.0, - "union_test_gap": "-0.0003149084688589543", - "union_val_decision_score": "0.6906967717983766", - "union_val_gap": "2.135845076300935e-05", - "union_val_ranking_score": "0.6906754133476136", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0006047777441790143", - "exact_ranking_score": "0.0006047777441790143", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.0034982508745627187", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.0034982508745627187", - "family": "FCN", - "model_tag": "fcn", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.0004059591718395402", - "ranking_selected_val_pr_auc": "0.00044924149635682735", - "scope": "global", - "seed": 123, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.3100602050525378", - "tolerated_ranking_score": "0.3100602050525378", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.30078743758851223", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.30078743758851223", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.3100602050525378", - "union_ranking_score": "0.3100602050525378", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.30078743758851223", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.30078743758851223", - "path": "" - }, - { - "exact_decision_head": "pixel MLP head", - "exact_decision_head_arch": "pixel_mlp", - "exact_decision_score": "0.11833577252770228", - "exact_ranking_score": "0.11833577252770228", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.11931877546755824", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.11931877546755824", - "family": "FCN", - "model_tag": "fcn", - "ranking_selected_head": "pixel MLP head", - "ranking_selected_head_arch": "pixel_mlp", - "ranking_selected_test_pr_auc": "0.058898489734440555", - "ranking_selected_val_pr_auc": "0.07226448398929738", - "scope": "top5", - "seed": 123, - "tolerated_decision_head": "linear probe", - "tolerated_decision_head_arch": "linear", - "tolerated_decision_score": "0.793939393939394", - "tolerated_ranking_score": "0.7714149204832434", - "tolerated_regret": 0.022524473456150496, - "tolerated_test_gap": "0.022524473456150496", - "tolerated_val_decision_score": "0.8166754936823787", - "tolerated_val_gap": "0.019482042295810076", - "tolerated_val_ranking_score": "0.7971934513865686", - "union_decision_head": "linear probe", - "union_decision_head_arch": "linear", - "union_decision_score": "0.793939393939394", - "union_ranking_score": "0.7714149204832434", - "union_regret": 0.022524473456150496, - "union_test_gap": "0.022524473456150496", - "union_val_decision_score": "0.8166754936823787", - "union_val_gap": "0.019482042295810076", - "union_val_ranking_score": "0.7971934513865686", - "path": "" - }, - { - "exact_decision_head": "pixel MLP head", - "exact_decision_head_arch": "pixel_mlp", - "exact_decision_score": "0.07325639739086803", - "exact_ranking_score": "0.07325639739086803", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.07460890493381468", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.07460890493381468", - "family": "FCN", - "model_tag": "fcn", - "ranking_selected_head": "pixel MLP head", - "ranking_selected_head_arch": "pixel_mlp", - "ranking_selected_test_pr_auc": "0.03552513494863446", - "ranking_selected_val_pr_auc": "0.042774542140641386", - "scope": "top10", - "seed": 123, - "tolerated_decision_head": "linear probe", - "tolerated_decision_head_arch": "linear", - "tolerated_decision_score": "0.7206692168649377", - "tolerated_ranking_score": "0.6996715316108496", - "tolerated_regret": 0.020997685254088072, - "tolerated_test_gap": "0.020997685254088072", - "tolerated_val_decision_score": "0.7230799450009822", - "tolerated_val_gap": "0.021561668194435613", - "tolerated_val_ranking_score": "0.7015182768065465", - "union_decision_head": "linear probe", - "union_decision_head_arch": "linear", - "union_decision_score": "0.7206692168649377", - "union_ranking_score": "0.6996715316108496", - "union_regret": 0.020997685254088072, - "union_test_gap": "0.020997685254088072", - "union_val_decision_score": "0.7230799450009822", - "union_val_gap": "0.021561668194435613", - "union_val_ranking_score": "0.7015182768065465", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.04104131693016963", - "exact_ranking_score": "0.04101254412979794", - "exact_regret": 2.877280037168889e-05, - "exact_test_gap": "2.877280037168889e-05", - "exact_val_decision_score": "0.04257343918258997", - "exact_val_gap": "4.154025839683134e-05", - "exact_val_ranking_score": "0.04253189892419314", - "family": "FCN", - "model_tag": "fcn", - "ranking_selected_head": "pixel MLP head", - "ranking_selected_head_arch": "pixel_mlp", - "ranking_selected_test_pr_auc": "0.01989120112622579", - "ranking_selected_val_pr_auc": "0.024039539026323706", - "scope": "top20", - "seed": 123, - "tolerated_decision_head": "linear probe", - "tolerated_decision_head_arch": "linear", - "tolerated_decision_score": "0.7310509974227326", - "tolerated_ranking_score": "0.7185324707231184", - "tolerated_regret": 0.012518526699614174, - "tolerated_test_gap": "0.012518526699614174", - "tolerated_val_decision_score": "0.7058203125", - "tolerated_val_gap": "0.01514489915238637", - "tolerated_val_ranking_score": "0.6906754133476136", - "union_decision_head": "linear probe", - "union_decision_head_arch": "linear", - "union_decision_score": "0.7310509974227326", - "union_ranking_score": "0.7185324707231184", - "union_regret": 0.012518526699614174, - "union_test_gap": "0.012518526699614174", - "union_val_decision_score": "0.7058203125", - "union_val_gap": "0.01514489915238637", - "union_val_ranking_score": "0.6906754133476136", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0", - "exact_ranking_score": "0.0", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.0030852294639413806", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.0030852294639413806", - "family": "FCN", - "model_tag": "fcn", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.00036191656354256076", - "ranking_selected_val_pr_auc": "0.0004136690018862186", - "scope": "global", - "seed": 42, - "tolerated_decision_head": "pixel MLP head", - "tolerated_decision_head_arch": "pixel_mlp", - "tolerated_decision_score": "0.2870596305028149", - "tolerated_ranking_score": "0.31490583114252463", - "tolerated_regret": 0.0, - "tolerated_test_gap": "-0.027846200639709717", - "tolerated_val_decision_score": "0.31159367647460456", - "tolerated_val_gap": "0.009129115930882226", - "tolerated_val_ranking_score": "0.30246456054372234", - "union_decision_head": "pixel MLP head", - "union_decision_head_arch": "pixel_mlp", - "union_decision_score": "0.2870596305028149", - "union_ranking_score": "0.31490583114252463", - "union_regret": 0.0, - "union_test_gap": "-0.027846200639709717", - "union_val_decision_score": "0.31159367647460456", - "union_val_gap": "0.009129115930882226", - "union_val_ranking_score": "0.30246456054372234", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.11846022968949382", - "exact_ranking_score": "0.11754630125254256", - "exact_regret": 0.0009139284369512612, - "exact_test_gap": "0.0009139284369512612", - "exact_val_decision_score": "0.12063219008708741", - "exact_val_gap": "0.0033196374333384765", - "exact_val_ranking_score": "0.11731255265374893", - "family": "FCN", - "model_tag": "fcn", - "ranking_selected_head": "linear probe", - "ranking_selected_head_arch": "linear", - "ranking_selected_test_pr_auc": "0.058729135567964254", - "ranking_selected_val_pr_auc": "0.05944644145445249", - "scope": "top5", - "seed": 42, - "tolerated_decision_head": "linear probe", - "tolerated_decision_head_arch": "linear", - "tolerated_decision_score": "0.7714385719285964", - "tolerated_ranking_score": "0.7714385719285964", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.7976559342583861", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.7976559342583861", - "union_decision_head": "linear probe", - "union_decision_head_arch": "linear", - "union_decision_score": "0.7714385719285964", - "union_ranking_score": "0.7714385719285964", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.7976559342583861", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.7976559342583861", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.07337579617834394", - "exact_ranking_score": "0.07337579617834394", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.07542516569901865", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.07542516569901865", - "family": "FCN", - "model_tag": "fcn", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.031759062375776324", - "ranking_selected_val_pr_auc": "0.034979775498206175", - "scope": "top10", - "seed": 42, - "tolerated_decision_head": "pixel MLP head", - "tolerated_decision_head_arch": "pixel_mlp", - "tolerated_decision_score": "0.49088319481106213", - "tolerated_ranking_score": "0.6996715316108496", - "tolerated_regret": 0.0, - "tolerated_test_gap": "-0.20878833679978748", - "tolerated_val_decision_score": "0.7133624143149648", - "tolerated_val_gap": "0.011793150245700712", - "tolerated_val_ranking_score": "0.7015692640692641", - "union_decision_head": "pixel MLP head", - "union_decision_head_arch": "pixel_mlp", - "union_decision_score": "0.49088319481106213", - "union_ranking_score": "0.6996715316108496", - "union_regret": 0.0, - "union_test_gap": "-0.20878833679978748", - "union_val_decision_score": "0.7133624143149648", - "union_val_gap": "0.011793150245700712", - "union_val_ranking_score": "0.7015692640692641", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.04137542565993292", - "exact_ranking_score": "0.04137542565993292", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.043099210968826795", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.043099210968826795", - "family": "FCN", - "model_tag": "fcn", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.017928450700376908", - "ranking_selected_val_pr_auc": "0.01946275083543708", - "scope": "top20", - "seed": 42, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7189733167245789", - "tolerated_ranking_score": "0.7189733167245789", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.6912897822445562", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.6912897822445562", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7189733167245789", - "union_ranking_score": "0.7189733167245789", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.6912897822445562", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.6912897822445562", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.001086071137659517", - "exact_ranking_score": "0.001086071137659517", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.003349460364719018", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.003349460364719018", - "family": "FCN", - "model_tag": "fcn", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.0003268363416054406", - "ranking_selected_val_pr_auc": "0.00039156287297214436", - "scope": "global", - "seed": 7, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.3051941376005135", - "tolerated_ranking_score": "0.3051941376005135", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.2948179787588662", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.2948179787588662", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.3051941376005135", - "union_ranking_score": "0.3051941376005135", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.2948179787588662", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.2948179787588662", - "path": "" - }, - { - "exact_decision_head": "linear probe", - "exact_decision_head_arch": "linear", - "exact_decision_score": "0.13187490371283314", - "exact_ranking_score": "0.13187490371283314", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.12086776859504132", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.12086776859504132", - "family": "FCN", - "model_tag": "fcn", - "ranking_selected_head": "linear probe", - "ranking_selected_head_arch": "linear", - "ranking_selected_test_pr_auc": "0.07575936549127547", - "ranking_selected_val_pr_auc": "0.0704268488739946", - "scope": "top5", - "seed": 7, - "tolerated_decision_head": "linear probe", - "tolerated_decision_head_arch": "linear", - "tolerated_decision_score": "0.7714065672477771", - "tolerated_ranking_score": "0.7714065672477771", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.7976559342583861", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.7976559342583861", - "union_decision_head": "linear probe", - "union_decision_head_arch": "linear", - "union_decision_score": "0.7714065672477771", - "union_ranking_score": "0.7714065672477771", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.7976559342583861", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.7976559342583861", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0705085565701155", - "exact_ranking_score": "0.07325639739086803", - "exact_regret": 0.0, - "exact_test_gap": "-0.0027478408207525368", - "exact_val_decision_score": "0.07484534819623935", - "exact_val_gap": "0.00023644326242466818", - "exact_val_ranking_score": "0.07460890493381468", - "family": "FCN", - "model_tag": "fcn", - "ranking_selected_head": "pixel MLP head", - "ranking_selected_head_arch": "pixel_mlp", - "ranking_selected_test_pr_auc": "0.03431519870355593", - "ranking_selected_val_pr_auc": "0.04346900418958381", - "scope": "top10", - "seed": 7, - "tolerated_decision_head": "linear probe", - "tolerated_decision_head_arch": "linear", - "tolerated_decision_score": "0.6993745656706046", - "tolerated_ranking_score": "0.6996715316108496", - "tolerated_regret": 0.0, - "tolerated_test_gap": "-0.00029696594024497536", - "tolerated_val_decision_score": "0.7015653241482795", - "tolerated_val_gap": "4.7047341732997516e-05", - "tolerated_val_ranking_score": "0.7015182768065465", - "union_decision_head": "linear probe", - "union_decision_head_arch": "linear", - "union_decision_score": "0.6993745656706046", - "union_ranking_score": "0.6996715316108496", - "union_regret": 0.0, - "union_test_gap": "-0.00029696594024497536", - "union_val_decision_score": "0.7015653241482795", - "union_val_gap": "4.7047341732997516e-05", - "union_val_ranking_score": "0.7015182768065465", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.040288313241635454", - "exact_ranking_score": "0.04101254412979794", - "exact_regret": 0.0, - "exact_test_gap": "-0.0007242308881624873", - "exact_val_decision_score": "0.04306006752022422", - "exact_val_gap": "0.0005281685960310811", - "exact_val_ranking_score": "0.04253189892419314", - "family": "FCN", - "model_tag": "fcn", - "ranking_selected_head": "pixel MLP head", - "ranking_selected_head_arch": "pixel_mlp", - "ranking_selected_test_pr_auc": "0.02050876208409485", - "ranking_selected_val_pr_auc": "0.02590116599560572", - "scope": "top20", - "seed": 7, - "tolerated_decision_head": "linear probe", - "tolerated_decision_head_arch": "linear", - "tolerated_decision_score": "0.7644129739607127", - "tolerated_ranking_score": "0.7185324707231184", - "tolerated_regret": 0.045880503237594294, - "tolerated_test_gap": "0.045880503237594294", - "tolerated_val_decision_score": "0.6962583126352054", - "tolerated_val_gap": "0.005582899287591814", - "tolerated_val_ranking_score": "0.6906754133476136", - "union_decision_head": "linear probe", - "union_decision_head_arch": "linear", - "union_decision_score": "0.7644129739607127", - "union_ranking_score": "0.7185324707231184", - "union_regret": 0.045880503237594294, - "union_test_gap": "0.045880503237594294", - "union_val_decision_score": "0.6962583126352054", - "union_val_gap": "0.005582899287591814", - "union_val_ranking_score": "0.6906754133476136", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.001445086705202312", - "exact_ranking_score": "0.001445086705202312", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.0036289408029031526", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.0036289408029031526", - "family": "FCN", - "model_tag": "fcn", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.00037608715321531055", - "ranking_selected_val_pr_auc": "0.0004397940562535402", - "scope": "global", - "seed": 99, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.3060314081779048", - "tolerated_ranking_score": "0.3060314081779048", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.29429605373755735", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.29429605373755735", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.3060314081779048", - "union_ranking_score": "0.3060314081779048", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.29429605373755735", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.29429605373755735", - "path": "" - }, - { - "exact_decision_head": "linear probe", - "exact_decision_head_arch": "linear", - "exact_decision_score": "0.12161541991739329", - "exact_ranking_score": "0.12161541991739329", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.12163050624589085", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.12163050624589085", - "family": "FCN", - "model_tag": "fcn", - "ranking_selected_head": "linear probe", - "ranking_selected_head_arch": "linear", - "ranking_selected_test_pr_auc": "0.09974311720982598", - "ranking_selected_val_pr_auc": "0.10326265767079287", - "scope": "top5", - "seed": 99, - "tolerated_decision_head": "linear probe", - "tolerated_decision_head_arch": "linear", - "tolerated_decision_score": "0.7714065672477771", - "tolerated_ranking_score": "0.7714065672477771", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.7976559342583861", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.7976559342583861", - "union_decision_head": "linear probe", - "union_decision_head_arch": "linear", - "union_decision_score": "0.7714065672477771", - "union_ranking_score": "0.7714065672477771", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.7976559342583861", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.7976559342583861", - "path": "" - }, - { - "exact_decision_head": "linear probe", - "exact_decision_head_arch": "linear", - "exact_decision_score": "0.07668301382077575", - "exact_ranking_score": "0.07668301382077575", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.07647961072618607", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.07647961072618607", - "family": "FCN", - "model_tag": "fcn", - "ranking_selected_head": "linear probe", - "ranking_selected_head_arch": "linear", - "ranking_selected_test_pr_auc": "0.058967493704484314", - "ranking_selected_val_pr_auc": "0.06262653319620513", - "scope": "top10", - "seed": 99, - "tolerated_decision_head": "linear probe", - "tolerated_decision_head_arch": "linear", - "tolerated_decision_score": "0.7063465223762824", - "tolerated_ranking_score": "0.7063465223762824", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.7021793497677742", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.7021793497677742", - "union_decision_head": "linear probe", - "union_decision_head_arch": "linear", - "union_decision_score": "0.7063465223762824", - "union_ranking_score": "0.7063465223762824", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.7021793497677742", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.7021793497677742", - "path": "" - }, - { - "exact_decision_head": "linear probe", - "exact_decision_head_arch": "linear", - "exact_decision_score": "0.042822140550172624", - "exact_ranking_score": "0.042822140550172624", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.043420983295568316", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.043420983295568316", - "family": "FCN", - "model_tag": "fcn", - "ranking_selected_head": "linear probe", - "ranking_selected_head_arch": "linear", - "ranking_selected_test_pr_auc": "0.029098665712304895", - "ranking_selected_val_pr_auc": "0.032221638150467385", - "scope": "top20", - "seed": 99, - "tolerated_decision_head": "linear probe", - "tolerated_decision_head_arch": "linear", - "tolerated_decision_score": "0.726408418760773", - "tolerated_ranking_score": "0.726408418760773", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.6933831038489982", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.6933831038489982", - "union_decision_head": "linear probe", - "union_decision_head_arch": "linear", - "union_decision_score": "0.726408418760773", - "union_ranking_score": "0.726408418760773", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.6933831038489982", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.6933831038489982", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0", - "exact_ranking_score": "0.0", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.0031948881789137383", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.0031948881789137383", - "family": "FengWu", - "model_tag": "fengwu", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.000398077435365184", - "ranking_selected_val_pr_auc": "0.00047223676755013246", - "scope": "global", - "seed": 1, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.31071390711162444", - "tolerated_ranking_score": "0.31071390711162444", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.3130246199713141", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.3130246199713141", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.31071390711162444", - "union_ranking_score": "0.31071390711162444", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.3130246199713141", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.3130246199713141", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.12371632233901173", - "exact_ranking_score": "0.12371632233901173", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.12572845629262244", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.12572845629262244", - "family": "FengWu", - "model_tag": "fengwu", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.06541062880458605", - "ranking_selected_val_pr_auc": "0.06522920883515884", - "scope": "top5", - "seed": 1, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7834980065869301", - "tolerated_ranking_score": "0.7834980065869301", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.8136773785024538", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.8136773785024538", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7834980065869301", - "union_ranking_score": "0.7834980065869301", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.8136773785024538", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.8136773785024538", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.07892387357895363", - "exact_ranking_score": "0.07892387357895363", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.08108271403540572", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.08108271403540572", - "family": "FengWu", - "model_tag": "fengwu", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.04061009784825756", - "ranking_selected_val_pr_auc": "0.040143560685329066", - "scope": "top10", - "seed": 1, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7136104648359283", - "tolerated_ranking_score": "0.7136104648359283", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.7207024927160893", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.7207024927160893", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7136104648359283", - "union_ranking_score": "0.7136104648359283", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.7207024927160893", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.7207024927160893", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.04452825597664091", - "exact_ranking_score": "0.04452825597664091", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.0455414882958375", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.0455414882958375", - "family": "FengWu", - "model_tag": "fengwu", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.022736128443885992", - "ranking_selected_val_pr_auc": "0.021286665509494304", - "scope": "top20", - "seed": 1, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7269980510116651", - "tolerated_ranking_score": "0.7269980510116651", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.7088464801269441", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.7088464801269441", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7269980510116651", - "union_ranking_score": "0.7269980510116651", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.7088464801269441", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.7088464801269441", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0", - "exact_ranking_score": "0.0", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.003656307129798903", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.003656307129798903", - "family": "FengWu", - "model_tag": "fengwu", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.0004116035724473925", - "ranking_selected_val_pr_auc": "0.0004377390722357372", - "scope": "global", - "seed": 123, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.3145618361221859", - "tolerated_ranking_score": "0.3145618361221859", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.3129112248106385", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.3129112248106385", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.3145618361221859", - "union_ranking_score": "0.3145618361221859", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.3129112248106385", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.3129112248106385", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.11817781690140844", - "exact_ranking_score": "0.11833577252770228", - "exact_regret": 0.0, - "exact_test_gap": "-0.00015795562629383753", - "exact_val_decision_score": "0.12163664665332445", - "exact_val_gap": "0.002317871185766207", - "exact_val_ranking_score": "0.11931877546755824", - "family": "FengWu", - "model_tag": "fengwu", - "ranking_selected_head": "pixel MLP head", - "ranking_selected_head_arch": "pixel_mlp", - "ranking_selected_test_pr_auc": "0.06604572317905974", - "ranking_selected_val_pr_auc": "0.0732912945840262", - "scope": "top5", - "seed": 123, - "tolerated_decision_head": "linear probe", - "tolerated_decision_head_arch": "linear", - "tolerated_decision_score": "0.7862562878335749", - "tolerated_ranking_score": "0.7714149204832434", - "tolerated_regret": 0.01484136735033148, - "tolerated_test_gap": "0.01484136735033148", - "tolerated_val_decision_score": "0.8131524008350731", - "tolerated_val_gap": "0.01595894944850451", - "tolerated_val_ranking_score": "0.7971934513865686", - "union_decision_head": "linear probe", - "union_decision_head_arch": "linear", - "union_decision_score": "0.7862562878335749", - "union_ranking_score": "0.7714149204832434", - "union_regret": 0.01484136735033148, - "union_test_gap": "0.01484136735033148", - "union_val_decision_score": "0.8131524008350731", - "union_val_gap": "0.01595894944850451", - "union_val_ranking_score": "0.7971934513865686", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.07409381663113006", - "exact_ranking_score": "0.07325639739086803", - "exact_regret": 0.0008374192402620245, - "exact_test_gap": "0.0008374192402620245", - "exact_val_decision_score": "0.07709239276319321", - "exact_val_gap": "0.0024834878293785306", - "exact_val_ranking_score": "0.07460890493381468", - "family": "FengWu", - "model_tag": "fengwu", - "ranking_selected_head": "pixel MLP head", - "ranking_selected_head_arch": "pixel_mlp", - "ranking_selected_test_pr_auc": "0.038869102384950775", - "ranking_selected_val_pr_auc": "0.04346587500401188", - "scope": "top10", - "seed": 123, - "tolerated_decision_head": "linear probe", - "tolerated_decision_head_arch": "linear", - "tolerated_decision_score": "0.7099454687094261", - "tolerated_ranking_score": "0.6996715316108496", - "tolerated_regret": 0.010273937098576491, - "tolerated_test_gap": "0.010273937098576491", - "tolerated_val_decision_score": "0.7148757170172084", - "tolerated_val_gap": "0.013357440210661875", - "tolerated_val_ranking_score": "0.7015182768065465", - "union_decision_head": "linear probe", - "union_decision_head_arch": "linear", - "union_decision_score": "0.7099454687094261", - "union_ranking_score": "0.6996715316108496", - "union_regret": 0.010273937098576491, - "union_test_gap": "0.010273937098576491", - "union_val_decision_score": "0.7148757170172084", - "union_val_gap": "0.013357440210661875", - "union_val_ranking_score": "0.7015182768065465", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.04171607200603644", - "exact_ranking_score": "0.04101254412979794", - "exact_regret": 0.0007035278762385008, - "exact_test_gap": "0.0007035278762385008", - "exact_val_decision_score": "0.043619564341543335", - "exact_val_gap": "0.0010876654173501976", - "exact_val_ranking_score": "0.04253189892419314", - "family": "FengWu", - "model_tag": "fengwu", - "ranking_selected_head": "pixel MLP head", - "ranking_selected_head_arch": "pixel_mlp", - "ranking_selected_test_pr_auc": "0.020495144103834257", - "ranking_selected_val_pr_auc": "0.023246350918909517", - "scope": "top20", - "seed": 123, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7251252524331628", - "tolerated_ranking_score": "0.7185324707231184", - "tolerated_regret": 0.006592781710044404, - "tolerated_test_gap": "0.006592781710044404", - "tolerated_val_decision_score": "0.7015002586652871", - "tolerated_val_gap": "0.01082484531767347", - "tolerated_val_ranking_score": "0.6906754133476136", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7251252524331628", - "union_ranking_score": "0.7185324707231184", - "union_regret": 0.006592781710044404, - "union_test_gap": "0.006592781710044404", - "union_val_decision_score": "0.7015002586652871", - "union_val_gap": "0.01082484531767347", - "union_val_ranking_score": "0.6906754133476136", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0", - "exact_ranking_score": "0.0", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.001528350909368791", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.001528350909368791", - "family": "FengWu", - "model_tag": "fengwu", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.00036987379624400454", - "ranking_selected_val_pr_auc": "0.00039852867365630293", - "scope": "global", - "seed": 42, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.31219058559732665", - "tolerated_ranking_score": "0.31219058559732665", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.31476872634310843", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.31476872634310843", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.31219058559732665", - "union_ranking_score": "0.31219058559732665", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.31476872634310843", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.31476872634310843", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.12233412322274881", - "exact_ranking_score": "0.12047131373196783", - "exact_regret": 0.0018628094907809783, - "exact_test_gap": "0.0018628094907809783", - "exact_val_decision_score": "0.12295192958700069", - "exact_val_gap": "0.004349514799095983", - "exact_val_ranking_score": "0.1186024147879047", - "family": "FengWu", - "model_tag": "fengwu", - "ranking_selected_head": "linear probe", - "ranking_selected_head_arch": "linear", - "ranking_selected_test_pr_auc": "0.08182944446297657", - "ranking_selected_val_pr_auc": "0.06677724388642242", - "scope": "top5", - "seed": 42, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7830574200952944", - "tolerated_ranking_score": "0.77137519758586", - "tolerated_regret": 0.011682222509434403, - "tolerated_test_gap": "0.011682222509434403", - "tolerated_val_decision_score": "0.8181423987511924", - "tolerated_val_gap": "0.020669744358910225", - "tolerated_val_ranking_score": "0.7974726543922822", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7830574200952944", - "union_ranking_score": "0.77137519758586", - "union_regret": 0.011682222509434403, - "union_test_gap": "0.011682222509434403", - "union_val_decision_score": "0.8181423987511924", - "union_val_gap": "0.020669744358910225", - "union_val_ranking_score": "0.7974726543922822", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0804971885173128", - "exact_ranking_score": "0.0804971885173128", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.08148543948704247", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.08148543948704247", - "family": "FengWu", - "model_tag": "fengwu", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.03819635082226218", - "ranking_selected_val_pr_auc": "0.038941681913664185", - "scope": "top10", - "seed": 42, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7115204223666349", - "tolerated_ranking_score": "0.7115204223666349", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.7256034753011779", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.7256034753011779", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7115204223666349", - "union_ranking_score": "0.7115204223666349", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.7256034753011779", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.7256034753011779", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.04574838388861263", - "exact_ranking_score": "0.04574838388861263", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.04613821138211382", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.04613821138211382", - "family": "FengWu", - "model_tag": "fengwu", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.021165185967854567", - "ranking_selected_val_pr_auc": "0.020671268187642432", - "scope": "top20", - "seed": 42, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7265705731122933", - "tolerated_ranking_score": "0.7265705731122933", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.7074814597750377", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.7074814597750377", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7265705731122933", - "union_ranking_score": "0.7265705731122933", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.7074814597750377", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.7074814597750377", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0", - "exact_ranking_score": "0.0", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.0027012425715829285", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.0027012425715829285", - "family": "FengWu", - "model_tag": "fengwu", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.00036963872610239895", - "ranking_selected_val_pr_auc": "0.00042054736055892957", - "scope": "global", - "seed": 7, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.30641904273669174", - "tolerated_ranking_score": "0.30641904273669174", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.30868675618673724", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.30868675618673724", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.30641904273669174", - "union_ranking_score": "0.30641904273669174", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.30868675618673724", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.30868675618673724", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.11923365086160762", - "exact_ranking_score": "0.120447564721369", - "exact_regret": 0.0, - "exact_test_gap": "-0.001213913859761384", - "exact_val_decision_score": "0.12151351351351353", - "exact_val_gap": "0.003267042862839506", - "exact_val_ranking_score": "0.11824647065067402", - "family": "FengWu", - "model_tag": "fengwu", - "ranking_selected_head": "linear probe", - "ranking_selected_head_arch": "linear", - "ranking_selected_test_pr_auc": "0.055750221614838426", - "ranking_selected_val_pr_auc": "0.06442862811569598", - "scope": "top5", - "seed": 7, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.779811109738517", - "tolerated_ranking_score": "0.77137519758586", - "tolerated_regret": 0.008435912152656933, - "tolerated_test_gap": "0.008435912152656933", - "tolerated_val_decision_score": "0.8111111111111111", - "tolerated_val_gap": "0.013638456718828906", - "tolerated_val_ranking_score": "0.7974726543922822", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.779811109738517", - "union_ranking_score": "0.77137519758586", - "union_regret": 0.008435912152656933, - "union_test_gap": "0.008435912152656933", - "union_val_decision_score": "0.8111111111111111", - "union_val_gap": "0.013638456718828906", - "union_val_ranking_score": "0.7974726543922822", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.07741464305276612", - "exact_ranking_score": "0.07741464305276612", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.07768666378938284", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.07768666378938284", - "family": "FengWu", - "model_tag": "fengwu", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.0356468915576778", - "ranking_selected_val_pr_auc": "0.03730439444340619", - "scope": "top10", - "seed": 7, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7094619133450868", - "tolerated_ranking_score": "0.7094619133450868", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.7188508982531667", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.7188508982531667", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7094619133450868", - "union_ranking_score": "0.7094619133450868", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.7188508982531667", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.7188508982531667", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.043763127370763974", - "exact_ranking_score": "0.04101254412979794", - "exact_regret": 0.0027505832409660327, - "exact_test_gap": "0.0027505832409660327", - "exact_val_decision_score": "0.04404735872043984", - "exact_val_gap": "0.0015154597962467042", - "exact_val_ranking_score": "0.04253189892419314", - "family": "FengWu", - "model_tag": "fengwu", - "ranking_selected_head": "pixel MLP head", - "ranking_selected_head_arch": "pixel_mlp", - "ranking_selected_test_pr_auc": "0.016801706970978273", - "ranking_selected_val_pr_auc": "0.022509307119807316", - "scope": "top20", - "seed": 7, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.722901134194411", - "tolerated_ranking_score": "0.7185324707231184", - "tolerated_regret": 0.004368663471292611, - "tolerated_test_gap": "0.004368663471292611", - "tolerated_val_decision_score": "0.7033268834654516", - "tolerated_val_gap": "0.01265147011783796", - "tolerated_val_ranking_score": "0.6906754133476136", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.722901134194411", - "union_ranking_score": "0.7185324707231184", - "union_regret": 0.004368663471292611, - "union_test_gap": "0.004368663471292611", - "union_val_decision_score": "0.7033268834654516", - "union_val_gap": "0.01265147011783796", - "union_val_ranking_score": "0.6906754133476136", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0", - "exact_ranking_score": "0.0", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.0041356492969396195", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.0041356492969396195", - "family": "FengWu", - "model_tag": "fengwu", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.00042782651526874734", - "ranking_selected_val_pr_auc": "0.0004271990954031648", - "scope": "global", - "seed": 99, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.3111214819309595", - "tolerated_ranking_score": "0.3111214819309595", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.30674980181224504", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.30674980181224504", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.3111214819309595", - "union_ranking_score": "0.3111214819309595", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.30674980181224504", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.30674980181224504", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.12190521103513322", - "exact_ranking_score": "0.12125309894072571", - "exact_regret": 0.000652112094407506, - "exact_test_gap": "0.000652112094407506", - "exact_val_decision_score": "0.12268433321064899", - "exact_val_gap": "0.0024291153021381295", - "exact_val_ranking_score": "0.12025521790851086", - "family": "FengWu", - "model_tag": "fengwu", - "ranking_selected_head": "linear probe", - "ranking_selected_head_arch": "linear", - "ranking_selected_test_pr_auc": "0.06535531985010146", - "ranking_selected_val_pr_auc": "0.07922769803833127", - "scope": "top5", - "seed": 99, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7803954553118283", - "tolerated_ranking_score": "0.77137519758586", - "tolerated_regret": 0.009020257725968261, - "tolerated_test_gap": "0.009020257725968261", - "tolerated_val_decision_score": "0.804416403785489", - "tolerated_val_gap": "0.006943749393206744", - "tolerated_val_ranking_score": "0.7974726543922822", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7803954553118283", - "union_ranking_score": "0.77137519758586", - "union_regret": 0.009020257725968261, - "union_test_gap": "0.009020257725968261", - "union_val_decision_score": "0.804416403785489", - "union_val_gap": "0.006943749393206744", - "union_val_ranking_score": "0.7974726543922822", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.07714516687839242", - "exact_ranking_score": "0.07550644567219153", - "exact_regret": 0.0016387212062008855, - "exact_test_gap": "0.0016387212062008855", - "exact_val_decision_score": "0.0780302129178066", - "exact_val_gap": "0.0027111964223195256", - "exact_val_ranking_score": "0.07531901649548707", - "family": "FengWu", - "model_tag": "fengwu", - "ranking_selected_head": "linear probe", - "ranking_selected_head_arch": "linear", - "ranking_selected_test_pr_auc": "0.04049928566381453", - "ranking_selected_val_pr_auc": "0.051914070260881295", - "scope": "top10", - "seed": 99, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7095126183114994", - "tolerated_ranking_score": "0.6996715316108496", - "tolerated_regret": 0.00984108670064976, - "tolerated_test_gap": "0.00984108670064976", - "tolerated_val_decision_score": "0.7096515175847117", - "tolerated_val_gap": "0.00813324077816513", - "tolerated_val_ranking_score": "0.7015182768065465", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7095126183114994", - "union_ranking_score": "0.6996715316108496", - "union_regret": 0.00984108670064976, - "union_test_gap": "0.00984108670064976", - "union_val_decision_score": "0.7096515175847117", - "union_val_gap": "0.00813324077816513", - "union_val_ranking_score": "0.7015182768065465", - "path": "" - }, - { - "exact_decision_head": "linear probe", - "exact_decision_head_arch": "linear", - "exact_decision_score": "0.0", - "exact_ranking_score": "0.0", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.04882017900732303", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.04882017900732303", - "family": "FengWu", - "model_tag": "fengwu", - "ranking_selected_head": "linear probe", - "ranking_selected_head_arch": "linear", - "ranking_selected_test_pr_auc": "0.02199779031689959", - "ranking_selected_val_pr_auc": "0.030768274290330223", - "scope": "top20", - "seed": 99, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7336829717908426", - "tolerated_ranking_score": "0.7185324707231184", - "tolerated_regret": 0.015150501067724198, - "tolerated_test_gap": "0.015150501067724198", - "tolerated_val_decision_score": "0.6987796297897932", - "tolerated_val_gap": "0.008104216442179624", - "tolerated_val_ranking_score": "0.6906754133476136", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7336829717908426", - "union_ranking_score": "0.7185324707231184", - "union_regret": 0.015150501067724198, - "union_test_gap": "0.015150501067724198", - "union_val_decision_score": "0.6987796297897932", - "union_val_gap": "0.008104216442179624", - "union_val_ranking_score": "0.6906754133476136", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0006866311182961118", - "exact_ranking_score": "0.0006866311182961118", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.0008811590083408097", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.0008811590083408097", - "family": "FuXi", - "model_tag": "fuxi", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.0003596048414560045", - "ranking_selected_val_pr_auc": "0.0004121169333774252", - "scope": "global", - "seed": 1, - "tolerated_decision_head": "linear probe", - "tolerated_decision_head_arch": "linear", - "tolerated_decision_score": "0.21516044416153002", - "tolerated_ranking_score": "0.3091927111996169", - "tolerated_regret": 0.0, - "tolerated_test_gap": "-0.09403226703808687", - "tolerated_val_decision_score": "0.3118144595732139", - "tolerated_val_gap": "0.001671993688703799", - "tolerated_val_ranking_score": "0.3101424658845101", - "union_decision_head": "linear probe", - "union_decision_head_arch": "linear", - "union_decision_score": "0.21516044416153002", - "union_ranking_score": "0.3091927111996169", - "union_regret": 0.0, - "union_test_gap": "-0.09403226703808687", - "union_val_decision_score": "0.3118144595732139", - "union_val_gap": "0.001671993688703799", - "union_val_ranking_score": "0.3101424658845101", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.12601927353595255", - "exact_ranking_score": "0.1189950593924104", - "exact_regret": 0.007024214143542151, - "exact_test_gap": "0.007024214143542151", - "exact_val_decision_score": "0.12394433210419888", - "exact_val_gap": "0.004525739612303029", - "exact_val_ranking_score": "0.11941859249189585", - "family": "FuXi", - "model_tag": "fuxi", - "ranking_selected_head": "pixel MLP head", - "ranking_selected_head_arch": "pixel_mlp", - "ranking_selected_test_pr_auc": "0.06032490713940969", - "ranking_selected_val_pr_auc": "0.0614132176206002", - "scope": "top5", - "seed": 1, - "tolerated_decision_head": "linear probe", - "tolerated_decision_head_arch": "linear", - "tolerated_decision_score": "0.8241199478487613", - "tolerated_ranking_score": "0.7739636861939019", - "tolerated_regret": 0.050156261654859424, - "tolerated_test_gap": "0.050156261654859424", - "tolerated_val_decision_score": "0.8658519277773397", - "tolerated_val_gap": "0.06831247872679147", - "tolerated_val_ranking_score": "0.7975394490505482", - "union_decision_head": "linear probe", - "union_decision_head_arch": "linear", - "union_decision_score": "0.8241199478487613", - "union_ranking_score": "0.7739636861939019", - "union_regret": 0.050156261654859424, - "union_test_gap": "0.050156261654859424", - "union_val_decision_score": "0.8658519277773397", - "union_val_gap": "0.06831247872679147", - "union_val_ranking_score": "0.7975394490505482", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.07447712973984016", - "exact_ranking_score": "0.07447712973984016", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.08072936959401543", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.08072936959401543", - "family": "FuXi", - "model_tag": "fuxi", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.03373982859419053", - "ranking_selected_val_pr_auc": "0.037115310159824", - "scope": "top10", - "seed": 1, - "tolerated_decision_head": "linear probe", - "tolerated_decision_head_arch": "linear", - "tolerated_decision_score": "0.6935334872979214", - "tolerated_ranking_score": "0.7046101309049516", - "tolerated_regret": 0.0, - "tolerated_test_gap": "-0.011076643607030201", - "tolerated_val_decision_score": "0.754546876833418", - "tolerated_val_gap": "0.04256271595760763", - "tolerated_val_ranking_score": "0.7119841608758104", - "union_decision_head": "linear probe", - "union_decision_head_arch": "linear", - "union_decision_score": "0.6935334872979214", - "union_ranking_score": "0.7046101309049516", - "union_regret": 0.0, - "union_test_gap": "-0.011076643607030201", - "union_val_decision_score": "0.754546876833418", - "union_val_gap": "0.04256271595760763", - "union_val_ranking_score": "0.7119841608758104", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.043196160341303", - "exact_ranking_score": "0.043196160341303", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.04576053854784421", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.04576053854784421", - "family": "FuXi", - "model_tag": "fuxi", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.01904942261850253", - "ranking_selected_val_pr_auc": "0.020787588115504", - "scope": "top20", - "seed": 1, - "tolerated_decision_head": "linear probe", - "tolerated_decision_head_arch": "linear", - "tolerated_decision_score": "0.7261195534617713", - "tolerated_ranking_score": "0.7246636456247585", - "tolerated_regret": 0.001455907837012771, - "tolerated_test_gap": "0.001455907837012771", - "tolerated_val_decision_score": "0.7033365957825429", - "tolerated_val_gap": "0.003706992036648815", - "tolerated_val_ranking_score": "0.6996296037458941", - "union_decision_head": "linear probe", - "union_decision_head_arch": "linear", - "union_decision_score": "0.7261195534617713", - "union_ranking_score": "0.7246636456247585", - "union_regret": 0.001455907837012771, - "union_test_gap": "0.001455907837012771", - "union_val_decision_score": "0.7033365957825429", - "union_val_gap": "0.003706992036648815", - "union_val_ranking_score": "0.6996296037458941", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0009888839378897987", - "exact_ranking_score": "0.0009888839378897987", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.0010117798317730948", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.0010117798317730948", - "family": "FuXi", - "model_tag": "fuxi", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.00039999784430104664", - "ranking_selected_val_pr_auc": "0.00043073606570656576", - "scope": "global", - "seed": 123, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.3096924811079745", - "tolerated_ranking_score": "0.3096924811079745", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.31039074285584656", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.31039074285584656", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.3096924811079745", - "union_ranking_score": "0.3096924811079745", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.31039074285584656", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.31039074285584656", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.12121896162528217", - "exact_ranking_score": "0.11833577252770228", - "exact_regret": 0.002883189097579897, - "exact_test_gap": "0.002883189097579897", - "exact_val_decision_score": "0.1213910761154856", - "exact_val_gap": "0.00207230064792735", - "exact_val_ranking_score": "0.11931877546755824", - "family": "FuXi", - "model_tag": "fuxi", - "ranking_selected_head": "pixel MLP head", - "ranking_selected_head_arch": "pixel_mlp", - "ranking_selected_test_pr_auc": "0.06578907632994217", - "ranking_selected_val_pr_auc": "0.0774014225555978", - "scope": "top5", - "seed": 123, - "tolerated_decision_head": "linear probe", - "tolerated_decision_head_arch": "linear", - "tolerated_decision_score": "0.7834545900433254", - "tolerated_ranking_score": "0.7714149204832434", - "tolerated_regret": 0.012039669560081956, - "tolerated_test_gap": "0.012039669560081956", - "tolerated_val_decision_score": "0.8107372400756144", - "tolerated_val_gap": "0.013543788689045821", - "tolerated_val_ranking_score": "0.7971934513865686", - "union_decision_head": "linear probe", - "union_decision_head_arch": "linear", - "union_decision_score": "0.7834545900433254", - "union_ranking_score": "0.7714149204832434", - "union_regret": 0.012039669560081956, - "union_test_gap": "0.012039669560081956", - "union_val_decision_score": "0.8107372400756144", - "union_val_gap": "0.013543788689045821", - "union_val_ranking_score": "0.7971934513865686", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.07623735302160241", - "exact_ranking_score": "0.07325639739086803", - "exact_regret": 0.002980955630734375, - "exact_test_gap": "0.002980955630734375", - "exact_val_decision_score": "0.07630118890356671", - "exact_val_gap": "0.0016922839697520314", - "exact_val_ranking_score": "0.07460890493381468", - "family": "FuXi", - "model_tag": "fuxi", - "ranking_selected_head": "pixel MLP head", - "ranking_selected_head_arch": "pixel_mlp", - "ranking_selected_test_pr_auc": "0.03878602989774424", - "ranking_selected_val_pr_auc": "0.04567884589440755", - "scope": "top10", - "seed": 123, - "tolerated_decision_head": "linear probe", - "tolerated_decision_head_arch": "linear", - "tolerated_decision_score": "0.7079514490958633", - "tolerated_ranking_score": "0.6996715316108496", - "tolerated_regret": 0.008279917485013666, - "tolerated_test_gap": "0.008279917485013666", - "tolerated_val_decision_score": "0.7130764176104518", - "tolerated_val_gap": "0.01155814080390527", - "tolerated_val_ranking_score": "0.7015182768065465", - "union_decision_head": "linear probe", - "union_decision_head_arch": "linear", - "union_decision_score": "0.7079514490958633", - "union_ranking_score": "0.6996715316108496", - "union_regret": 0.008279917485013666, - "union_test_gap": "0.008279917485013666", - "union_val_decision_score": "0.7130764176104518", - "union_val_gap": "0.01155814080390527", - "union_val_ranking_score": "0.7015182768065465", - "path": "" - }, - { - "exact_decision_head": "pixel MLP head", - "exact_decision_head_arch": "pixel_mlp", - "exact_decision_score": "0.018543768748295608", - "exact_ranking_score": "0.018543768748295608", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.04600811907983762", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.04600811907983762", - "family": "FuXi", - "model_tag": "fuxi", - "ranking_selected_head": "pixel MLP head", - "ranking_selected_head_arch": "pixel_mlp", - "ranking_selected_test_pr_auc": "0.021045021724179047", - "ranking_selected_val_pr_auc": "0.025380726482200633", - "scope": "top20", - "seed": 123, - "tolerated_decision_head": "linear probe", - "tolerated_decision_head_arch": "linear", - "tolerated_decision_score": "0.7224961571477053", - "tolerated_ranking_score": "0.7185324707231184", - "tolerated_regret": 0.0039636864245868875, - "tolerated_test_gap": "0.0039636864245868875", - "tolerated_val_decision_score": "0.6975587357465372", - "tolerated_val_gap": "0.00688332239892353", - "tolerated_val_ranking_score": "0.6906754133476136", - "union_decision_head": "linear probe", - "union_decision_head_arch": "linear", - "union_decision_score": "0.7224961571477053", - "union_ranking_score": "0.7185324707231184", - "union_regret": 0.0039636864245868875, - "union_test_gap": "0.0039636864245868875", - "union_val_decision_score": "0.6975587357465372", - "union_val_gap": "0.00688332239892353", - "union_val_ranking_score": "0.6906754133476136", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0003188165529554294", - "exact_ranking_score": "0.0003188165529554294", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.0011176098051633573", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.0011176098051633573", - "family": "FuXi", - "model_tag": "fuxi", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.0003830459807690152", - "ranking_selected_val_pr_auc": "0.00042901589997107405", - "scope": "global", - "seed": 42, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.316668570748256", - "tolerated_ranking_score": "0.316668570748256", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.3180456774666437", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.3180456774666437", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.316668570748256", - "union_ranking_score": "0.316668570748256", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.3180456774666437", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.3180456774666437", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.12472723096359249", - "exact_ranking_score": "0.12343429742655432", - "exact_regret": 0.001292933537038174, - "exact_test_gap": "0.001292933537038174", - "exact_val_decision_score": "0.12229812968977491", - "exact_val_gap": "0.0007780236358641696", - "exact_val_ranking_score": "0.12152010605391074", - "family": "FuXi", - "model_tag": "fuxi", - "ranking_selected_head": "linear probe", - "ranking_selected_head_arch": "linear", - "ranking_selected_test_pr_auc": "0.07205944796033896", - "ranking_selected_val_pr_auc": "0.06887409447218568", - "scope": "top5", - "seed": 42, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7659067086974988", - "tolerated_ranking_score": "0.7763794772507261", - "tolerated_regret": 0.0, - "tolerated_test_gap": "-0.010472768553227318", - "tolerated_val_decision_score": "0.810906174819567", - "tolerated_val_gap": "0.009041372954765037", - "tolerated_val_ranking_score": "0.8018648018648019", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7659067086974988", - "union_ranking_score": "0.7763794772507261", - "union_regret": 0.0, - "union_test_gap": "-0.010472768553227318", - "union_val_decision_score": "0.810906174819567", - "union_val_gap": "0.009041372954765037", - "union_val_ranking_score": "0.8018648018648019", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.08151658767772511", - "exact_ranking_score": "0.07757857459052678", - "exact_regret": 0.003938013087198336, - "exact_test_gap": "0.003938013087198336", - "exact_val_decision_score": "0.07882782007806002", - "exact_val_gap": "0.003017081109426334", - "exact_val_ranking_score": "0.07581073896863369", - "family": "FuXi", - "model_tag": "fuxi", - "ranking_selected_head": "linear probe", - "ranking_selected_head_arch": "linear", - "ranking_selected_test_pr_auc": "0.0420642409494585", - "ranking_selected_val_pr_auc": "0.037860183103839665", - "scope": "top10", - "seed": 42, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.6658250522289358", - "tolerated_ranking_score": "0.707219799470722", - "tolerated_regret": 0.0, - "tolerated_test_gap": "-0.041394747241786156", - "tolerated_val_decision_score": "0.7266319786024423", - "tolerated_val_gap": "0.020155767669137514", - "tolerated_val_ranking_score": "0.7064762109333048", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.6658250522289358", - "union_ranking_score": "0.707219799470722", - "union_regret": 0.0, - "union_test_gap": "-0.041394747241786156", - "union_val_decision_score": "0.7266319786024423", - "union_val_gap": "0.020155767669137514", - "union_val_ranking_score": "0.7064762109333048", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.04603830266616599", - "exact_ranking_score": "0.04603830266616599", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.04428715218026559", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.04428715218026559", - "family": "FuXi", - "model_tag": "fuxi", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.020363596550511454", - "ranking_selected_val_pr_auc": "0.020674507603092498", - "scope": "top20", - "seed": 42, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7054843984273774", - "tolerated_ranking_score": "0.7054843984273774", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.72648949860566", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.72648949860566", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7054843984273774", - "union_ranking_score": "0.7054843984273774", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.72648949860566", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.72648949860566", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0008450613619815959", - "exact_ranking_score": "0.0008450613619815959", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.0009152735719220856", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.0009152735719220856", - "family": "FuXi", - "model_tag": "fuxi", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.00037118462783325537", - "ranking_selected_val_pr_auc": "0.0004157684867215522", - "scope": "global", - "seed": 7, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.3062654921252336", - "tolerated_ranking_score": "0.3062654921252336", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.30677382687191324", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.30677382687191324", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.3062654921252336", - "union_ranking_score": "0.3062654921252336", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.30677382687191324", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.30677382687191324", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.12366683595733877", - "exact_ranking_score": "0.12257405515832481", - "exact_regret": 0.0010927807990139538, - "exact_test_gap": "0.0010927807990139538", - "exact_val_decision_score": "0.1239751552795031", - "exact_val_gap": "0.004866804386744986", - "exact_val_ranking_score": "0.11910835089275812", - "family": "FuXi", - "model_tag": "fuxi", - "ranking_selected_head": "linear probe", - "ranking_selected_head_arch": "linear", - "ranking_selected_test_pr_auc": "0.06705127569812482", - "ranking_selected_val_pr_auc": "0.0609692021164841", - "scope": "top5", - "seed": 7, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7746207358980806", - "tolerated_ranking_score": "0.77137519758586", - "tolerated_regret": 0.0032455383122205195, - "tolerated_test_gap": "0.0032455383122205195", - "tolerated_val_decision_score": "0.8054623862002874", - "tolerated_val_gap": "0.007989731808005218", - "tolerated_val_ranking_score": "0.7974726543922822", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7746207358980806", - "union_ranking_score": "0.77137519758586", - "union_regret": 0.0032455383122205195, - "union_test_gap": "0.0032455383122205195", - "union_val_decision_score": "0.8054623862002874", - "union_val_gap": "0.007989731808005218", - "union_val_ranking_score": "0.7974726543922822", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.07886239443616493", - "exact_ranking_score": "0.07886239443616493", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.07953855494839102", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.07953855494839102", - "family": "FuXi", - "model_tag": "fuxi", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.03317855540638228", - "ranking_selected_val_pr_auc": "0.0369116762896455", - "scope": "top10", - "seed": 7, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7025612895058863", - "tolerated_ranking_score": "0.7025612895058863", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.710349352140397", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.710349352140397", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7025612895058863", - "union_ranking_score": "0.7025612895058863", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.710349352140397", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.710349352140397", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.044572342126298965", - "exact_ranking_score": "0.044572342126298965", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.04500911929271383", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.04500911929271383", - "family": "FuXi", - "model_tag": "fuxi", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.018717347905238046", - "ranking_selected_val_pr_auc": "0.020626878606552698", - "scope": "top20", - "seed": 7, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7235687421646468", - "tolerated_ranking_score": "0.7235687421646468", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.6980281690140845", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.6980281690140845", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7235687421646468", - "union_ranking_score": "0.7235687421646468", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.6980281690140845", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.6980281690140845", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0008606592649969877", - "exact_ranking_score": "0.0008606592649969877", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.000883817480228723", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.000883817480228723", - "family": "FuXi", - "model_tag": "fuxi", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.0003867174887645146", - "ranking_selected_val_pr_auc": "0.00040688996624133387", - "scope": "global", - "seed": 99, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.3109717285865725", - "tolerated_ranking_score": "0.3109717285865725", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.31099917685286793", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.31099917685286793", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.3109717285865725", - "union_ranking_score": "0.3109717285865725", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.31099917685286793", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.31099917685286793", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.12165729369907036", - "exact_ranking_score": "0.11908268462023987", - "exact_regret": 0.0025746090788304893, - "exact_test_gap": "0.0025746090788304893", - "exact_val_decision_score": "0.12040952592922323", - "exact_val_gap": "0.0009159610029911419", - "exact_val_ranking_score": "0.11949356492623209", - "family": "FuXi", - "model_tag": "fuxi", - "ranking_selected_head": "pixel MLP head", - "ranking_selected_head_arch": "pixel_mlp", - "ranking_selected_test_pr_auc": "0.0679936659760212", - "ranking_selected_val_pr_auc": "0.08132317939286735", - "scope": "top5", - "seed": 99, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7764511526620109", - "tolerated_ranking_score": "0.7741670094611272", - "tolerated_regret": 0.0022841432008837703, - "tolerated_test_gap": "0.0022841432008837703", - "tolerated_val_decision_score": "0.8023909485519105", - "tolerated_val_gap": "0.004531416779335218", - "tolerated_val_ranking_score": "0.7978595317725753", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7764511526620109", - "union_ranking_score": "0.7741670094611272", - "union_regret": 0.0022841432008837703, - "union_test_gap": "0.0022841432008837703", - "union_val_decision_score": "0.8023909485519105", - "union_val_gap": "0.004531416779335218", - "union_val_ranking_score": "0.7978595317725753", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.06876734046769718", - "exact_ranking_score": "0.07384553133377168", - "exact_regret": 0.0, - "exact_test_gap": "-0.0050781908660745", - "exact_val_decision_score": "0.07681263460157932", - "exact_val_gap": "0.0019560091865476192", - "exact_val_ranking_score": "0.0748566254150317", - "family": "FuXi", - "model_tag": "fuxi", - "ranking_selected_head": "pixel MLP head", - "ranking_selected_head_arch": "pixel_mlp", - "ranking_selected_test_pr_auc": "0.03972371884586323", - "ranking_selected_val_pr_auc": "0.04922131117303804", - "scope": "top10", - "seed": 99, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7026029306512976", - "tolerated_ranking_score": "0.7027966072827605", - "tolerated_regret": 0.0, - "tolerated_test_gap": "-0.00019367663146285796", - "tolerated_val_decision_score": "0.7072442664725155", - "tolerated_val_gap": "0.0042447748609242275", - "tolerated_val_ranking_score": "0.7029994916115913", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7026029306512976", - "union_ranking_score": "0.7027966072827605", - "union_regret": 0.0, - "union_test_gap": "-0.00019367663146285796", - "union_val_decision_score": "0.7072442664725155", - "union_val_gap": "0.0042447748609242275", - "union_val_ranking_score": "0.7029994916115913", - "path": "" - }, - { - "exact_decision_head": "pixel MLP head", - "exact_decision_head_arch": "pixel_mlp", - "exact_decision_score": "0.027261462205700127", - "exact_ranking_score": "0.04233736419209311", - "exact_regret": 0.0, - "exact_test_gap": "-0.01507590198639298", - "exact_val_decision_score": "0.04932875429285045", - "exact_val_gap": "0.006512143511318333", - "exact_val_ranking_score": "0.042816610781532115", - "family": "FuXi", - "model_tag": "fuxi", - "ranking_selected_head": "linear probe", - "ranking_selected_head_arch": "linear", - "ranking_selected_test_pr_auc": "0.026062499558490423", - "ranking_selected_val_pr_auc": "0.02771096205491839", - "scope": "top20", - "seed": 99, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7272787942491131", - "tolerated_ranking_score": "0.7185324707231184", - "tolerated_regret": 0.008746323525994693, - "tolerated_test_gap": "0.008746323525994693", - "tolerated_val_decision_score": "0.6966977214053203", - "tolerated_val_gap": "0.006022308057706649", - "tolerated_val_ranking_score": "0.6906754133476136", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7272787942491131", - "union_ranking_score": "0.7185324707231184", - "union_regret": 0.008746323525994693, - "union_test_gap": "0.008746323525994693", - "union_val_decision_score": "0.6966977214053203", - "union_val_gap": "0.006022308057706649", - "union_val_ranking_score": "0.6906754133476136", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0", - "exact_ranking_score": "0.0", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.004253509145044662", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.004253509145044662", - "family": "Pangu-Weather", - "model_tag": "pangu6", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.00042881385578730365", - "ranking_selected_val_pr_auc": "0.0004901027948814223", - "scope": "global", - "seed": 1, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.32066974948800614", - "tolerated_ranking_score": "0.32066974948800614", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.3120952399779768", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.3120952399779768", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.32066974948800614", - "union_ranking_score": "0.32066974948800614", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.3120952399779768", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.3120952399779768", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.12675350701402807", - "exact_ranking_score": "0.12675350701402807", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.1289902280130293", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.1289902280130293", - "family": "Pangu-Weather", - "model_tag": "pangu6", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.07239587203738174", - "ranking_selected_val_pr_auc": "0.06555234277501942", - "scope": "top5", - "seed": 1, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.776857122271057", - "tolerated_ranking_score": "0.776857122271057", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.8022016303211871", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.8022016303211871", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.776857122271057", - "union_ranking_score": "0.776857122271057", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.8022016303211871", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.8022016303211871", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.07904849039341263", - "exact_ranking_score": "0.07904849039341263", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.08153057396523697", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.08153057396523697", - "family": "Pangu-Weather", - "model_tag": "pangu6", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.04315307069319409", - "ranking_selected_val_pr_auc": "0.03843818318005278", - "scope": "top10", - "seed": 1, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7050669731030272", - "tolerated_ranking_score": "0.7050669731030272", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.7088338471351122", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.7088338471351122", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7050669731030272", - "union_ranking_score": "0.7050669731030272", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.7088338471351122", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.7088338471351122", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.04418397717161179", - "exact_ranking_score": "0.04418397717161179", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.04535556523362161", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.04535556523362161", - "family": "Pangu-Weather", - "model_tag": "pangu6", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.02328829578760066", - "ranking_selected_val_pr_auc": "0.02010310006243177", - "scope": "top20", - "seed": 1, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7231388400090598", - "tolerated_ranking_score": "0.7231388400090598", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.698733466652633", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.698733466652633", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7231388400090598", - "union_ranking_score": "0.7231388400090598", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.698733466652633", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.698733466652633", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0", - "exact_ranking_score": "0.0", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.0042863266180882984", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.0042863266180882984", - "family": "Pangu-Weather", - "model_tag": "pangu6", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.00043738577276574255", - "ranking_selected_val_pr_auc": "0.0004554894460132563", - "scope": "global", - "seed": 123, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.31812983009681495", - "tolerated_ranking_score": "0.31812983009681495", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.3113263265858816", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.3113263265858816", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.31812983009681495", - "union_ranking_score": "0.31812983009681495", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.3113263265858816", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.3113263265858816", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.11929903392496069", - "exact_ranking_score": "0.11833577252770228", - "exact_regret": 0.0009632613972584136, - "exact_test_gap": "0.0009632613972584136", - "exact_val_decision_score": "0.12107419208010925", - "exact_val_gap": "0.0017554166125510057", - "exact_val_ranking_score": "0.11931877546755824", - "family": "Pangu-Weather", - "model_tag": "pangu6", - "ranking_selected_head": "pixel MLP head", - "ranking_selected_head_arch": "pixel_mlp", - "ranking_selected_test_pr_auc": "0.07483645613667965", - "ranking_selected_val_pr_auc": "0.08124531938595873", - "scope": "top5", - "seed": 123, - "tolerated_decision_head": "linear probe", - "tolerated_decision_head_arch": "linear", - "tolerated_decision_score": "0.7856782652546647", - "tolerated_ranking_score": "0.7714149204832434", - "tolerated_regret": 0.014263344771421216, - "tolerated_test_gap": "0.014263344771421216", - "tolerated_val_decision_score": "0.8139394851723619", - "tolerated_val_gap": "0.016746033785793313", - "tolerated_val_ranking_score": "0.7971934513865686", - "union_decision_head": "linear probe", - "union_decision_head_arch": "linear", - "union_decision_score": "0.7856782652546647", - "union_ranking_score": "0.7714149204832434", - "union_regret": 0.014263344771421216, - "union_test_gap": "0.014263344771421216", - "union_val_decision_score": "0.8139394851723619", - "union_val_gap": "0.016746033785793313", - "union_val_ranking_score": "0.7971934513865686", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.07468382032272133", - "exact_ranking_score": "0.07325639739086803", - "exact_regret": 0.0014274229318533005, - "exact_test_gap": "0.0014274229318533005", - "exact_val_decision_score": "0.07596458629358399", - "exact_val_gap": "0.001355681359769309", - "exact_val_ranking_score": "0.07460890493381468", - "family": "Pangu-Weather", - "model_tag": "pangu6", - "ranking_selected_head": "pixel MLP head", - "ranking_selected_head_arch": "pixel_mlp", - "ranking_selected_test_pr_auc": "0.04354779125348941", - "ranking_selected_val_pr_auc": "0.04714252503011615", - "scope": "top10", - "seed": 123, - "tolerated_decision_head": "linear probe", - "tolerated_decision_head_arch": "linear", - "tolerated_decision_score": "0.711332312404288", - "tolerated_ranking_score": "0.6996715316108496", - "tolerated_regret": 0.011660780793438352, - "tolerated_test_gap": "0.011660780793438352", - "tolerated_val_decision_score": "0.7182744065368083", - "tolerated_val_gap": "0.01675612973026175", - "tolerated_val_ranking_score": "0.7015182768065465", - "union_decision_head": "linear probe", - "union_decision_head_arch": "linear", - "union_decision_score": "0.711332312404288", - "union_ranking_score": "0.6996715316108496", - "union_regret": 0.011660780793438352, - "union_test_gap": "0.011660780793438352", - "union_val_decision_score": "0.7182744065368083", - "union_val_gap": "0.01675612973026175", - "union_val_ranking_score": "0.7015182768065465", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.04194314737445728", - "exact_ranking_score": "0.04101254412979794", - "exact_regret": 0.0009306032446593365, - "exact_test_gap": "0.0009306032446593365", - "exact_val_decision_score": "0.04314380355334118", - "exact_val_gap": "0.0006119046291480426", - "exact_val_ranking_score": "0.04253189892419314", - "family": "Pangu-Weather", - "model_tag": "pangu6", - "ranking_selected_head": "pixel MLP head", - "ranking_selected_head_arch": "pixel_mlp", - "ranking_selected_test_pr_auc": "0.02276041840297606", - "ranking_selected_val_pr_auc": "0.02572965797764821", - "scope": "top20", - "seed": 123, - "tolerated_decision_head": "linear probe", - "tolerated_decision_head_arch": "linear", - "tolerated_decision_score": "0.7261721555350364", - "tolerated_ranking_score": "0.7185324707231184", - "tolerated_regret": 0.007639684811918013, - "tolerated_test_gap": "0.007639684811918013", - "tolerated_val_decision_score": "0.7019436364687074", - "tolerated_val_gap": "0.011268223121093768", - "tolerated_val_ranking_score": "0.6906754133476136", - "union_decision_head": "linear probe", - "union_decision_head_arch": "linear", - "union_decision_score": "0.7261721555350364", - "union_ranking_score": "0.7185324707231184", - "union_regret": 0.007639684811918013, - "union_test_gap": "0.007639684811918013", - "union_val_decision_score": "0.7019436364687074", - "union_val_gap": "0.011268223121093768", - "union_val_ranking_score": "0.6906754133476136", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0", - "exact_ranking_score": "0.0", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.0024802356223841263", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.0024802356223841263", - "family": "Pangu-Weather", - "model_tag": "pangu6", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.00040651309469793043", - "ranking_selected_val_pr_auc": "0.0004045310999550776", - "scope": "global", - "seed": 42, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.32154249424354786", - "tolerated_ranking_score": "0.32154249424354786", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.31489341288382094", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.31489341288382094", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.32154249424354786", - "union_ranking_score": "0.32154249424354786", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.31489341288382094", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.31489341288382094", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.12204234122042343", - "exact_ranking_score": "0.11810512654120699", - "exact_regret": 0.003937214679216441, - "exact_test_gap": "0.003937214679216441", - "exact_val_decision_score": "0.12285777876697085", - "exact_val_gap": "0.0046720888180781794", - "exact_val_ranking_score": "0.11818568994889267", - "family": "Pangu-Weather", - "model_tag": "pangu6", - "ranking_selected_head": "linear probe", - "ranking_selected_head_arch": "linear", - "ranking_selected_test_pr_auc": "0.08436045538757507", - "ranking_selected_val_pr_auc": "0.06648389466400928", - "scope": "top5", - "seed": 42, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7898933615140744", - "tolerated_ranking_score": "0.7701027275947573", - "tolerated_regret": 0.019790633919317124, - "tolerated_test_gap": "0.019790633919317124", - "tolerated_val_decision_score": "0.8039800995024876", - "tolerated_val_gap": "0.005996860420931505", - "tolerated_val_ranking_score": "0.7979832390815561", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7898933615140744", - "union_ranking_score": "0.7701027275947573", - "union_regret": 0.019790633919317124, - "union_test_gap": "0.019790633919317124", - "union_val_decision_score": "0.8039800995024876", - "union_val_gap": "0.005996860420931505", - "union_val_ranking_score": "0.7979832390815561", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.07639117550214027", - "exact_ranking_score": "0.07639117550214027", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.07748859672658975", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.07748859672658975", - "family": "Pangu-Weather", - "model_tag": "pangu6", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.03994690077965589", - "ranking_selected_val_pr_auc": "0.03757162574160602", - "scope": "top10", - "seed": 42, - "tolerated_decision_head": "pixel MLP head", - "tolerated_decision_head_arch": "pixel_mlp", - "tolerated_decision_score": "0.5502983042755504", - "tolerated_ranking_score": "0.7194503995322549", - "tolerated_regret": 0.0, - "tolerated_test_gap": "-0.1691520952567045", - "tolerated_val_decision_score": "0.7221556788281226", - "tolerated_val_gap": "0.013579036492356145", - "tolerated_val_ranking_score": "0.7085766423357664", - "union_decision_head": "pixel MLP head", - "union_decision_head_arch": "pixel_mlp", - "union_decision_score": "0.5502983042755504", - "union_ranking_score": "0.7194503995322549", - "union_regret": 0.0, - "union_test_gap": "-0.1691520952567045", - "union_val_decision_score": "0.7221556788281226", - "union_val_gap": "0.013579036492356145", - "union_val_ranking_score": "0.7085766423357664", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0430065972893146", - "exact_ranking_score": "0.0430065972893146", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.044111302847282145", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.044111302847282145", - "family": "Pangu-Weather", - "model_tag": "pangu6", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.021420904714594617", - "ranking_selected_val_pr_auc": "0.019667925605377917", - "scope": "top20", - "seed": 42, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7241279907754397", - "tolerated_ranking_score": "0.7241279907754397", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.6965685921876124", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.6965685921876124", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7241279907754397", - "union_ranking_score": "0.7241279907754397", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.6965685921876124", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.6965685921876124", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0", - "exact_ranking_score": "0.0", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.003894080996884735", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.003894080996884735", - "family": "Pangu-Weather", - "model_tag": "pangu6", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.00038395193539280824", - "ranking_selected_val_pr_auc": "0.0005470853500668261", - "scope": "global", - "seed": 7, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.31120670593952293", - "tolerated_ranking_score": "0.31120670593952293", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.30691963220059004", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.30691963220059004", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.31120670593952293", - "union_ranking_score": "0.31120670593952293", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.30691963220059004", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.30691963220059004", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.12325789540979415", - "exact_ranking_score": "0.11754630125254256", - "exact_regret": 0.005711594157251587, - "exact_test_gap": "0.005711594157251587", - "exact_val_decision_score": "0.12198694899453988", - "exact_val_gap": "0.004674396340790948", - "exact_val_ranking_score": "0.11731255265374893", - "family": "Pangu-Weather", - "model_tag": "pangu6", - "ranking_selected_head": "linear probe", - "ranking_selected_head_arch": "linear", - "ranking_selected_test_pr_auc": "0.05616248338975538", - "ranking_selected_val_pr_auc": "0.0679796765173011", - "scope": "top5", - "seed": 7, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7753490339867437", - "tolerated_ranking_score": "0.7714385719285964", - "tolerated_regret": 0.003910462058147268, - "tolerated_test_gap": "0.003910462058147268", - "tolerated_val_decision_score": "0.8010033444816053", - "tolerated_val_gap": "0.0033474102232191605", - "tolerated_val_ranking_score": "0.7976559342583861", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7753490339867437", - "union_ranking_score": "0.7714385719285964", - "union_regret": 0.003910462058147268, - "union_test_gap": "0.003910462058147268", - "union_val_decision_score": "0.8010033444816053", - "union_val_gap": "0.0033474102232191605", - "union_val_ranking_score": "0.7976559342583861", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.07779171894604768", - "exact_ranking_score": "0.07325639739086803", - "exact_regret": 0.004535321555179647, - "exact_test_gap": "0.004535321555179647", - "exact_val_decision_score": "0.07823677257201779", - "exact_val_gap": "0.0036278676382031083", - "exact_val_ranking_score": "0.07460890493381468", - "family": "Pangu-Weather", - "model_tag": "pangu6", - "ranking_selected_head": "pixel MLP head", - "ranking_selected_head_arch": "pixel_mlp", - "ranking_selected_test_pr_auc": "0.03227259772140749", - "ranking_selected_val_pr_auc": "0.04023132643529763", - "scope": "top10", - "seed": 7, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7032262609185687", - "tolerated_ranking_score": "0.6996715316108496", - "tolerated_regret": 0.003554729307719051, - "tolerated_test_gap": "0.003554729307719051", - "tolerated_val_decision_score": "0.7063110230649498", - "tolerated_val_gap": "0.004792746258403202", - "tolerated_val_ranking_score": "0.7015182768065465", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7032262609185687", - "union_ranking_score": "0.6996715316108496", - "union_regret": 0.003554729307719051, - "union_test_gap": "0.003554729307719051", - "union_val_decision_score": "0.7063110230649498", - "union_val_gap": "0.004792746258403202", - "union_val_ranking_score": "0.7015182768065465", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.04372215277109978", - "exact_ranking_score": "0.04101254412979794", - "exact_regret": 0.0027096086413018403, - "exact_test_gap": "0.0027096086413018403", - "exact_val_decision_score": "0.043845404352062356", - "exact_val_gap": "0.0013135054278692188", - "exact_val_ranking_score": "0.04253189892419314", - "family": "Pangu-Weather", - "model_tag": "pangu6", - "ranking_selected_head": "pixel MLP head", - "ranking_selected_head_arch": "pixel_mlp", - "ranking_selected_test_pr_auc": "0.018664183428008234", - "ranking_selected_val_pr_auc": "0.024661788918260808", - "scope": "top20", - "seed": 7, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.719461376864001", - "tolerated_ranking_score": "0.7185324707231184", - "tolerated_regret": 0.0009289061408825905, - "tolerated_test_gap": "0.0009289061408825905", - "tolerated_val_decision_score": "0.6947253251451462", - "tolerated_val_gap": "0.004049911797532557", - "tolerated_val_ranking_score": "0.6906754133476136", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.719461376864001", - "union_ranking_score": "0.7185324707231184", - "union_regret": 0.0009289061408825905, - "union_test_gap": "0.0009289061408825905", - "union_val_decision_score": "0.6947253251451462", - "union_val_gap": "0.004049911797532557", - "union_val_ranking_score": "0.6906754133476136", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0", - "exact_ranking_score": "0.0", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.003532320734722713", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.003532320734722713", - "family": "Pangu-Weather", - "model_tag": "pangu6", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.00044306685015413984", - "ranking_selected_val_pr_auc": "0.0004923922514041874", - "scope": "global", - "seed": 99, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.32045281275439075", - "tolerated_ranking_score": "0.32045281275439075", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.3031747624303057", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.3031747624303057", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.32045281275439075", - "union_ranking_score": "0.32045281275439075", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.3031747624303057", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.3031747624303057", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.12431310294297229", - "exact_ranking_score": "0.11915180074049142", - "exact_regret": 0.005161302202480872, - "exact_test_gap": "0.005161302202480872", - "exact_val_decision_score": "0.12584327970939285", - "exact_val_gap": "0.005682776510173682", - "exact_val_ranking_score": "0.12016050319921917", - "family": "Pangu-Weather", - "model_tag": "pangu6", - "ranking_selected_head": "linear probe", - "ranking_selected_head_arch": "linear", - "ranking_selected_test_pr_auc": "0.0711724674009212", - "ranking_selected_val_pr_auc": "0.08911464580470933", - "scope": "top5", - "seed": 99, - "tolerated_decision_head": "linear probe", - "tolerated_decision_head_arch": "linear", - "tolerated_decision_score": "0.7714385719285964", - "tolerated_ranking_score": "0.7714385719285964", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.7976559342583861", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.7976559342583861", - "union_decision_head": "linear probe", - "union_decision_head_arch": "linear", - "union_decision_score": "0.7714385719285964", - "union_ranking_score": "0.7714385719285964", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.7976559342583861", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.7976559342583861", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.07693219535324798", - "exact_ranking_score": "0.07422212618841832", - "exact_regret": 0.0027100691648296593, - "exact_test_gap": "0.0027100691648296593", - "exact_val_decision_score": "0.07818598599838919", - "exact_val_gap": "0.0029847395688903794", - "exact_val_ranking_score": "0.07520124642949881", - "family": "Pangu-Weather", - "model_tag": "pangu6", - "ranking_selected_head": "linear probe", - "ranking_selected_head_arch": "linear", - "ranking_selected_test_pr_auc": "0.045241782115254044", - "ranking_selected_val_pr_auc": "0.05810049257853764", - "scope": "top10", - "seed": 99, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.6996952251947172", - "tolerated_ranking_score": "0.6996715316108496", - "tolerated_regret": 2.369358386755671e-05, - "tolerated_test_gap": "2.369358386755671e-05", - "tolerated_val_decision_score": "0.7016606351675854", - "tolerated_val_gap": "0.00014235836103881905", - "tolerated_val_ranking_score": "0.7015182768065465", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.6996952251947172", - "union_ranking_score": "0.6996715316108496", - "union_regret": 2.369358386755671e-05, - "union_test_gap": "2.369358386755671e-05", - "union_val_decision_score": "0.7016606351675854", - "union_val_gap": "0.00014235836103881905", - "union_val_ranking_score": "0.7015182768065465", - "path": "" - }, - { - "exact_decision_head": "linear probe", - "exact_decision_head_arch": "linear", - "exact_decision_score": "0.0044004400440044", - "exact_ranking_score": "0.0044004400440044", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.06142857142857142", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.06142857142857142", - "family": "Pangu-Weather", - "model_tag": "pangu6", - "ranking_selected_head": "linear probe", - "ranking_selected_head_arch": "linear", - "ranking_selected_test_pr_auc": "0.024740444457605402", - "ranking_selected_val_pr_auc": "0.036676155849720546", - "scope": "top20", - "seed": 99, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7191095957774646", - "tolerated_ranking_score": "0.7183382629739177", - "tolerated_regret": 0.0007713328035469091, - "tolerated_test_gap": "0.0007713328035469091", - "tolerated_val_decision_score": "0.6915201983814487", - "tolerated_val_gap": "0.0008234265830721066", - "tolerated_val_ranking_score": "0.6906967717983766", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7191095957774646", - "union_ranking_score": "0.7183382629739177", - "union_regret": 0.0007713328035469091, - "union_test_gap": "0.0007713328035469091", - "union_val_decision_score": "0.6915201983814487", - "union_val_gap": "0.0008234265830721066", - "union_val_ranking_score": "0.6906967717983766", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0004925501476206198", - "exact_ranking_score": "0.0004925501476206198", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.0004727121748000204", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.0004727121748000204", - "family": "Prithvi-WxC", - "model_tag": "prithvi_wxc", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.0002463357401629007", - "ranking_selected_val_pr_auc": "0.00023641196480702655", - "scope": "global", - "seed": 1, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.240793572992212", - "tolerated_ranking_score": "0.240793572992212", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.2179368807313983", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.2179368807313983", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.240793572992212", - "union_ranking_score": "0.240793572992212", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.2179368807313983", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.2179368807313983", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.11833577252770228", - "exact_ranking_score": "0.11833577252770228", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.11931877546755824", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.11931877546755824", - "family": "Prithvi-WxC", - "model_tag": "prithvi_wxc", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.06288888888888888", - "ranking_selected_val_pr_auc": "0.06344444444444444", - "scope": "top5", - "seed": 1, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7714149204832434", - "tolerated_ranking_score": "0.7714149204832434", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.7971934513865686", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.7971934513865686", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7714149204832434", - "union_ranking_score": "0.7714149204832434", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.7971934513865686", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.7971934513865686", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.07325639739086803", - "exact_ranking_score": "0.07325639739086803", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.07460890493381468", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.07460890493381468", - "family": "Prithvi-WxC", - "model_tag": "prithvi_wxc", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.03802083333333333", - "ranking_selected_val_pr_auc": "0.03875", - "scope": "top10", - "seed": 1, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.6996715316108496", - "tolerated_ranking_score": "0.6996715316108496", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.7015182768065465", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.7015182768065465", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.6996715316108496", - "union_ranking_score": "0.6996715316108496", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.7015182768065465", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.7015182768065465", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.04101254412979794", - "exact_ranking_score": "0.04101254412979794", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.04253189892419314", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.04253189892419314", - "family": "Prithvi-WxC", - "model_tag": "prithvi_wxc", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.02093558282208589", - "ranking_selected_val_pr_auc": "0.0217280163599182", - "scope": "top20", - "seed": 1, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7185324707231184", - "tolerated_ranking_score": "0.7185324707231184", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.6906754133476136", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.6906754133476136", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7185324707231184", - "union_ranking_score": "0.7185324707231184", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.6906754133476136", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.6906754133476136", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0004925501476206198", - "exact_ranking_score": "0.0004925501476206198", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.0004727121748000204", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.0004727121748000204", - "family": "Prithvi-WxC", - "model_tag": "prithvi_wxc", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.0002463357401629007", - "ranking_selected_val_pr_auc": "0.00023641196480702655", - "scope": "global", - "seed": 123, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.240793572992212", - "tolerated_ranking_score": "0.240793572992212", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.2179368807313983", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.2179368807313983", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.240793572992212", - "union_ranking_score": "0.240793572992212", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.2179368807313983", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.2179368807313983", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.11833577252770228", - "exact_ranking_score": "0.11833577252770228", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.11931877546755824", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.11931877546755824", - "family": "Prithvi-WxC", - "model_tag": "prithvi_wxc", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.06288888888888888", - "ranking_selected_val_pr_auc": "0.06344444444444444", - "scope": "top5", - "seed": 123, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7714149204832434", - "tolerated_ranking_score": "0.7714149204832434", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.7971934513865686", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.7971934513865686", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7714149204832434", - "union_ranking_score": "0.7714149204832434", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.7971934513865686", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.7971934513865686", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.07325639739086803", - "exact_ranking_score": "0.07325639739086803", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.07460890493381468", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.07460890493381468", - "family": "Prithvi-WxC", - "model_tag": "prithvi_wxc", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.03802083333333333", - "ranking_selected_val_pr_auc": "0.03875", - "scope": "top10", - "seed": 123, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.6996715316108496", - "tolerated_ranking_score": "0.6996715316108496", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.7015182768065465", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.7015182768065465", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.6996715316108496", - "union_ranking_score": "0.6996715316108496", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.7015182768065465", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.7015182768065465", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.04101254412979794", - "exact_ranking_score": "0.04101254412979794", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.04253189892419314", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.04253189892419314", - "family": "Prithvi-WxC", - "model_tag": "prithvi_wxc", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.02093558282208589", - "ranking_selected_val_pr_auc": "0.0217280163599182", - "scope": "top20", - "seed": 123, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7185324707231184", - "tolerated_ranking_score": "0.7185324707231184", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.6906754133476136", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.6906754133476136", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7185324707231184", - "union_ranking_score": "0.7185324707231184", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.6906754133476136", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.6906754133476136", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0004925501476206198", - "exact_ranking_score": "0.0004925501476206198", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.0004727121748000204", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.0004727121748000204", - "family": "Prithvi-WxC", - "model_tag": "prithvi_wxc", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.0002463357401629007", - "ranking_selected_val_pr_auc": "0.00023641196480702655", - "scope": "global", - "seed": 42, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.240793572992212", - "tolerated_ranking_score": "0.240793572992212", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.2179368807313983", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.2179368807313983", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.240793572992212", - "union_ranking_score": "0.240793572992212", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.2179368807313983", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.2179368807313983", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.11833577252770228", - "exact_ranking_score": "0.11833577252770228", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.11931877546755824", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.11931877546755824", - "family": "Prithvi-WxC", - "model_tag": "prithvi_wxc", - "ranking_selected_head": "pixel MLP head", - "ranking_selected_head_arch": "pixel_mlp", - "ranking_selected_test_pr_auc": "0.06288888888888888", - "ranking_selected_val_pr_auc": "0.06344444444444444", - "scope": "top5", - "seed": 42, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7714149204832434", - "tolerated_ranking_score": "0.7714149204832434", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.7971934513865686", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.7971934513865686", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7714149204832434", - "union_ranking_score": "0.7714149204832434", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.7971934513865686", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.7971934513865686", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.07325639739086803", - "exact_ranking_score": "0.07325639739086803", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.07460890493381468", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.07460890493381468", - "family": "Prithvi-WxC", - "model_tag": "prithvi_wxc", - "ranking_selected_head": "pixel MLP head", - "ranking_selected_head_arch": "pixel_mlp", - "ranking_selected_test_pr_auc": "0.03802083333333333", - "ranking_selected_val_pr_auc": "0.03875", - "scope": "top10", - "seed": 42, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.6996715316108496", - "tolerated_ranking_score": "0.6996715316108496", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.7015182768065465", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.7015182768065465", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.6996715316108496", - "union_ranking_score": "0.6996715316108496", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.7015182768065465", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.7015182768065465", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.04101254412979794", - "exact_ranking_score": "0.04101254412979794", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.04253189892419314", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.04253189892419314", - "family": "Prithvi-WxC", - "model_tag": "prithvi_wxc", - "ranking_selected_head": "pixel MLP head", - "ranking_selected_head_arch": "pixel_mlp", - "ranking_selected_test_pr_auc": "0.02093558282208589", - "ranking_selected_val_pr_auc": "0.0217280163599182", - "scope": "top20", - "seed": 42, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7185324707231184", - "tolerated_ranking_score": "0.7185324707231184", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.6906754133476136", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.6906754133476136", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7185324707231184", - "union_ranking_score": "0.7185324707231184", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.6906754133476136", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.6906754133476136", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0004925501476206198", - "exact_ranking_score": "0.0004925501476206198", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.0004727121748000204", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.0004727121748000204", - "family": "Prithvi-WxC", - "model_tag": "prithvi_wxc", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.0002463357401629007", - "ranking_selected_val_pr_auc": "0.00023641196480702655", - "scope": "global", - "seed": 7, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.240793572992212", - "tolerated_ranking_score": "0.240793572992212", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.2179368807313983", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.2179368807313983", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.240793572992212", - "union_ranking_score": "0.240793572992212", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.2179368807313983", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.2179368807313983", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.11833577252770228", - "exact_ranking_score": "0.11833577252770228", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.11931877546755824", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.11931877546755824", - "family": "Prithvi-WxC", - "model_tag": "prithvi_wxc", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.06288888888888888", - "ranking_selected_val_pr_auc": "0.06344444444444444", - "scope": "top5", - "seed": 7, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7714149204832434", - "tolerated_ranking_score": "0.7714149204832434", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.7971934513865686", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.7971934513865686", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7714149204832434", - "union_ranking_score": "0.7714149204832434", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.7971934513865686", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.7971934513865686", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.07325639739086803", - "exact_ranking_score": "0.07325639739086803", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.07460890493381468", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.07460890493381468", - "family": "Prithvi-WxC", - "model_tag": "prithvi_wxc", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.03802083333333333", - "ranking_selected_val_pr_auc": "0.03875", - "scope": "top10", - "seed": 7, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.6996715316108496", - "tolerated_ranking_score": "0.6996715316108496", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.7015182768065465", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.7015182768065465", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.6996715316108496", - "union_ranking_score": "0.6996715316108496", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.7015182768065465", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.7015182768065465", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.04101254412979794", - "exact_ranking_score": "0.04101254412979794", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.04253189892419314", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.04253189892419314", - "family": "Prithvi-WxC", - "model_tag": "prithvi_wxc", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.02093558282208589", - "ranking_selected_val_pr_auc": "0.0217280163599182", - "scope": "top20", - "seed": 7, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7185324707231184", - "tolerated_ranking_score": "0.7185324707231184", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.6906754133476136", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.6906754133476136", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7185324707231184", - "union_ranking_score": "0.7185324707231184", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.6906754133476136", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.6906754133476136", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.00048497822606044473", - "exact_ranking_score": "0.00048497822606044473", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.00046544514931712863", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.00046544514931712863", - "family": "Prithvi-WxC", - "model_tag": "prithvi_wxc", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.000244434472614863", - "ranking_selected_val_pr_auc": "0.00023458729090319847", - "scope": "global", - "seed": 99, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.23755358049655212", - "tolerated_ranking_score": "0.23755358049655212", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.2149459035878704", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.2149459035878704", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.23755358049655212", - "union_ranking_score": "0.23755358049655212", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.2149459035878704", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.2149459035878704", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.11833577252770228", - "exact_ranking_score": "0.11833577252770228", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.11931877546755824", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.11931877546755824", - "family": "Prithvi-WxC", - "model_tag": "prithvi_wxc", - "ranking_selected_head": "linear probe", - "ranking_selected_head_arch": "linear", - "ranking_selected_test_pr_auc": "0.06288888888888888", - "ranking_selected_val_pr_auc": "0.06344444444444444", - "scope": "top5", - "seed": 99, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7714149204832434", - "tolerated_ranking_score": "0.7714149204832434", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.7971934513865686", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.7971934513865686", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7714149204832434", - "union_ranking_score": "0.7714149204832434", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.7971934513865686", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.7971934513865686", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.07325639739086803", - "exact_ranking_score": "0.07325639739086803", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.07460890493381468", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.07460890493381468", - "family": "Prithvi-WxC", - "model_tag": "prithvi_wxc", - "ranking_selected_head": "linear probe", - "ranking_selected_head_arch": "linear", - "ranking_selected_test_pr_auc": "0.03802083333333333", - "ranking_selected_val_pr_auc": "0.03875", - "scope": "top10", - "seed": 99, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.6996715316108496", - "tolerated_ranking_score": "0.6996715316108496", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.7015182768065465", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.7015182768065465", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.6996715316108496", - "union_ranking_score": "0.6996715316108496", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.7015182768065465", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.7015182768065465", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.04101254412979794", - "exact_ranking_score": "0.04101254412979794", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.04253189892419314", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.04253189892419314", - "family": "Prithvi-WxC", - "model_tag": "prithvi_wxc", - "ranking_selected_head": "linear probe", - "ranking_selected_head_arch": "linear", - "ranking_selected_test_pr_auc": "0.02093558282208589", - "ranking_selected_val_pr_auc": "0.0217280163599182", - "scope": "top20", - "seed": 99, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7185324707231184", - "tolerated_ranking_score": "0.7185324707231184", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.6906754133476136", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.6906754133476136", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7185324707231184", - "union_ranking_score": "0.7185324707231184", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.6906754133476136", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.6906754133476136", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.019669096378572257", - "exact_ranking_score": "0.019669096378572257", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.016877637130801686", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.016877637130801686", - "family": "Reference", - "model_tag": "reference", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.0027275598484246552", - "ranking_selected_val_pr_auc": "0.0024772994278327914", - "scope": "global", - "seed": 1, - "tolerated_decision_head": "linear probe", - "tolerated_decision_head_arch": "linear", - "tolerated_decision_score": "0.7286109603326707", - "tolerated_ranking_score": "0.6293736539482548", - "tolerated_regret": 0.09923730638441586, - "tolerated_test_gap": "0.09923730638441586", - "tolerated_val_decision_score": "0.7800653928063089", - "tolerated_val_gap": "0.02211882039338331", - "tolerated_val_ranking_score": "0.7579465724129256", - "union_decision_head": "linear probe", - "union_decision_head_arch": "linear", - "union_decision_score": "0.7286109603326707", - "union_ranking_score": "0.6293736539482548", - "union_regret": 0.09923730638441586, - "union_test_gap": "0.09923730638441586", - "union_val_decision_score": "0.7800653928063089", - "union_val_gap": "0.02211882039338331", - "union_val_ranking_score": "0.7579465724129256", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.24975320829220138", - "exact_ranking_score": "0.24975320829220138", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.2308050255932992", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.2308050255932992", - "family": "Reference", - "model_tag": "reference", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.1876883857001677", - "ranking_selected_val_pr_auc": "0.15408804654455288", - "scope": "top5", - "seed": 1, - "tolerated_decision_head": "linear probe", - "tolerated_decision_head_arch": "linear", - "tolerated_decision_score": "0.9093400335144248", - "tolerated_ranking_score": "0.8936632713133041", - "tolerated_regret": 0.015676762201120686, - "tolerated_test_gap": "0.015676762201120686", - "tolerated_val_decision_score": "0.8904903385864028", - "tolerated_val_gap": "0.029999615649296207", - "tolerated_val_ranking_score": "0.8604907229371066", - "union_decision_head": "linear probe", - "union_decision_head_arch": "linear", - "union_decision_score": "0.9093400335144248", - "union_ranking_score": "0.8936632713133041", - "union_regret": 0.015676762201120686, - "union_test_gap": "0.015676762201120686", - "union_val_decision_score": "0.8904903385864028", - "union_val_gap": "0.029999615649296207", - "union_val_ranking_score": "0.8604907229371066", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.22468179302711677", - "exact_ranking_score": "0.22468179302711677", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.20189905047476264", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.20189905047476264", - "family": "Reference", - "model_tag": "reference", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.1514864942696933", - "ranking_selected_val_pr_auc": "0.12361527601973549", - "scope": "top10", - "seed": 1, - "tolerated_decision_head": "linear probe", - "tolerated_decision_head_arch": "linear", - "tolerated_decision_score": "0.8764832842263638", - "tolerated_ranking_score": "0.8527949979229895", - "tolerated_regret": 0.023688286303374273, - "tolerated_test_gap": "0.023688286303374273", - "tolerated_val_decision_score": "0.8461060598607641", - "tolerated_val_gap": "0.02668959040704999", - "tolerated_val_ranking_score": "0.8194164694537142", - "union_decision_head": "linear probe", - "union_decision_head_arch": "linear", - "union_decision_score": "0.8764832842263638", - "union_ranking_score": "0.8527949979229895", - "union_regret": 0.023688286303374273, - "union_test_gap": "0.023688286303374273", - "union_val_decision_score": "0.8461060598607641", - "union_val_gap": "0.02668959040704999", - "union_val_ranking_score": "0.8194164694537142", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.18739352640545145", - "exact_ranking_score": "0.18739352640545145", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.16846229187071501", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.16846229187071501", - "family": "Reference", - "model_tag": "reference", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.10815899441738173", - "ranking_selected_val_pr_auc": "0.08968210725224908", - "scope": "top20", - "seed": 1, - "tolerated_decision_head": "linear probe", - "tolerated_decision_head_arch": "linear", - "tolerated_decision_score": "0.8685988450931266", - "tolerated_ranking_score": "0.8118474430575734", - "tolerated_regret": 0.05675140203555318, - "tolerated_test_gap": "0.05675140203555318", - "tolerated_val_decision_score": "0.8546299882836719", - "tolerated_val_gap": "0.06584679531636395", - "tolerated_val_ranking_score": "0.7887831929673079", - "union_decision_head": "linear probe", - "union_decision_head_arch": "linear", - "union_decision_score": "0.8685988450931266", - "union_ranking_score": "0.8118474430575734", - "union_regret": 0.05675140203555318, - "union_test_gap": "0.05675140203555318", - "union_val_decision_score": "0.8546299882836719", - "union_val_gap": "0.06584679531636395", - "union_val_ranking_score": "0.7887831929673079", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.02046783625730994", - "exact_ranking_score": "0.02046783625730994", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.022231722958529286", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.022231722958529286", - "family": "Reference", - "model_tag": "reference", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.0027479122488527847", - "ranking_selected_val_pr_auc": "0.002960918884295192", - "scope": "global", - "seed": 123, - "tolerated_decision_head": "pixel MLP head", - "tolerated_decision_head_arch": "pixel_mlp", - "tolerated_decision_score": "0.7342945764100982", - "tolerated_ranking_score": "0.5593234977538011", - "tolerated_regret": 0.17497107865629713, - "tolerated_test_gap": "0.17497107865629713", - "tolerated_val_decision_score": "0.7498273537587483", - "tolerated_val_gap": "0.059135931802591135", - "tolerated_val_ranking_score": "0.6906914219561572", - "union_decision_head": "pixel MLP head", - "union_decision_head_arch": "pixel_mlp", - "union_decision_score": "0.7342945764100982", - "union_ranking_score": "0.5593234977538011", - "union_regret": 0.17497107865629713, - "union_test_gap": "0.17497107865629713", - "union_val_decision_score": "0.7498273537587483", - "union_val_gap": "0.059135931802591135", - "union_val_ranking_score": "0.6906914219561572", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.30023455824863177", - "exact_ranking_score": "0.30023455824863177", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.32366412213740453", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.32366412213740453", - "family": "Reference", - "model_tag": "reference", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.22871793267947133", - "ranking_selected_val_pr_auc": "0.2090698081382707", - "scope": "top5", - "seed": 123, - "tolerated_decision_head": "pixel MLP head", - "tolerated_decision_head_arch": "pixel_mlp", - "tolerated_decision_score": "0.879307317963807", - "tolerated_ranking_score": "0.8766654895872674", - "tolerated_regret": 0.00264182837653959, - "tolerated_test_gap": "0.00264182837653959", - "tolerated_val_decision_score": "0.8667860857895108", - "tolerated_val_gap": "0.027700659869665634", - "tolerated_val_ranking_score": "0.8390854259198451", - "union_decision_head": "pixel MLP head", - "union_decision_head_arch": "pixel_mlp", - "union_decision_score": "0.879307317963807", - "union_ranking_score": "0.8766654895872674", - "union_regret": 0.00264182837653959, - "union_test_gap": "0.00264182837653959", - "union_val_decision_score": "0.8667860857895108", - "union_val_gap": "0.027700659869665634", - "union_val_ranking_score": "0.8390854259198451", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.26836581709145424", - "exact_ranking_score": "0.26836581709145424", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.2748815165876777", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.2748815165876777", - "family": "Reference", - "model_tag": "reference", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.18388043920625555", - "ranking_selected_val_pr_auc": "0.1622717982352778", - "scope": "top10", - "seed": 123, - "tolerated_decision_head": "pixel MLP head", - "tolerated_decision_head_arch": "pixel_mlp", - "tolerated_decision_score": "0.8621715199854308", - "tolerated_ranking_score": "0.8511911675536367", - "tolerated_regret": 0.010980352431794183, - "tolerated_test_gap": "0.010980352431794183", - "tolerated_val_decision_score": "0.8240905796307538", - "tolerated_val_gap": "0.034993802443587296", - "tolerated_val_ranking_score": "0.7890967771871665", - "union_decision_head": "pixel MLP head", - "union_decision_head_arch": "pixel_mlp", - "union_decision_score": "0.8621715199854308", - "union_ranking_score": "0.8511911675536367", - "union_regret": 0.010980352431794183, - "union_test_gap": "0.010980352431794183", - "union_val_decision_score": "0.8240905796307538", - "union_val_gap": "0.034993802443587296", - "union_val_ranking_score": "0.7890967771871665", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.2156398104265403", - "exact_ranking_score": "0.2156398104265403", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.20444444444444446", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.20444444444444446", - "family": "Reference", - "model_tag": "reference", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.12031424704824639", - "ranking_selected_val_pr_auc": "0.10479806700502603", - "scope": "top20", - "seed": 123, - "tolerated_decision_head": "pixel MLP head", - "tolerated_decision_head_arch": "pixel_mlp", - "tolerated_decision_score": "0.8587104723511633", - "tolerated_ranking_score": "0.8190550527106161", - "tolerated_regret": 0.039655419640547196, - "tolerated_test_gap": "0.039655419640547196", - "tolerated_val_decision_score": "0.8349865778850624", - "tolerated_val_gap": "0.06746718455313094", - "tolerated_val_ranking_score": "0.7675193933319314", - "union_decision_head": "pixel MLP head", - "union_decision_head_arch": "pixel_mlp", - "union_decision_score": "0.8587104723511633", - "union_ranking_score": "0.8190550527106161", - "union_regret": 0.039655419640547196, - "union_test_gap": "0.039655419640547196", - "union_val_decision_score": "0.8349865778850624", - "union_val_gap": "0.06746718455313094", - "union_val_ranking_score": "0.7675193933319314", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.02939068100358423", - "exact_ranking_score": "0.02939068100358423", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.0233226297110022", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.0233226297110022", - "family": "Reference", - "model_tag": "reference", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.0036593166775145225", - "ranking_selected_val_pr_auc": "0.0030153084181398305", - "scope": "global", - "seed": 42, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.6375251353613843", - "tolerated_ranking_score": "0.6375251353613843", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.7530848622062031", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.7530848622062031", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.6375251353613843", - "union_ranking_score": "0.6375251353613843", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.7530848622062031", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.7530848622062031", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.2930207778369739", - "exact_ranking_score": "0.2930207778369739", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.28806133625410735", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.28806133625410735", - "family": "Reference", - "model_tag": "reference", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.2392007486709378", - "ranking_selected_val_pr_auc": "0.18866470031812624", - "scope": "top5", - "seed": 42, - "tolerated_decision_head": "linear probe", - "tolerated_decision_head_arch": "linear", - "tolerated_decision_score": "0.8978895622772368", - "tolerated_ranking_score": "0.898795526537839", - "tolerated_regret": 0.0, - "tolerated_test_gap": "-0.0009059642606021878", - "tolerated_val_decision_score": "0.8718139484243548", - "tolerated_val_gap": "0.02207407287605323", - "tolerated_val_ranking_score": "0.8497398755483015", - "union_decision_head": "linear probe", - "union_decision_head_arch": "linear", - "union_decision_score": "0.8978895622772368", - "union_ranking_score": "0.898795526537839", - "union_regret": 0.0, - "union_test_gap": "-0.0009059642606021878", - "union_val_decision_score": "0.8718139484243548", - "union_val_gap": "0.02207407287605323", - "union_val_ranking_score": "0.8497398755483015", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.2800697269029634", - "exact_ranking_score": "0.2800697269029634", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.23774647887323946", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.23774647887323946", - "family": "Reference", - "model_tag": "reference", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.19246520724380978", - "ranking_selected_val_pr_auc": "0.14576604274214022", - "scope": "top10", - "seed": 42, - "tolerated_decision_head": "linear probe", - "tolerated_decision_head_arch": "linear", - "tolerated_decision_score": "0.8187209263802845", - "tolerated_ranking_score": "0.8592617140093417", - "tolerated_regret": 0.0, - "tolerated_test_gap": "-0.04054078762905722", - "tolerated_val_decision_score": "0.796582029663564", - "tolerated_val_gap": "0.006808162632569004", - "tolerated_val_ranking_score": "0.789773867030995", - "union_decision_head": "linear probe", - "union_decision_head_arch": "linear", - "union_decision_score": "0.8187209263802845", - "union_ranking_score": "0.8592617140093417", - "union_regret": 0.0, - "union_test_gap": "-0.04054078762905722", - "union_val_decision_score": "0.796582029663564", - "union_val_gap": "0.006808162632569004", - "union_val_ranking_score": "0.789773867030995", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.22105263157894736", - "exact_ranking_score": "0.22105263157894736", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.17796610169491525", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.17796610169491525", - "family": "Reference", - "model_tag": "reference", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.12467047355946634", - "ranking_selected_val_pr_auc": "0.0921769344676084", - "scope": "top20", - "seed": 42, - "tolerated_decision_head": "linear probe", - "tolerated_decision_head_arch": "linear", - "tolerated_decision_score": "0.8018567656897366", - "tolerated_ranking_score": "0.8307715292671506", - "tolerated_regret": 0.0, - "tolerated_test_gap": "-0.02891476357741396", - "tolerated_val_decision_score": "0.7802052162248743", - "tolerated_val_gap": "0.02094585104470381", - "tolerated_val_ranking_score": "0.7592593651801705", - "union_decision_head": "linear probe", - "union_decision_head_arch": "linear", - "union_decision_score": "0.8018567656897366", - "union_ranking_score": "0.8307715292671506", - "union_regret": 0.0, - "union_test_gap": "-0.02891476357741396", - "union_val_decision_score": "0.7802052162248743", - "union_val_gap": "0.02094585104470381", - "union_val_ranking_score": "0.7592593651801705", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.022511431586352448", - "exact_ranking_score": "0.022511431586352448", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.02669537136706136", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.02669537136706136", - "family": "Reference", - "model_tag": "reference", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.0030782058188584395", - "ranking_selected_val_pr_auc": "0.0034856171692621155", - "scope": "global", - "seed": 7, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.4897984500499446", - "tolerated_ranking_score": "0.4897984500499446", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.6888590116997952", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.6888590116997952", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.4897984500499446", - "union_ranking_score": "0.4897984500499446", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.6888590116997952", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.6888590116997952", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.34154929577464793", - "exact_ranking_score": "0.34154929577464793", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.3481654957064793", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.3481654957064793", - "family": "Reference", - "model_tag": "reference", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.26178619067169895", - "ranking_selected_val_pr_auc": "0.22178697667275007", - "scope": "top5", - "seed": 7, - "tolerated_decision_head": "pixel MLP head", - "tolerated_decision_head_arch": "pixel_mlp", - "tolerated_decision_score": "0.8388961892247042", - "tolerated_ranking_score": "0.8740083557882952", - "tolerated_regret": 0.0, - "tolerated_test_gap": "-0.03511216656359095", - "tolerated_val_decision_score": "0.8463626765874013", - "tolerated_val_gap": "0.020898995996049274", - "tolerated_val_ranking_score": "0.825463680591352", - "union_decision_head": "pixel MLP head", - "union_decision_head_arch": "pixel_mlp", - "union_decision_score": "0.8388961892247042", - "union_ranking_score": "0.8740083557882952", - "union_regret": 0.0, - "union_test_gap": "-0.03511216656359095", - "union_val_decision_score": "0.8463626765874013", - "union_val_gap": "0.020898995996049274", - "union_val_ranking_score": "0.825463680591352", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.305226480836237", - "exact_ranking_score": "0.305226480836237", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.29323758228605623", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.29323758228605623", - "family": "Reference", - "model_tag": "reference", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.21329613245127957", - "ranking_selected_val_pr_auc": "0.17698631108606733", - "scope": "top10", - "seed": 7, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.8407348467432524", - "tolerated_ranking_score": "0.8407348467432524", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.7878499830095744", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.7878499830095744", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.8407348467432524", - "union_ranking_score": "0.8407348467432524", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.7878499830095744", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.7878499830095744", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.23666210670314639", - "exact_ranking_score": "0.23666210670314639", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.22416812609457093", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.22416812609457093", - "family": "Reference", - "model_tag": "reference", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.14100646513486956", - "ranking_selected_val_pr_auc": "0.11693412738760335", - "scope": "top20", - "seed": 7, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.8085758039816233", - "tolerated_ranking_score": "0.8085758039816233", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.7682832316376906", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.7682832316376906", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.8085758039816233", - "union_ranking_score": "0.8085758039816233", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.7682832316376906", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.7682832316376906", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.021876258220373104", - "exact_ranking_score": "0.021876258220373104", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.017964421114027414", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.017964421114027414", - "family": "Reference", - "model_tag": "reference", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.0031953098868323193", - "ranking_selected_val_pr_auc": "0.0026642660544025525", - "scope": "global", - "seed": 99, - "tolerated_decision_head": "linear probe", - "tolerated_decision_head_arch": "linear", - "tolerated_decision_score": "0.7647466876509988", - "tolerated_ranking_score": "0.6698005926442897", - "tolerated_regret": 0.09494609500670914, - "tolerated_test_gap": "0.09494609500670914", - "tolerated_val_decision_score": "0.7870299254989238", - "tolerated_val_gap": "0.010195461444427734", - "tolerated_val_ranking_score": "0.7768344640544961", - "union_decision_head": "linear probe", - "union_decision_head_arch": "linear", - "union_decision_score": "0.7647466876509988", - "union_ranking_score": "0.6698005926442897", - "union_regret": 0.09494609500670914, - "union_test_gap": "0.09494609500670914", - "union_val_decision_score": "0.7870299254989238", - "union_val_gap": "0.010195461444427734", - "union_val_ranking_score": "0.7768344640544961", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.2789445341949381", - "exact_ranking_score": "0.2789445341949381", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.2726772952171523", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.2726772952171523", - "family": "Reference", - "model_tag": "reference", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.22946688569711154", - "ranking_selected_val_pr_auc": "0.17172613901536987", - "scope": "top5", - "seed": 99, - "tolerated_decision_head": "linear probe", - "tolerated_decision_head_arch": "linear", - "tolerated_decision_score": "0.9093512887929417", - "tolerated_ranking_score": "0.9158278502929932", - "tolerated_regret": 0.0, - "tolerated_test_gap": "-0.006476561500051492", - "tolerated_val_decision_score": "0.8688807502119548", - "tolerated_val_gap": "0.0021751625260553187", - "tolerated_val_ranking_score": "0.8667055876858994", - "union_decision_head": "linear probe", - "union_decision_head_arch": "linear", - "union_decision_score": "0.9093512887929417", - "union_ranking_score": "0.9158278502929932", - "union_regret": 0.0, - "union_test_gap": "-0.006476561500051492", - "union_val_decision_score": "0.8688807502119548", - "union_val_gap": "0.0021751625260553187", - "union_val_ranking_score": "0.8667055876858994", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.24522698345354263", - "exact_ranking_score": "0.24522698345354263", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.22014622258326566", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.22014622258326566", - "family": "Reference", - "model_tag": "reference", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.1797901995298074", - "ranking_selected_val_pr_auc": "0.12829763032218214", - "scope": "top10", - "seed": 99, - "tolerated_decision_head": "linear probe", - "tolerated_decision_head_arch": "linear", - "tolerated_decision_score": "0.9129217910828317", - "tolerated_ranking_score": "0.8862129818575714", - "tolerated_regret": 0.02670880922526031, - "tolerated_test_gap": "0.02670880922526031", - "tolerated_val_decision_score": "0.8661975382820603", - "tolerated_val_gap": "0.05189228302437687", - "tolerated_val_ranking_score": "0.8143052552576834", - "union_decision_head": "linear probe", - "union_decision_head_arch": "linear", - "union_decision_score": "0.9129217910828317", - "union_ranking_score": "0.8862129818575714", - "union_regret": 0.02670880922526031, - "union_test_gap": "0.02670880922526031", - "union_val_decision_score": "0.8661975382820603", - "union_val_gap": "0.05189228302437687", - "union_val_ranking_score": "0.8143052552576834", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.19477124183006533", - "exact_ranking_score": "0.19477124183006533", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.1633352370074243", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.1633352370074243", - "family": "Reference", - "model_tag": "reference", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.11947246238005743", - "ranking_selected_val_pr_auc": "0.08253093816555805", - "scope": "top20", - "seed": 99, - "tolerated_decision_head": "linear probe", - "tolerated_decision_head_arch": "linear", - "tolerated_decision_score": "0.9039923296574045", - "tolerated_ranking_score": "0.8534759193943048", - "tolerated_regret": 0.05051641026309972, - "tolerated_test_gap": "0.05051641026309972", - "tolerated_val_decision_score": "0.8730039040005804", - "tolerated_val_gap": "0.08884125994391634", - "tolerated_val_ranking_score": "0.784162644056664", - "union_decision_head": "linear probe", - "union_decision_head_arch": "linear", - "union_decision_score": "0.9039923296574045", - "union_ranking_score": "0.8534759193943048", - "union_regret": 0.05051641026309972, - "union_test_gap": "0.05051641026309972", - "union_val_decision_score": "0.8730039040005804", - "union_val_gap": "0.08884125994391634", - "union_val_ranking_score": "0.784162644056664", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0004925501476206198", - "exact_ranking_score": "0.0004925501476206198", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.0004727121748000204", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.0004727121748000204", - "family": "StormCast", - "model_tag": "stormcast", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.0002463357401629007", - "ranking_selected_val_pr_auc": "0.00023641196480702655", - "scope": "global", - "seed": 1, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.240793572992212", - "tolerated_ranking_score": "0.240793572992212", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.2179368807313983", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.2179368807313983", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.240793572992212", - "union_ranking_score": "0.240793572992212", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.2179368807313983", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.2179368807313983", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.11833577252770228", - "exact_ranking_score": "0.11833577252770228", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.11931877546755824", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.11931877546755824", - "family": "StormCast", - "model_tag": "stormcast", - "ranking_selected_head": "linear probe", - "ranking_selected_head_arch": "linear", - "ranking_selected_test_pr_auc": "0.06288888888888888", - "ranking_selected_val_pr_auc": "0.06344444444444444", - "scope": "top5", - "seed": 1, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7714149204832434", - "tolerated_ranking_score": "0.7714149204832434", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.7971934513865686", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.7971934513865686", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7714149204832434", - "union_ranking_score": "0.7714149204832434", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.7971934513865686", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.7971934513865686", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.07325639739086803", - "exact_ranking_score": "0.07325639739086803", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.07460890493381468", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.07460890493381468", - "family": "StormCast", - "model_tag": "stormcast", - "ranking_selected_head": "linear probe", - "ranking_selected_head_arch": "linear", - "ranking_selected_test_pr_auc": "0.03802083333333333", - "ranking_selected_val_pr_auc": "0.03875", - "scope": "top10", - "seed": 1, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.6996715316108496", - "tolerated_ranking_score": "0.6996715316108496", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.7015182768065465", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.7015182768065465", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.6996715316108496", - "union_ranking_score": "0.6996715316108496", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.7015182768065465", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.7015182768065465", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.04101254412979794", - "exact_ranking_score": "0.04101254412979794", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.04253189892419314", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.04253189892419314", - "family": "StormCast", - "model_tag": "stormcast", - "ranking_selected_head": "linear probe", - "ranking_selected_head_arch": "linear", - "ranking_selected_test_pr_auc": "0.02093558282208589", - "ranking_selected_val_pr_auc": "0.0217280163599182", - "scope": "top20", - "seed": 1, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7185324707231184", - "tolerated_ranking_score": "0.7185324707231184", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.6906754133476136", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.6906754133476136", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7185324707231184", - "union_ranking_score": "0.7185324707231184", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.6906754133476136", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.6906754133476136", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0004925501476206198", - "exact_ranking_score": "0.0004925501476206198", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.0004727121748000204", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.0004727121748000204", - "family": "StormCast", - "model_tag": "stormcast", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.0002463357401629007", - "ranking_selected_val_pr_auc": "0.00023641196480702655", - "scope": "global", - "seed": 123, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.240793572992212", - "tolerated_ranking_score": "0.240793572992212", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.2179368807313983", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.2179368807313983", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.240793572992212", - "union_ranking_score": "0.240793572992212", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.2179368807313983", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.2179368807313983", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.11833577252770228", - "exact_ranking_score": "0.11833577252770228", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.11931877546755824", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.11931877546755824", - "family": "StormCast", - "model_tag": "stormcast", - "ranking_selected_head": "linear probe", - "ranking_selected_head_arch": "linear", - "ranking_selected_test_pr_auc": "0.06288888888888888", - "ranking_selected_val_pr_auc": "0.06344444444444444", - "scope": "top5", - "seed": 123, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7714149204832434", - "tolerated_ranking_score": "0.7714149204832434", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.7971934513865686", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.7971934513865686", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7714149204832434", - "union_ranking_score": "0.7714149204832434", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.7971934513865686", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.7971934513865686", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.07325639739086803", - "exact_ranking_score": "0.07325639739086803", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.07460890493381468", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.07460890493381468", - "family": "StormCast", - "model_tag": "stormcast", - "ranking_selected_head": "linear probe", - "ranking_selected_head_arch": "linear", - "ranking_selected_test_pr_auc": "0.03802083333333333", - "ranking_selected_val_pr_auc": "0.03875", - "scope": "top10", - "seed": 123, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.6996715316108496", - "tolerated_ranking_score": "0.6996715316108496", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.7015182768065465", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.7015182768065465", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.6996715316108496", - "union_ranking_score": "0.6996715316108496", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.7015182768065465", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.7015182768065465", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.04101254412979794", - "exact_ranking_score": "0.04101254412979794", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.04253189892419314", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.04253189892419314", - "family": "StormCast", - "model_tag": "stormcast", - "ranking_selected_head": "linear probe", - "ranking_selected_head_arch": "linear", - "ranking_selected_test_pr_auc": "0.02093558282208589", - "ranking_selected_val_pr_auc": "0.0217280163599182", - "scope": "top20", - "seed": 123, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7185324707231184", - "tolerated_ranking_score": "0.7185324707231184", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.6906754133476136", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.6906754133476136", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7185324707231184", - "union_ranking_score": "0.7185324707231184", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.6906754133476136", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.6906754133476136", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0004925501476206198", - "exact_ranking_score": "0.0004925501476206198", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.0004727121748000204", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.0004727121748000204", - "family": "StormCast", - "model_tag": "stormcast", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.0002463357401629007", - "ranking_selected_val_pr_auc": "0.00023641196480702655", - "scope": "global", - "seed": 42, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.240793572992212", - "tolerated_ranking_score": "0.240793572992212", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.2179368807313983", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.2179368807313983", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.240793572992212", - "union_ranking_score": "0.240793572992212", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.2179368807313983", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.2179368807313983", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.11833577252770228", - "exact_ranking_score": "0.11833577252770228", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.11931877546755824", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.11931877546755824", - "family": "StormCast", - "model_tag": "stormcast", - "ranking_selected_head": "linear probe", - "ranking_selected_head_arch": "linear", - "ranking_selected_test_pr_auc": "0.06288888888888888", - "ranking_selected_val_pr_auc": "0.06344444444444444", - "scope": "top5", - "seed": 42, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7714149204832434", - "tolerated_ranking_score": "0.7714149204832434", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.7971934513865686", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.7971934513865686", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7714149204832434", - "union_ranking_score": "0.7714149204832434", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.7971934513865686", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.7971934513865686", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.07325639739086803", - "exact_ranking_score": "0.07325639739086803", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.07460890493381468", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.07460890493381468", - "family": "StormCast", - "model_tag": "stormcast", - "ranking_selected_head": "linear probe", - "ranking_selected_head_arch": "linear", - "ranking_selected_test_pr_auc": "0.03802083333333333", - "ranking_selected_val_pr_auc": "0.03875", - "scope": "top10", - "seed": 42, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.6996715316108496", - "tolerated_ranking_score": "0.6996715316108496", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.7015182768065465", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.7015182768065465", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.6996715316108496", - "union_ranking_score": "0.6996715316108496", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.7015182768065465", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.7015182768065465", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.04101254412979794", - "exact_ranking_score": "0.04101254412979794", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.04253189892419314", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.04253189892419314", - "family": "StormCast", - "model_tag": "stormcast", - "ranking_selected_head": "linear probe", - "ranking_selected_head_arch": "linear", - "ranking_selected_test_pr_auc": "0.02093558282208589", - "ranking_selected_val_pr_auc": "0.0217280163599182", - "scope": "top20", - "seed": 42, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7185324707231184", - "tolerated_ranking_score": "0.7185324707231184", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.6906754133476136", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.6906754133476136", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7185324707231184", - "union_ranking_score": "0.7185324707231184", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.6906754133476136", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.6906754133476136", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0004925501476206198", - "exact_ranking_score": "0.0004925501476206198", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.0004727121748000204", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.0004727121748000204", - "family": "StormCast", - "model_tag": "stormcast", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.0002463357401629007", - "ranking_selected_val_pr_auc": "0.00023641196480702655", - "scope": "global", - "seed": 7, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.240793572992212", - "tolerated_ranking_score": "0.240793572992212", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.2179368807313983", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.2179368807313983", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.240793572992212", - "union_ranking_score": "0.240793572992212", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.2179368807313983", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.2179368807313983", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.11833577252770228", - "exact_ranking_score": "0.11833577252770228", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.11931877546755824", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.11931877546755824", - "family": "StormCast", - "model_tag": "stormcast", - "ranking_selected_head": "linear probe", - "ranking_selected_head_arch": "linear", - "ranking_selected_test_pr_auc": "0.06288888888888888", - "ranking_selected_val_pr_auc": "0.06344444444444444", - "scope": "top5", - "seed": 7, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7714149204832434", - "tolerated_ranking_score": "0.7714149204832434", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.7971934513865686", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.7971934513865686", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7714149204832434", - "union_ranking_score": "0.7714149204832434", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.7971934513865686", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.7971934513865686", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.07325639739086803", - "exact_ranking_score": "0.07325639739086803", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.07460890493381468", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.07460890493381468", - "family": "StormCast", - "model_tag": "stormcast", - "ranking_selected_head": "linear probe", - "ranking_selected_head_arch": "linear", - "ranking_selected_test_pr_auc": "0.03802083333333333", - "ranking_selected_val_pr_auc": "0.03875", - "scope": "top10", - "seed": 7, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.6996715316108496", - "tolerated_ranking_score": "0.6996715316108496", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.7015182768065465", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.7015182768065465", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.6996715316108496", - "union_ranking_score": "0.6996715316108496", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.7015182768065465", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.7015182768065465", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.04101254412979794", - "exact_ranking_score": "0.04101254412979794", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.04253189892419314", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.04253189892419314", - "family": "StormCast", - "model_tag": "stormcast", - "ranking_selected_head": "linear probe", - "ranking_selected_head_arch": "linear", - "ranking_selected_test_pr_auc": "0.02093558282208589", - "ranking_selected_val_pr_auc": "0.0217280163599182", - "scope": "top20", - "seed": 7, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7185324707231184", - "tolerated_ranking_score": "0.7185324707231184", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.6906754133476136", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.6906754133476136", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7185324707231184", - "union_ranking_score": "0.7185324707231184", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.6906754133476136", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.6906754133476136", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.0004849854226143435", - "exact_ranking_score": "0.0004849854226143435", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.00046545205608870404", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.00046545205608870404", - "family": "StormCast", - "model_tag": "stormcast", - "ranking_selected_head": "shallow spatial adapter", - "ranking_selected_head_arch": "shallow", - "ranking_selected_test_pr_auc": "0.0002463357401629007", - "ranking_selected_val_pr_auc": "0.00023641196480702655", - "scope": "global", - "seed": 99, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.23755668759067805", - "tolerated_ranking_score": "0.23755668759067805", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.21494875104722905", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.21494875104722905", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.23755668759067805", - "union_ranking_score": "0.23755668759067805", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.21494875104722905", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.21494875104722905", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.11833577252770228", - "exact_ranking_score": "0.11833577252770228", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.11931877546755824", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.11931877546755824", - "family": "StormCast", - "model_tag": "stormcast", - "ranking_selected_head": "pixel MLP head", - "ranking_selected_head_arch": "pixel_mlp", - "ranking_selected_test_pr_auc": "0.06288888888888888", - "ranking_selected_val_pr_auc": "0.06344444444444444", - "scope": "top5", - "seed": 99, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7714149204832434", - "tolerated_ranking_score": "0.7714149204832434", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.7971934513865686", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.7971934513865686", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7714149204832434", - "union_ranking_score": "0.7714149204832434", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.7971934513865686", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.7971934513865686", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.07325639739086803", - "exact_ranking_score": "0.07325639739086803", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.07460890493381468", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.07460890493381468", - "family": "StormCast", - "model_tag": "stormcast", - "ranking_selected_head": "pixel MLP head", - "ranking_selected_head_arch": "pixel_mlp", - "ranking_selected_test_pr_auc": "0.03802083333333333", - "ranking_selected_val_pr_auc": "0.03875", - "scope": "top10", - "seed": 99, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.6996715316108496", - "tolerated_ranking_score": "0.6996715316108496", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.7015182768065465", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.7015182768065465", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.6996715316108496", - "union_ranking_score": "0.6996715316108496", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.7015182768065465", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.7015182768065465", - "path": "" - }, - { - "exact_decision_head": "shallow spatial adapter", - "exact_decision_head_arch": "shallow", - "exact_decision_score": "0.04101254412979794", - "exact_ranking_score": "0.04101254412979794", - "exact_regret": 0.0, - "exact_test_gap": "0.0", - "exact_val_decision_score": "0.04253189892419314", - "exact_val_gap": "0.0", - "exact_val_ranking_score": "0.04253189892419314", - "family": "StormCast", - "model_tag": "stormcast", - "ranking_selected_head": "pixel MLP head", - "ranking_selected_head_arch": "pixel_mlp", - "ranking_selected_test_pr_auc": "0.02093558282208589", - "ranking_selected_val_pr_auc": "0.0217280163599182", - "scope": "top20", - "seed": 99, - "tolerated_decision_head": "shallow spatial adapter", - "tolerated_decision_head_arch": "shallow", - "tolerated_decision_score": "0.7185324707231184", - "tolerated_ranking_score": "0.7185324707231184", - "tolerated_regret": 0.0, - "tolerated_test_gap": "0.0", - "tolerated_val_decision_score": "0.6906754133476136", - "tolerated_val_gap": "0.0", - "tolerated_val_ranking_score": "0.6906754133476136", - "union_decision_head": "shallow spatial adapter", - "union_decision_head_arch": "shallow", - "union_decision_score": "0.7185324707231184", - "union_ranking_score": "0.7185324707231184", - "union_regret": 0.0, - "union_test_gap": "0.0", - "union_val_decision_score": "0.6906754133476136", - "union_val_gap": "0.0", - "union_val_ranking_score": "0.6906754133476136", - "path": "" - } - ], - "summary": [ - { - "model_tag": "reference", - "label": "FireWx-FM ref.", - "scope": "global", - "scope_label": "\\(\\Omega=\\)global", - "n": 5, - "seeds": [ - 1, - 7, - 42, - 99, - 123 - ], - "paths": [ - "", - "", - "", - "", - "" - ], - "exact_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - }, - "tolerated_regret": { - "n": 5, - "mean": 0.07383089600948442, - "std": 0.07453636071372995, - "min": 0.0, - "max": 0.17497107865629713 - }, - "union_regret": { - "n": 5, - "mean": 0.07383089600948442, - "std": 0.07453636071372995, - "min": 0.0, - "max": 0.17497107865629713 - } - }, - { - "model_tag": "reference", - "label": "FireWx-FM ref.", - "scope": "top5", - "scope_label": "\\(\\Omega=\\)top 5\\%", - "n": 5, - "seeds": [ - 1, - 7, - 42, - 99, - 123 - ], - "paths": [ - "", - "", - "", - "", - "" - ], - "exact_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - }, - "tolerated_regret": { - "n": 5, - "mean": 0.003663718115532055, - "std": 0.006812231244812292, - "min": 0.0, - "max": 0.015676762201120686 - }, - "union_regret": { - "n": 5, - "mean": 0.003663718115532055, - "std": 0.006812231244812292, - "min": 0.0, - "max": 0.015676762201120686 - } - }, - { - "model_tag": "reference", - "label": "FireWx-FM ref.", - "scope": "top10", - "scope_label": "\\(\\Omega=\\)top 10\\%", - "n": 5, - "seeds": [ - 1, - 7, - 42, - 99, - 123 - ], - "paths": [ - "", - "", - "", - "", - "" - ], - "exact_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - }, - "tolerated_regret": { - "n": 5, - "mean": 0.012275489592085752, - "std": 0.012665162001740834, - "min": 0.0, - "max": 0.02670880922526031 - }, - "union_regret": { - "n": 5, - "mean": 0.012275489592085752, - "std": 0.012665162001740834, - "min": 0.0, - "max": 0.02670880922526031 - } - }, - { - "model_tag": "reference", - "label": "FireWx-FM ref.", - "scope": "top20", - "scope_label": "\\(\\Omega=\\)top 20\\%", - "n": 5, - "seeds": [ - 1, - 7, - 42, - 99, - 123 - ], - "paths": [ - "", - "", - "", - "", - "" - ], - "exact_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - }, - "tolerated_regret": { - "n": 5, - "mean": 0.029384646387840017, - "std": 0.02751315335001922, - "min": 0.0, - "max": 0.05675140203555318 - }, - "union_regret": { - "n": 5, - "mean": 0.029384646387840017, - "std": 0.02751315335001922, - "min": 0.0, - "max": 0.05675140203555318 - } - }, - { - "model_tag": "prithvi_wxc", - "label": "Prithvi-WxC", - "scope": "global", - "scope_label": "\\(\\Omega=\\)global", - "n": 5, - "seeds": [ - 1, - 7, - 42, - 99, - 123 - ], - "paths": [ - "", - "", - "", - "", - "" - ], - "exact_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - }, - "tolerated_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - }, - "union_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - } - }, - { - "model_tag": "prithvi_wxc", - "label": "Prithvi-WxC", - "scope": "top5", - "scope_label": "\\(\\Omega=\\)top 5\\%", - "n": 5, - "seeds": [ - 1, - 7, - 42, - 99, - 123 - ], - "paths": [ - "", - "", - "", - "", - "" - ], - "exact_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - }, - "tolerated_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - }, - "union_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - } - }, - { - "model_tag": "prithvi_wxc", - "label": "Prithvi-WxC", - "scope": "top10", - "scope_label": "\\(\\Omega=\\)top 10\\%", - "n": 5, - "seeds": [ - 1, - 7, - 42, - 99, - 123 - ], - "paths": [ - "", - "", - "", - "", - "" - ], - "exact_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - }, - "tolerated_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - }, - "union_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - } - }, - { - "model_tag": "prithvi_wxc", - "label": "Prithvi-WxC", - "scope": "top20", - "scope_label": "\\(\\Omega=\\)top 20\\%", - "n": 5, - "seeds": [ - 1, - 7, - 42, - 99, - 123 - ], - "paths": [ - "", - "", - "", - "", - "" - ], - "exact_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - }, - "tolerated_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - }, - "union_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - } - }, - { - "model_tag": "aurora", - "label": "Aurora", - "scope": "global", - "scope_label": "\\(\\Omega=\\)global", - "n": 5, - "seeds": [ - 1, - 7, - 42, - 99, - 123 - ], - "paths": [ - "", - "", - "", - "", - "" - ], - "exact_regret": { - "n": 5, - "mean": 0.00010153879814819402, - "std": 0.00021861477435572763, - "min": 0.0, - "max": 0.0004925501476206198 - }, - "tolerated_regret": { - "n": 5, - "mean": 0.04945471159670635, - "std": 0.10697394238964528, - "min": 0.0, - "max": 0.240793572992212 - }, - "union_regret": { - "n": 5, - "mean": 0.04945471159670635, - "std": 0.10697394238964528, - "min": 0.0, - "max": 0.240793572992212 - } - }, - { - "model_tag": "aurora", - "label": "Aurora", - "scope": "top5", - "scope_label": "\\(\\Omega=\\)top 5\\%", - "n": 5, - "seeds": [ - 1, - 7, - 42, - 99, - 123 - ], - "paths": [ - "", - "", - "", - "", - "" - ], - "exact_regret": { - "n": 5, - "mean": 0.023667154505540456, - "std": 0.05292136630837888, - "min": 0.0, - "max": 0.11833577252770228 - }, - "tolerated_regret": { - "n": 5, - "mean": 0.1542829840966487, - "std": 0.34498724021162547, - "min": 0.0, - "max": 0.7714149204832434 - }, - "union_regret": { - "n": 5, - "mean": 0.1542829840966487, - "std": 0.34498724021162547, - "min": 0.0, - "max": 0.7714149204832434 - } - }, - { - "model_tag": "aurora", - "label": "Aurora", - "scope": "top10", - "scope_label": "\\(\\Omega=\\)top 10\\%", - "n": 5, - "seeds": [ - 1, - 7, - 42, - 99, - 123 - ], - "paths": [ - "", - "", - "", - "", - "" - ], - "exact_regret": { - "n": 5, - "mean": 0.014651279478173606, - "std": 0.032761256870543834, - "min": 0.0, - "max": 0.07325639739086803 - }, - "tolerated_regret": { - "n": 5, - "mean": 0.1399343063221699, - "std": 0.31290262132065055, - "min": 0.0, - "max": 0.6996715316108496 - }, - "union_regret": { - "n": 5, - "mean": 0.1399343063221699, - "std": 0.31290262132065055, - "min": 0.0, - "max": 0.6996715316108496 - } - }, - { - "model_tag": "aurora", - "label": "Aurora", - "scope": "top20", - "scope_label": "\\(\\Omega=\\)top 20\\%", - "n": 5, - "seeds": [ - 1, - 7, - 42, - 99, - 123 - ], - "paths": [ - "", - "", - "", - "", - "" - ], - "exact_regret": { - "n": 5, - "mean": 0.008202508825959588, - "std": 0.01834136732088763, - "min": 0.0, - "max": 0.04101254412979794 - }, - "tolerated_regret": { - "n": 5, - "mean": 0.1437064941446237, - "std": 0.32133748971555404, - "min": 0.0, - "max": 0.7185324707231184 - }, - "union_regret": { - "n": 5, - "mean": 0.1437064941446237, - "std": 0.32133748971555404, - "min": 0.0, - "max": 0.7185324707231184 - } - }, - { - "model_tag": "climax", - "label": "ClimaX", - "scope": "global", - "scope_label": "\\(\\Omega=\\)global", - "n": 5, - "seeds": [ - 1, - 7, - 42, - 99, - 123 - ], - "paths": [ - "", - "", - "", - "", - "" - ], - "exact_regret": { - "n": 5, - "mean": 3.0287686240700486e-06, - "std": 4.147312242167625e-06, - "min": 0.0, - "max": 7.571921560175121e-06 - }, - "tolerated_regret": { - "n": 5, - "mean": 0.0012959969982639485, - "std": 0.0017746169760203706, - "min": 0.0, - "max": 0.0032399924956598714 - }, - "union_regret": { - "n": 5, - "mean": 0.0012959969982639485, - "std": 0.0017746169760203706, - "min": 0.0, - "max": 0.0032399924956598714 - } - }, - { - "model_tag": "climax", - "label": "ClimaX", - "scope": "top5", - "scope_label": "\\(\\Omega=\\)top 5\\%", - "n": 5, - "seeds": [ - 1, - 7, - 42, - 99, - 123 - ], - "paths": [ - "", - "", - "", - "", - "" - ], - "exact_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - }, - "tolerated_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - }, - "union_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - } - }, - { - "model_tag": "climax", - "label": "ClimaX", - "scope": "top10", - "scope_label": "\\(\\Omega=\\)top 10\\%", - "n": 5, - "seeds": [ - 1, - 7, - 42, - 99, - 123 - ], - "paths": [ - "", - "", - "", - "", - "" - ], - "exact_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - }, - "tolerated_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - }, - "union_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - } - }, - { - "model_tag": "climax", - "label": "ClimaX", - "scope": "top20", - "scope_label": "\\(\\Omega=\\)top 20\\%", - "n": 5, - "seeds": [ - 1, - 7, - 42, - 99, - 123 - ], - "paths": [ - "", - "", - "", - "", - "" - ], - "exact_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - }, - "tolerated_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - }, - "union_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - } - }, - { - "model_tag": "stormcast", - "label": "StormCast", - "scope": "global", - "scope_label": "\\(\\Omega=\\)global", - "n": 5, - "seeds": [ - 1, - 7, - 42, - 99, - 123 - ], - "paths": [ - "", - "", - "", - "", - "" - ], - "exact_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - }, - "tolerated_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - }, - "union_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - } - }, - { - "model_tag": "stormcast", - "label": "StormCast", - "scope": "top5", - "scope_label": "\\(\\Omega=\\)top 5\\%", - "n": 5, - "seeds": [ - 1, - 7, - 42, - 99, - 123 - ], - "paths": [ - "", - "", - "", - "", - "" - ], - "exact_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - }, - "tolerated_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - }, - "union_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - } - }, - { - "model_tag": "stormcast", - "label": "StormCast", - "scope": "top10", - "scope_label": "\\(\\Omega=\\)top 10\\%", - "n": 5, - "seeds": [ - 1, - 7, - 42, - 99, - 123 - ], - "paths": [ - "", - "", - "", - "", - "" - ], - "exact_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - }, - "tolerated_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - }, - "union_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - } - }, - { - "model_tag": "stormcast", - "label": "StormCast", - "scope": "top20", - "scope_label": "\\(\\Omega=\\)top 20\\%", - "n": 5, - "seeds": [ - 1, - 7, - 42, - 99, - 123 - ], - "paths": [ - "", - "", - "", - "", - "" - ], - "exact_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - }, - "tolerated_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - }, - "union_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - } - }, - { - "model_tag": "dlwp", - "label": "DLWP", - "scope": "global", - "scope_label": "\\(\\Omega=\\)global", - "n": 5, - "seeds": [ - 1, - 7, - 42, - 99, - 123 - ], - "paths": [ - "", - "", - "", - "", - "" - ], - "exact_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - }, - "tolerated_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - }, - "union_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - } - }, - { - "model_tag": "dlwp", - "label": "DLWP", - "scope": "top5", - "scope_label": "\\(\\Omega=\\)top 5\\%", - "n": 5, - "seeds": [ - 1, - 7, - 42, - 99, - 123 - ], - "paths": [ - "", - "", - "", - "", - "" - ], - "exact_regret": { - "n": 5, - "mean": 0.0048037709215293075, - "std": 0.006217185202152866, - "min": 0.0, - "max": 0.015203005078871956 - }, - "tolerated_regret": { - "n": 5, - "mean": 0.016716228534155796, - "std": 0.016079313546074458, - "min": 0.0, - "max": 0.03305057342744666 - }, - "union_regret": { - "n": 5, - "mean": 0.016716228534155796, - "std": 0.016079313546074458, - "min": 0.0, - "max": 0.03305057342744666 - } - }, - { - "model_tag": "dlwp", - "label": "DLWP", - "scope": "top10", - "scope_label": "\\(\\Omega=\\)top 10\\%", - "n": 5, - "seeds": [ - 1, - 7, - 42, - 99, - 123 - ], - "paths": [ - "", - "", - "", - "", - "" - ], - "exact_regret": { - "n": 5, - "mean": 0.0017281632798742507, - "std": 0.002514722758075371, - "min": 0.0, - "max": 0.005523780499856246 - }, - "tolerated_regret": { - "n": 5, - "mean": 0.02846514801700826, - "std": 0.026938012702643194, - "min": 0.0, - "max": 0.053927677500854476 - }, - "union_regret": { - "n": 5, - "mean": 0.02846514801700826, - "std": 0.026938012702643194, - "min": 0.0, - "max": 0.053927677500854476 - } - }, - { - "model_tag": "dlwp", - "label": "DLWP", - "scope": "top20", - "scope_label": "\\(\\Omega=\\)top 20\\%", - "n": 5, - "seeds": [ - 1, - 7, - 42, - 99, - 123 - ], - "paths": [ - "", - "", - "", - "", - "" - ], - "exact_regret": { - "n": 5, - "mean": 0.0007702319787454587, - "std": 0.0010995336594539604, - "min": 0.0, - "max": 0.0023651634514294945 - }, - "tolerated_regret": { - "n": 5, - "mean": 0.04463354681768479, - "std": 0.04356064433532197, - "min": 0.0, - "max": 0.11215006616165157 - }, - "union_regret": { - "n": 5, - "mean": 0.04463354681768479, - "std": 0.04356064433532197, - "min": 0.0, - "max": 0.11215006616165157 - } - }, - { - "model_tag": "fcn", - "label": "FCN", - "scope": "global", - "scope_label": "\\(\\Omega=\\)global", - "n": 5, - "seeds": [ - 1, - 7, - 42, - 99, - 123 - ], - "paths": [ - "", - "", - "", - "", - "" - ], - "exact_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - }, - "tolerated_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - }, - "union_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - } - }, - { - "model_tag": "fcn", - "label": "FCN", - "scope": "top5", - "scope_label": "\\(\\Omega=\\)top 5\\%", - "n": 5, - "seeds": [ - 1, - 7, - 42, - 99, - 123 - ], - "paths": [ - "", - "", - "", - "", - "" - ], - "exact_regret": { - "n": 5, - "mean": 0.0006342898232943345, - "std": 0.0009899554165032742, - "min": 0.0, - "max": 0.002257520679520411 - }, - "tolerated_regret": { - "n": 5, - "mean": 0.004509624980300697, - "std": 0.010070611656609236, - "min": 0.0, - "max": 0.022524473456150496 - }, - "union_regret": { - "n": 5, - "mean": 0.004509624980300697, - "std": 0.010070611656609236, - "min": 0.0, - "max": 0.022524473456150496 - } - }, - { - "model_tag": "fcn", - "label": "FCN", - "scope": "top10", - "scope_label": "\\(\\Omega=\\)top 10\\%", - "n": 5, - "seeds": [ - 1, - 7, - 42, - 99, - 123 - ], - "paths": [ - "", - "", - "", - "", - "" - ], - "exact_regret": { - "n": 5, - "mean": 0.00021156854817603877, - "std": 0.0004730816556225618, - "min": 0.0, - "max": 0.0010578427408801938 - }, - "tolerated_regret": { - "n": 5, - "mean": 0.004199537050817615, - "std": 0.009390450319657174, - "min": 0.0, - "max": 0.020997685254088072 - }, - "union_regret": { - "n": 5, - "mean": 0.004199537050817615, - "std": 0.009390450319657174, - "min": 0.0, - "max": 0.020997685254088072 - } - }, - { - "model_tag": "fcn", - "label": "FCN", - "scope": "top20", - "scope_label": "\\(\\Omega=\\)top 20\\%", - "n": 5, - "seeds": [ - 1, - 7, - 42, - 99, - 123 - ], - "paths": [ - "", - "", - "", - "", - "" - ], - "exact_regret": { - "n": 5, - "mean": 5.754560074337778e-06, - "std": 1.2867587506825515e-05, - "min": 0.0, - "max": 2.877280037168889e-05 - }, - "tolerated_regret": { - "n": 5, - "mean": 0.011679805987441694, - "std": 0.019872372458657642, - "min": 0.0, - "max": 0.045880503237594294 - }, - "union_regret": { - "n": 5, - "mean": 0.011679805987441694, - "std": 0.019872372458657642, - "min": 0.0, - "max": 0.045880503237594294 - } - }, - { - "model_tag": "fengwu", - "label": "FengWu", - "scope": "global", - "scope_label": "\\(\\Omega=\\)global", - "n": 5, - "seeds": [ - 1, - 7, - 42, - 99, - 123 - ], - "paths": [ - "", - "", - "", - "", - "" - ], - "exact_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - }, - "tolerated_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - }, - "union_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - } - }, - { - "model_tag": "fengwu", - "label": "FengWu", - "scope": "top5", - "scope_label": "\\(\\Omega=\\)top 5\\%", - "n": 5, - "seeds": [ - 1, - 7, - 42, - 99, - 123 - ], - "paths": [ - "", - "", - "", - "", - "" - ], - "exact_regret": { - "n": 5, - "mean": 0.0005029843170376968, - "std": 0.0008109166521114917, - "min": 0.0, - "max": 0.0018628094907809783 - }, - "tolerated_regret": { - "n": 5, - "mean": 0.008795951947678215, - "std": 0.005532321338017505, - "min": 0.0, - "max": 0.01484136735033148 - }, - "union_regret": { - "n": 5, - "mean": 0.008795951947678215, - "std": 0.005532321338017505, - "min": 0.0, - "max": 0.01484136735033148 - } - }, - { - "model_tag": "fengwu", - "label": "FengWu", - "scope": "top10", - "scope_label": "\\(\\Omega=\\)top 10\\%", - "n": 5, - "seeds": [ - 1, - 7, - 42, - 99, - 123 - ], - "paths": [ - "", - "", - "", - "", - "" - ], - "exact_regret": { - "n": 5, - "mean": 0.000495228089292582, - "std": 0.0007349190216431337, - "min": 0.0, - "max": 0.0016387212062008855 - }, - "tolerated_regret": { - "n": 5, - "mean": 0.00402300475984525, - "std": 0.005510851442075993, - "min": 0.0, - "max": 0.010273937098576491 - }, - "union_regret": { - "n": 5, - "mean": 0.00402300475984525, - "std": 0.005510851442075993, - "min": 0.0, - "max": 0.010273937098576491 - } - }, - { - "model_tag": "fengwu", - "label": "FengWu", - "scope": "top20", - "scope_label": "\\(\\Omega=\\)top 20\\%", - "n": 5, - "seeds": [ - 1, - 7, - 42, - 99, - 123 - ], - "paths": [ - "", - "", - "", - "", - "" - ], - "exact_regret": { - "n": 5, - "mean": 0.0006908222234409067, - "std": 0.0011910586589384115, - "min": 0.0, - "max": 0.0027505832409660327 - }, - "tolerated_regret": { - "n": 5, - "mean": 0.005222389249812243, - "std": 0.0062394095558402415, - "min": 0.0, - "max": 0.015150501067724198 - }, - "union_regret": { - "n": 5, - "mean": 0.005222389249812243, - "std": 0.0062394095558402415, - "min": 0.0, - "max": 0.015150501067724198 - } - }, - { - "model_tag": "fuxi", - "label": "FuXi", - "scope": "global", - "scope_label": "\\(\\Omega=\\)global", - "n": 5, - "seeds": [ - 1, - 7, - 42, - 99, - 123 - ], - "paths": [ - "", - "", - "", - "", - "" - ], - "exact_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - }, - "tolerated_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - }, - "union_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - } - }, - { - "model_tag": "fuxi", - "label": "FuXi", - "scope": "top5", - "scope_label": "\\(\\Omega=\\)top 5\\%", - "n": 5, - "seeds": [ - 1, - 7, - 42, - 99, - 123 - ], - "paths": [ - "", - "", - "", - "", - "" - ], - "exact_regret": { - "n": 5, - "mean": 0.002973545331200933, - "std": 0.0023946274991058026, - "min": 0.0010927807990139538, - "max": 0.007024214143542151 - }, - "tolerated_regret": { - "n": 5, - "mean": 0.013545122545609134, - "std": 0.02097023683418404, - "min": 0.0, - "max": 0.050156261654859424 - }, - "union_regret": { - "n": 5, - "mean": 0.013545122545609134, - "std": 0.02097023683418404, - "min": 0.0, - "max": 0.050156261654859424 - } - }, - { - "model_tag": "fuxi", - "label": "FuXi", - "scope": "top10", - "scope_label": "\\(\\Omega=\\)top 10\\%", - "n": 5, - "seeds": [ - 1, - 7, - 42, - 99, - 123 - ], - "paths": [ - "", - "", - "", - "", - "" - ], - "exact_regret": { - "n": 5, - "mean": 0.001383793743586542, - "std": 0.0019248128430711165, - "min": 0.0, - "max": 0.003938013087198336 - }, - "tolerated_regret": { - "n": 5, - "mean": 0.0016559834970027332, - "std": 0.0037028916689159307, - "min": 0.0, - "max": 0.008279917485013666 - }, - "union_regret": { - "n": 5, - "mean": 0.0016559834970027332, - "std": 0.0037028916689159307, - "min": 0.0, - "max": 0.008279917485013666 - } - }, - { - "model_tag": "fuxi", - "label": "FuXi", - "scope": "top20", - "scope_label": "\\(\\Omega=\\)top 20\\%", - "n": 5, - "seeds": [ - 1, - 7, - 42, - 99, - 123 - ], - "paths": [ - "", - "", - "", - "", - "" - ], - "exact_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - }, - "tolerated_regret": { - "n": 5, - "mean": 0.00283318355751887, - "std": 0.0036808289681375247, - "min": 0.0, - "max": 0.008746323525994693 - }, - "union_regret": { - "n": 5, - "mean": 0.00283318355751887, - "std": 0.0036808289681375247, - "min": 0.0, - "max": 0.008746323525994693 - } - }, - { - "model_tag": "pangu6", - "label": "Pangu-Weather", - "scope": "global", - "scope_label": "\\(\\Omega=\\)global", - "n": 5, - "seeds": [ - 1, - 7, - 42, - 99, - 123 - ], - "paths": [ - "", - "", - "", - "", - "" - ], - "exact_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - }, - "tolerated_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - }, - "union_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - } - }, - { - "model_tag": "pangu6", - "label": "Pangu-Weather", - "scope": "top5", - "scope_label": "\\(\\Omega=\\)top 5\\%", - "n": 5, - "seeds": [ - 1, - 7, - 42, - 99, - 123 - ], - "paths": [ - "", - "", - "", - "", - "" - ], - "exact_regret": { - "n": 5, - "mean": 0.003154674487241463, - "std": 0.002546125713211599, - "min": 0.0, - "max": 0.005711594157251587 - }, - "tolerated_regret": { - "n": 5, - "mean": 0.007592888149777122, - "std": 0.00897418737588444, - "min": 0.0, - "max": 0.019790633919317124 - }, - "union_regret": { - "n": 5, - "mean": 0.007592888149777122, - "std": 0.00897418737588444, - "min": 0.0, - "max": 0.019790633919317124 - } - }, - { - "model_tag": "pangu6", - "label": "Pangu-Weather", - "scope": "top10", - "scope_label": "\\(\\Omega=\\)top 10\\%", - "n": 5, - "seeds": [ - 1, - 7, - 42, - 99, - 123 - ], - "paths": [ - "", - "", - "", - "", - "" - ], - "exact_regret": { - "n": 5, - "mean": 0.0017345627303725214, - "std": 0.0019305189318827886, - "min": 0.0, - "max": 0.004535321555179647 - }, - "tolerated_regret": { - "n": 5, - "mean": 0.003047840737004992, - "std": 0.005053805614558161, - "min": 0.0, - "max": 0.011660780793438352 - }, - "union_regret": { - "n": 5, - "mean": 0.003047840737004992, - "std": 0.005053805614558161, - "min": 0.0, - "max": 0.011660780793438352 - } - }, - { - "model_tag": "pangu6", - "label": "Pangu-Weather", - "scope": "top20", - "scope_label": "\\(\\Omega=\\)top 20\\%", - "n": 5, - "seeds": [ - 1, - 7, - 42, - 99, - 123 - ], - "paths": [ - "", - "", - "", - "", - "" - ], - "exact_regret": { - "n": 5, - "mean": 0.0007280423771922354, - "std": 0.001178746460551365, - "min": 0.0, - "max": 0.0027096086413018403 - }, - "tolerated_regret": { - "n": 5, - "mean": 0.0018679847512695024, - "std": 0.0032548337047755126, - "min": 0.0, - "max": 0.007639684811918013 - }, - "union_regret": { - "n": 5, - "mean": 0.0018679847512695024, - "std": 0.0032548337047755126, - "min": 0.0, - "max": 0.007639684811918013 - } - }, - { - "model_tag": "alphaearth", - "label": "AlphaEarth", - "scope": "global", - "scope_label": "\\(\\Omega=\\)global", - "n": 5, - "seeds": [ - 1, - 7, - 42, - 99, - 123 - ], - "paths": [ - "", - "", - "", - "", - "" - ], - "exact_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - }, - "tolerated_regret": { - "n": 5, - "mean": 0.1722171037486726, - "std": 0.08849214830495522, - "min": 0.050655329891645895, - "max": 0.272468682523694 - }, - "union_regret": { - "n": 5, - "mean": 0.1722171037486726, - "std": 0.08849214830495522, - "min": 0.050655329891645895, - "max": 0.272468682523694 - } - }, - { - "model_tag": "alphaearth", - "label": "AlphaEarth", - "scope": "top5", - "scope_label": "\\(\\Omega=\\)top 5\\%", - "n": 5, - "seeds": [ - 1, - 7, - 42, - 99, - 123 - ], - "paths": [ - "", - "", - "", - "", - "" - ], - "exact_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - }, - "tolerated_regret": { - "n": 5, - "mean": 0.06384618090125256, - "std": 0.04965276403138872, - "min": 0.0, - "max": 0.1365277562230962 - }, - "union_regret": { - "n": 5, - "mean": 0.06384618090125256, - "std": 0.04965276403138872, - "min": 0.0, - "max": 0.1365277562230962 - } - }, - { - "model_tag": "alphaearth", - "label": "AlphaEarth", - "scope": "top10", - "scope_label": "\\(\\Omega=\\)top 10\\%", - "n": 5, - "seeds": [ - 1, - 7, - 42, - 99, - 123 - ], - "paths": [ - "", - "", - "", - "", - "" - ], - "exact_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - }, - "tolerated_regret": { - "n": 5, - "mean": 0.06573776411084173, - "std": 0.06897015340160571, - "min": 0.0, - "max": 0.1615566566666954 - }, - "union_regret": { - "n": 5, - "mean": 0.06573776411084173, - "std": 0.06897015340160571, - "min": 0.0, - "max": 0.1615566566666954 - } - }, - { - "model_tag": "alphaearth", - "label": "AlphaEarth", - "scope": "top20", - "scope_label": "\\(\\Omega=\\)top 20\\%", - "n": 5, - "seeds": [ - 1, - 7, - 42, - 99, - 123 - ], - "paths": [ - "", - "", - "", - "", - "" - ], - "exact_regret": { - "n": 5, - "mean": 0.0, - "std": 0.0, - "min": 0.0, - "max": 0.0 - }, - "tolerated_regret": { - "n": 5, - "mean": 0.038803552655092256, - "std": 0.0594825313313219, - "min": 0.0, - "max": 0.13482302181308625 - }, - "union_regret": { - "n": 5, - "mean": 0.038803552655092256, - "std": 0.0594825313313219, - "min": 0.0, - "max": 0.13482302181308625 - } - } - ], - "issues": [], - "notes": [ - "FireWx-FM ref. global exact_regret has true zero displayed std", - "FireWx-FM ref. top5 exact_regret has true zero displayed std", - "FireWx-FM ref. top10 exact_regret has true zero displayed std", - "FireWx-FM ref. top20 exact_regret has true zero displayed std", - "Prithvi-WxC global exact_regret has true zero displayed std", - "Prithvi-WxC global tolerated_regret has true zero displayed std", - "Prithvi-WxC global union_regret has true zero displayed std", - "Prithvi-WxC top5 exact_regret has true zero displayed std", - "Prithvi-WxC top5 tolerated_regret has true zero displayed std", - "Prithvi-WxC top5 union_regret has true zero displayed std", - "Prithvi-WxC top10 exact_regret has true zero displayed std", - "Prithvi-WxC top10 tolerated_regret has true zero displayed std", - "Prithvi-WxC top10 union_regret has true zero displayed std", - "Prithvi-WxC top20 exact_regret has true zero displayed std", - "Prithvi-WxC top20 tolerated_regret has true zero displayed std", - "Prithvi-WxC top20 union_regret has true zero displayed std", - "ClimaX top5 exact_regret has true zero displayed std", - "ClimaX top5 tolerated_regret has true zero displayed std", - "ClimaX top5 union_regret has true zero displayed std", - "ClimaX top10 exact_regret has true zero displayed std", - "ClimaX top10 tolerated_regret has true zero displayed std", - "ClimaX top10 union_regret has true zero displayed std", - "ClimaX top20 exact_regret has true zero displayed std", - "ClimaX top20 tolerated_regret has true zero displayed std", - "ClimaX top20 union_regret has true zero displayed std", - "StormCast global exact_regret has true zero displayed std", - "StormCast global tolerated_regret has true zero displayed std", - "StormCast global union_regret has true zero displayed std", - "StormCast top5 exact_regret has true zero displayed std", - "StormCast top5 tolerated_regret has true zero displayed std", - "StormCast top5 union_regret has true zero displayed std", - "StormCast top10 exact_regret has true zero displayed std", - "StormCast top10 tolerated_regret has true zero displayed std", - "StormCast top10 union_regret has true zero displayed std", - "StormCast top20 exact_regret has true zero displayed std", - "StormCast top20 tolerated_regret has true zero displayed std", - "StormCast top20 union_regret has true zero displayed std", - "DLWP global exact_regret has true zero displayed std", - "DLWP global tolerated_regret has true zero displayed std", - "DLWP global union_regret has true zero displayed std", - "FCN global exact_regret has true zero displayed std", - "FCN global tolerated_regret has true zero displayed std", - "FCN global union_regret has true zero displayed std", - "FengWu global exact_regret has true zero displayed std", - "FengWu global tolerated_regret has true zero displayed std", - "FengWu global union_regret has true zero displayed std", - "FuXi global exact_regret has true zero displayed std", - "FuXi global tolerated_regret has true zero displayed std", - "FuXi global union_regret has true zero displayed std", - "FuXi top20 exact_regret has true zero displayed std", - "Pangu-Weather global exact_regret has true zero displayed std", - "Pangu-Weather global tolerated_regret has true zero displayed std", - "Pangu-Weather global union_regret has true zero displayed std", - "AlphaEarth global exact_regret has true zero displayed std", - "AlphaEarth top5 exact_regret has true zero displayed std", - "AlphaEarth top10 exact_regret has true zero displayed std", - "AlphaEarth top20 exact_regret has true zero displayed std" - ] -} diff --git a/artifacts/results/selection_regret_summary.csv b/artifacts/results/selection_regret_summary.csv deleted file mode 100644 index ea8a02982a8c66171b1344716be24df46cd272c8..0000000000000000000000000000000000000000 --- a/artifacts/results/selection_regret_summary.csv +++ /dev/null @@ -1,25 +0,0 @@ -model_tag,label,scope,n,seeds,exact_regret_mean,exact_regret_std,tolerated_regret_mean,tolerated_regret_std,union_regret_mean,union_regret_std -reference,Reference,global,5,1 7 42 99 123,0.0,0.0,0.08783024981138902,0.09670495645481135,0.08783024981138902,0.09670495645481135 -reference,Reference,fire_prone,5,1 7 42 99 123,0.0,0.0,0.03402707057672223,0.032044658643147844,0.03402707057672223,0.032044658643147844 -prithvi_wxc,Prithvi-WxC,global,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0 -prithvi_wxc,Prithvi-WxC,fire_prone,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0 -aurora,Aurora,global,5,1 7 42 99 123,0.00020004882767231798,0.00026703384456332115,0.09851983041506818,0.1298781980037557,0.09851983041506818,0.1298781980037557 -aurora,Aurora,fire_prone,5,1 7 42 99 123,0.008202508825959588,0.01834136732088763,0.14391889430974364,0.32121904665016227,0.14391889430974364,0.32121904665016227 -climax,ClimaX,global,5,1 7 42 99 123,3.0287686240700486e-06,4.147312242167625e-06,0.0012959969982639485,0.0017746169760203706,0.0012959969982639485,0.0017746169760203706 -climax,ClimaX,fire_prone,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0 -stormcast,StormCast,global,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0 -stormcast,StormCast,fire_prone,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0 -pangu_weather,Pangu-Weather,global,5,1 7 42 99 123,0.00013033979247265275,0.0002685372203690466,0.048806713097574374,0.10733308684741971,0.048806713097574374,0.10733308684741971 -pangu_weather,Pangu-Weather,fire_prone,5,1 7 42 99 123,0.027875386332505546,0.02348779386900393,0.43111948243387105,0.39355644251497235,0.43111948243387105,0.39355644251497235 -dlwp,DLWP,global,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0 -dlwp,DLWP,fire_prone,5,1 7 42 99 123,0.0007702319787454587,0.0010995336594539604,0.043265915053601556,0.04332331365579739,0.043265915053601556,0.04332331365579739 -fcn,FCN,global,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0 -fcn,FCN,fire_prone,5,1 7 42 99 123,5.960229415004348e-06,1.3327478133443526e-05,0.011679805987441694,0.019872372458657642,0.011679805987441694,0.019872372458657642 -fengwu,FengWu,global,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0 -fengwu,FengWu,fire_prone,5,1 7 42 99 123,0.0006908222234409067,0.0011910586589384115,0.005222389249812243,0.0062394095558402415,0.005222389249812243,0.0062394095558402415 -fuxi,FuXi,global,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0 -fuxi,FuXi,fire_prone,5,1 7 42 99 123,0.0,0.0,0.0010839188523199318,0.0017288780545672386,0.0010839188523199318,0.0017288780545672386 -pangu6,Pangu-Weather,global,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0 -pangu6,Pangu-Weather,fire_prone,5,1 7 42 99 123,0.0007280423771922354,0.001178746460551365,0.0018491271881979853,0.0032630386057089294,0.0018491271881979853,0.0032630386057089294 -alphaearth,AlphaEarth,global,5,1 7 42 99 123,0.0,0.0,0.1722171037486726,0.08849214830495522,0.1722171037486726,0.08849214830495522 -alphaearth,AlphaEarth,fire_prone,5,1 7 42 99 123,0.0,0.0,0.038803552655092256,0.0594825313313219,0.038803552655092256,0.0594825313313219 diff --git a/artifacts/results/supporting_table4_event_repaired_compsci_20260902_summary_clean.md b/artifacts/results/supporting_table4_event_repaired_compsci_20260902_summary_clean.md new file mode 100644 index 0000000000000000000000000000000000000000..2428d43c12d8c71d91cafb7777cf2fb46d66135b --- /dev/null +++ b/artifacts/results/supporting_table4_event_repaired_compsci_20260902_summary_clean.md @@ -0,0 +1,36 @@ +# Forced Mean/Std Gap-Fill Summary + +## table2_headcontrol + + +## table3_occupancy + + +## table3_spread + + +## table4_final_area + +- AlphaEarth: log_rmse 2.46 +/- 0.00 (n=5); log_mae 2.10 +/- 0.00 (n=5); log_spearman -0.30 +/- 0.00 (n=5) +- Aurora: log_rmse 1.69 +/- 0.00 (n=5); log_mae 1.51 +/- 0.00 (n=5); log_spearman -0.12 +/- 0.00 (n=5) +- ClimaX: log_rmse 1.69 +/- 0.00 (n=5); log_mae 1.51 +/- 0.00 (n=5); log_spearman -0.12 +/- 0.00 (n=5) +- Pangu24: log_rmse 1.69 +/- 0.00 (n=5); log_mae 1.51 +/- 0.00 (n=5); log_spearman -0.12 +/- 0.00 (n=5) +- Prithvi-WxC: log_rmse 1.69 +/- 0.00 (n=5); log_mae 1.51 +/- 0.00 (n=5); log_spearman -0.12 +/- 0.00 (n=5) +- Reference: log_rmse 1.17 +/- 0.01 (n=5); log_mae 1.05 +/- 0.01 (n=5); log_spearman 0.59 +/- 0.03 (n=5) +- StormCast: log_rmse 1.69 +/- 0.00 (n=5); log_mae 1.51 +/- 0.00 (n=5); log_spearman -0.12 +/- 0.00 (n=5) + +## table4_analog + +- AlphaEarth: ndcg_at_10 0.50 +/- 0.00 (n=5); log_rmse 1.20 +/- 0.00 (n=5); log_mae 1.02 +/- 0.00 (n=5) +- Aurora: ndcg_at_10 0.41 +/- 0.00 (n=5); log_rmse 1.41 +/- 0.00 (n=5); log_mae 1.32 +/- 0.00 (n=5) +- ClimaX: ndcg_at_10 0.41 +/- 0.00 (n=5); log_rmse 1.41 +/- 0.00 (n=5); log_mae 1.32 +/- 0.00 (n=5) +- Pangu24: ndcg_at_10 0.41 +/- 0.00 (n=5); log_rmse 1.41 +/- 0.00 (n=5); log_mae 1.32 +/- 0.00 (n=5) +- Prithvi-WxC: ndcg_at_10 0.41 +/- 0.00 (n=5); log_rmse 1.41 +/- 0.00 (n=5); log_mae 1.32 +/- 0.00 (n=5) +- StormCast: ndcg_at_10 0.41 +/- 0.00 (n=5); log_rmse 1.41 +/- 0.00 (n=5); log_mae 1.32 +/- 0.00 (n=5) +- reference_strict: ndcg_at_10 0.52 +/- 0.00 (n=5); log_rmse 1.18 +/- 0.00 (n=5); log_mae 0.97 +/- 0.00 (n=5) + +## table4_smoke + + +## table4_heat + diff --git a/artifacts/results/supporting_table4_smoke_heat_compsci_20260902_summary_clean.md b/artifacts/results/supporting_table4_smoke_heat_compsci_20260902_summary_clean.md new file mode 100644 index 0000000000000000000000000000000000000000..3fc0e42d941df4b2062798591b50cdd34d427de4 --- /dev/null +++ b/artifacts/results/supporting_table4_smoke_heat_compsci_20260902_summary_clean.md @@ -0,0 +1,29 @@ +# Forced Mean/Std Gap-Fill Summary + +## table2_headcontrol + + +## table3_occupancy + + +## table3_spread + + +## table4_final_area + + +## table4_analog + + +## table4_smoke + +- AlphaEarth: rmse 4.49 +/- 0.00 (n=5); mae 2.40 +/- 0.00 (n=5); pearson_r 0.63 +/- 0.00 (n=5) +- Aurora: rmse 5.93 +/- 0.00 (n=5); mae 3.73 +/- 0.00 (n=5); pearson_r 0.02 +/- 0.00 (n=5) +- ClimaX: rmse 5.93 +/- 0.00 (n=5); mae 3.73 +/- 0.00 (n=5); pearson_r 0.00 +/- 0.00 (n=5) +- Pangu24: rmse 5.95 +/- 0.00 (n=5); mae 3.75 +/- 0.00 (n=5); pearson_r 0.04 +/- 0.00 (n=5) +- Reference: rmse 4.47 +/- 0.01 (n=5); mae 2.41 +/- 0.00 (n=5); pearson_r 0.64 +/- 0.00 (n=5) +- StormCast: rmse 6.02 +/- 0.00 (n=5); mae 3.82 +/- 0.00 (n=5); pearson_r 0.02 +/- 0.00 (n=5) + +## table4_heat + +- Reference: rmse_c 0.22 +/- 0.00 (n=5); mae_c 0.18 +/- 0.00 (n=5); pearson_r 1.00 +/- 0.00 (n=5) diff --git a/artifacts/results/supporting_table4_smoke_heat_fill_compsci_20260902_summary_clean.md b/artifacts/results/supporting_table4_smoke_heat_fill_compsci_20260902_summary_clean.md new file mode 100644 index 0000000000000000000000000000000000000000..6730f18e8125d12ec1973701750e2dafd7fa936e --- /dev/null +++ b/artifacts/results/supporting_table4_smoke_heat_fill_compsci_20260902_summary_clean.md @@ -0,0 +1,24 @@ +# Forced Mean/Std Gap-Fill Summary + +## table2_headcontrol + + +## table3_occupancy + + +## table3_spread + + +## table4_final_area + + +## table4_analog + + +## table4_smoke + +- Prithvi-WxC: rmse 5.93 +/- 0.00 (n=5); mae 3.73 +/- 0.00 (n=5); pearson_r 0.02 +/- 0.00 (n=5) + +## table4_heat + +- AlphaEarth: rmse_c 0.22 +/- 0.00 (n=5); mae_c 0.18 +/- 0.00 (n=5); pearson_r 1.00 +/- 0.00 (n=5) diff --git a/artifacts/results/tune4_repaired_eventfm_summary.md b/artifacts/results/tune4_repaired_eventfm_summary.md new file mode 100644 index 0000000000000000000000000000000000000000..063e27a32ac4efedc30af1f489b98182f614420d --- /dev/null +++ b/artifacts/results/tune4_repaired_eventfm_summary.md @@ -0,0 +1,32 @@ +# Forced Mean/Std Gap-Fill Summary + +## table2_headcontrol + + +## table3_occupancy + + +## table3_spread + + +## table4_final_area + +- Aurora: log_rmse 1.69 +/- 0.00 (n=5); log_mae 1.51 +/- 0.00 (n=5); log_spearman -0.12 +/- 0.00 (n=5) +- ClimaX: log_rmse 1.69 +/- 0.00 (n=5); log_mae 1.51 +/- 0.00 (n=5); log_spearman -0.12 +/- 0.00 (n=5) +- Pangu24: log_rmse 1.69 +/- 0.00 (n=5); log_mae 1.51 +/- 0.00 (n=5); log_spearman -0.12 +/- 0.00 (n=5) +- Prithvi-WxC: log_rmse 1.69 +/- 0.00 (n=5); log_mae 1.51 +/- 0.00 (n=5); log_spearman -0.12 +/- 0.00 (n=5) +- StormCast: log_rmse 1.69 +/- 0.00 (n=5); log_mae 1.51 +/- 0.00 (n=5); log_spearman -0.12 +/- 0.00 (n=5) + +## table4_analog + +- Aurora: ndcg_at_10 0.41 +/- 0.00 (n=5); log_rmse 1.41 +/- 0.00 (n=5); log_mae 1.32 +/- 0.00 (n=5) +- ClimaX: ndcg_at_10 0.41 +/- 0.00 (n=5); log_rmse 1.41 +/- 0.00 (n=5); log_mae 1.32 +/- 0.00 (n=5) +- Pangu24: ndcg_at_10 0.41 +/- 0.00 (n=5); log_rmse 1.41 +/- 0.00 (n=5); log_mae 1.32 +/- 0.00 (n=5) +- Prithvi-WxC: ndcg_at_10 0.41 +/- 0.00 (n=5); log_rmse 1.41 +/- 0.00 (n=5); log_mae 1.32 +/- 0.00 (n=5) +- StormCast: ndcg_at_10 0.41 +/- 0.00 (n=5); log_rmse 1.41 +/- 0.00 (n=5); log_mae 1.32 +/- 0.00 (n=5) + +## table4_smoke + + +## table4_heat + diff --git a/artifacts/results/tune4_throttled_summary.md b/artifacts/results/tune4_throttled_summary.md new file mode 100644 index 0000000000000000000000000000000000000000..b70277e06dadb1f047b2d9adfb6c5271374c8bb7 --- /dev/null +++ b/artifacts/results/tune4_throttled_summary.md @@ -0,0 +1,35 @@ +# Forced Mean/Std Gap-Fill Summary + +## table2_headcontrol + + +## table3_occupancy + + +## table3_spread + + +## table4_final_area + +- AlphaEarth: log_rmse 2.46 +/- 0.00 (n=5); log_mae 2.10 +/- 0.00 (n=5); log_spearman -0.30 +/- 0.00 (n=5) +- Reference: log_rmse 1.17 +/- 0.01 (n=5); log_mae 1.05 +/- 0.01 (n=5); log_spearman 0.59 +/- 0.03 (n=5) + +## table4_analog + +- AlphaEarth: ndcg_at_10 0.50 +/- 0.00 (n=5); log_rmse 1.20 +/- 0.00 (n=5); log_mae 1.02 +/- 0.00 (n=5) +- reference_strict: ndcg_at_10 0.52 +/- 0.00 (n=5); log_rmse 1.18 +/- 0.00 (n=5); log_mae 0.97 +/- 0.00 (n=5) + +## table4_smoke + +- AlphaEarth: rmse 4.49 +/- 0.00 (n=5); mae 2.40 +/- 0.00 (n=5); pearson_r 0.63 +/- 0.00 (n=5) +- Aurora: rmse 5.93 +/- 0.00 (n=5); mae 3.73 +/- 0.00 (n=5); pearson_r 0.02 +/- 0.00 (n=5) +- ClimaX: rmse 5.93 +/- 0.00 (n=5); mae 3.73 +/- 0.00 (n=5); pearson_r 0.00 +/- 0.00 (n=5) +- Pangu24: rmse 5.95 +/- 0.00 (n=5); mae 3.75 +/- 0.00 (n=5); pearson_r 0.04 +/- 0.00 (n=5) +- Prithvi-WxC: rmse 5.93 +/- 0.00 (n=5); mae 3.73 +/- 0.00 (n=5); pearson_r 0.02 +/- 0.00 (n=5) +- Reference: rmse 4.47 +/- 0.01 (n=5); mae 2.41 +/- 0.00 (n=5); pearson_r 0.64 +/- 0.00 (n=5) +- StormCast: rmse 6.02 +/- 0.00 (n=5); mae 3.82 +/- 0.00 (n=5); pearson_r 0.02 +/- 0.00 (n=5) + +## table4_heat + +- AlphaEarth: rmse_c 0.22 +/- 0.00 (n=5); mae_c 0.18 +/- 0.00 (n=5); pearson_r 1.00 +/- 0.00 (n=5) +- Reference: rmse_c 0.22 +/- 0.00 (n=5); mae_c 0.18 +/- 0.00 (n=5); pearson_r 1.00 +/- 0.00 (n=5) diff --git a/assets/overview_final.png b/assets/overview_final.png deleted file mode 100644 index 9c0d825a1c72a4a8896877940219cd4243cf9a3c..0000000000000000000000000000000000000000 --- a/assets/overview_final.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:6db4f4aff90da8709edff97e4782aee9b4c5e8feefec7c7431a4ec8787cfe57c -size 883514 diff --git a/assets/primary_rank_change_final.png b/assets/primary_rank_change_final.png deleted file mode 100644 index 7bef671900c96d84c0822a9eb1b497bd0ec6b966..0000000000000000000000000000000000000000 --- a/assets/primary_rank_change_final.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:ca11c75c03078a9be26421b527ab5a49f5fc43ce8e5edd7da14af120a247b67c -size 863809 diff --git a/assets/release_contents.svg b/assets/release_contents.svg deleted file mode 100644 index bdfb61b65b8e834698b47948ac3e189f57d92875..0000000000000000000000000000000000000000 --- a/assets/release_contents.svg +++ /dev/null @@ -1,33 +0,0 @@ - - -Release contents -A model release with paper artifacts as reproducibility support, not a raw-data mirror. - - - -Model -Compact U-Net backbone with -occupancy and auxiliary -spatial-support heads. -models/wildfire_fm/ - - -Checkpoints -Five seeded PyTorch weights with -SHA-256 manifest. -models/wildfire_fm/checkpoints/ - - -Data Boundary -Raw data are not redistributed; -source entry points are -documented. -data_sources/ - - -Paper Artifacts -Final manuscript, figures, tables, -and compact summaries for -reproducibility. -paper_outputs/ + artifacts/ - \ No newline at end of file diff --git a/assets/selection_regret_final.png b/assets/selection_regret_final.png deleted file mode 100644 index d1ce6c446eab7163228d2b383ca315c659d894ca..0000000000000000000000000000000000000000 --- a/assets/selection_regret_final.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:5552fb6cca6a0a683592e724b4bd562f923cf99c04e2abdb846546b1d67aecc4 -size 1780812 diff --git a/assets/supporting_rank_map_final.png b/assets/supporting_rank_map_final.png deleted file mode 100644 index 1dd9e9ca078c3f27a359f20f68f197d05b3c1c73..0000000000000000000000000000000000000000 --- a/assets/supporting_rank_map_final.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:34807e65ca71365a26a3b74cae70e6b40ae6f2151110e12c53e0efa9f8b726aa -size 696701 diff --git a/assets/wildfire_fm_model_card.svg b/assets/wildfire_fm_model_card.svg deleted file mode 100644 index d1a3676f89c91b36bf1256437941afbd721799fb..0000000000000000000000000000000000000000 --- a/assets/wildfire_fm_model_card.svg +++ /dev/null @@ -1,48 +0,0 @@ - - - -WildFIRE-FM -Wildfire-specialized reference backbone for fixed-contract wildfire evaluation - - -16-channel input -weather + masks + static context - -5 checkpoints -seeds 1, 7, 42, 99, 123 - -raw data not bundled -source-data access documented -Best values under final fixed-contract task comparisons - -Occupancy union F1 -60.15% -ClimaX -± 7.59 - -Spread spatial F1 -80.97% -WildFIRE-FM -± 2.02 - -Burned-area log-RMSE -1.166 -WildFIRE-FM -lower better - -Analog nDCG@10 -0.510 -WildFIRE-FM -higher better - -Smoke PM2.5 RMSE -4.440 -AlphaEarth -lower better - -Extreme-heat RMSE-C -0.218 -WildFIRE-FM -lower better -Each value is the best mean for that task contract in the final paper tables; winner labels identify the backbone attaining it. - diff --git a/data_downloader/README.md b/data_downloader/README.md deleted file mode 100644 index 5a6285b4d9a92d2c7d670be4ed6ca8f4cc37fead..0000000000000000000000000000000000000000 --- a/data_downloader/README.md +++ /dev/null @@ -1,133 +0,0 @@ -# Data Downloader - -This directory contains a lightweight, client-facing downloader for data sources -used by FireWx-FM and planned general hazard-model work. It is intentionally a -raw-data downloader, not a model-ready dataloader. Dataset adapters and aligned -caches should be built after raw files are obtained under each provider's terms. - -## Design - -The downloader follows the older `Pyhazard_data_downloader` API pattern: - -```python -from data_downloader import DownloadRequest, download_data, list_datasets -``` - -Each dataset declares: - -- `access`: whether it is public direct download, ArcGIS REST, auth-required, or instructions-only. -- `time_mode`: how time selection works. -- `aoi_mode`: whether bounding-box selection is native, post-processing only, or unsupported. - -This avoids pretending that every source supports the same query semantics. - -## Quick Start - -List datasets: - -```bash -python3 -m data_downloader.cli --list -``` - -Dry-run a small client request: - -```bash -python3 -m data_downloader.cli \ - --datasets aqs_pm25 hms_smoke ibtracs \ - --output-root ./downloads_demo \ - --start-year 2024 \ - --end-year 2024 \ - --dry-run \ - --max-files 3 \ - --report ./downloads_demo/report.json -``` - -For datasets that use a bounding box, pass negative longitudes with the equals -form so the shell/argparse does not parse them as flags: - -```bash -python3 -m data_downloader.cli \ - --datasets firms_area \ - --output-root ./downloads_firms \ - --start-date 2024-06-01 \ - --end-date 2024-06-02 \ - --bbox=-125,32,-114,42 \ - --dry-run -``` - -Python API: - -```python -from data_downloader import DownloadRequest, download_data - -request = DownloadRequest( - datasets=["aqs_pm25", "hms_smoke"], - output_root="./downloads", - years=(2024, 2025), - dry_run=False, -) -results = download_data(request) -``` - -Download HRRR files from the public NOAA AWS archive: - -```bash -python3 data_downloader/hrrr_downloader.py \ - --start-date 2024-06-01 \ - --end-date 2024-06-02 \ - --hours 00,06,12,18 \ - --forecast-hours 00 \ - --output-root ./downloads_hrrr \ - --include-idx \ - --dry-run -``` - -Remove `--dry-run` to download the files. The script writes -`hrrr_download_manifest.json` with every URL, output path, and download status. -The HRRR downloader retrieves native HRRR files; county, sub-county, or any -other custom spatial granularity should be produced later by the serving or -preprocessing layer using user-provided boundaries. - -For a small real-download smoke test, download only the HRRR index sidecar: - -```bash -python3 data_downloader/hrrr_downloader.py \ - --start-date 2024-06-01 \ - --end-date 2024-06-01 \ - --hours 00 \ - --forecast-hours 00 \ - --idx-only \ - --max-files 1 \ - --output-root ./downloads_hrrr_idx_test -``` - -## Implemented Direct Downloads - -These sources are implemented as direct public downloads: - -- `aqs_pm25`: EPA AirData hourly PM2.5 files and site/monitor metadata. -- `hms_smoke`: NOAA HMS annual smoke-polygon and fire-point bundles. -- `ibtracs`: NOAA/NCEI IBTrACS v04r01 CSV tracks and documentation. -- `hurdat2`: NOAA/NHC HURDAT2 latest best-track text files and format docs. -- `usdm`: U.S. Drought Monitor weekly GIS shapefile archive. -- `hrrr_fireseason`: NOAA HRRR public S3 files for selected years, fire-season dates, hours, and product. -- `wfigs_current`: current WFIGS perimeter snapshot through ArcGIS REST. -- `landfire_static`: known LF2024 CONUS `FBFM40` and `CC` static products. - -## Credential or Terms-Limited Sources - -These are intentionally not silently downloaded: - -- `firms_area`: requires a NASA FIRMS `MAP_KEY`; set `FIRMS_MAP_KEY` or pass it in `extra_options`. -- `merra2`: requires NASA Earthdata/GES DISC authenticated access. -- `landscan`: governed by ORNL/LandScan provider-specific access terms. -- `wrc_housing`: provider links can redirect/change; pass an explicit URL or use the provider portal. -- `mtbs_perimeters`: heavy ArcGIS chunk pulls are available in internal Slurm scripts, but this public client records instructions first. - -## Notes for Clients - -- Raw source files are not redistributed by this repository. -- The downloader writes `download_manifest.json` under each dataset output folder. -- Use `--dry-run` and `--max-files` before large requests. -- For large HRRR, GOES, MERRA-2, NWM, or full MTBS/RAVG pulls, run downloads on compute/storage infrastructure rather than a login node. -- `hrrr_downloader.py` is the direct HRRR client for custom date/hour/forecast-hour requests. diff --git a/data_sources/DATA_SOURCES.md b/data_sources/DATA_SOURCES.md index afe045f9dd97572dc6bd642a47a5417e18387d1f..c542147dbeb5ad512c9111aec4b9473acf782540 100644 --- a/data_sources/DATA_SOURCES.md +++ b/data_sources/DATA_SOURCES.md @@ -1,72 +1,55 @@ # Data Sources -This repository documents source-data access for the resources used by the -paper. The table below records each resource, its role in the experiments, and -public access entry points. Users should obtain each source under its own terms. +This release documents the source datasets needed for the FireWx-FM active-fire occupancy model. Raw source data are not redistributed. Users should obtain each resource from the original provider and follow the provider terms. -| Source | Role in paper | Public access entry point | -|---|---|---| -| NOAA High-Resolution Rapid Refresh (HRRR) | Dynamic weather fields for the California regional gridded occupancy inputs. | NOAA/NCEI HRRR product page: ; AWS Open Data archive: . | -| NASA FIRMS active-fire detections | Active-fire detections used to derive gridded occupancy labels. | FIRMS download and API services: and . | -| LANDFIRE 40 Fire Behavior Fuel Models | Static fuel layer used in the FireWx-FM gridded input. | LANDFIRE data access portal: . | -| LANDFIRE Forest Canopy Cover | Static canopy layer used in the FireWx-FM gridded input. | LANDFIRE data access portal: . | -| Wildfire Risk to Communities housing-unit density | Static exposure layer used in the FireWx-FM gridded input. | Wildfire Risk to Communities data access: . | -| LandScan Global 2024 | Static population layer used in the FireWx-FM gridded input. | Oak Ridge National Laboratory LandScan access: . | -| WFIGS incident/perimeter attributes | Event-level incident metadata for supporting burned-area and analog tasks. | NIFC Open Data portal for WFIGS layers: . | -| MTBS burned area and burn severity | Event-scale burned-area and burn-severity records for supporting tasks. | MTBS data access and direct download pages: and . | -| Earth-FM/backbone sources | Frozen feature sources for transferred Earth-FM comparisons. | Original model providers and their terms. Examples include Hugging Face model cards, model-provider GitHub repositories, and provider-hosted model files. | - -## Pretrained Input Channels - -The released checkpoints expect a fixed 16-channel tensor in `[channel, y, x]` -order. Channel order matters. The pretrained 5 km, 12-hour-lead occupancy input -uses the following contract. A machine-readable copy is stored at -`models/wildfire_fm/input_channels.json`. - -No mean/std normalization, min/max scaling, or unit conversion is applied before -inference. HRRR fields remain in native units after reprojection to the model -grid. Invalid or missing values are zero-filled, with validity masks included as -channels 10 and 11. +## Source Inventory -| Channel | Name | Dataset/source | Level or selection | Units | Normalization | Role | -|---:|---|---|---|---|---|---| -| 0 | `t2m` | NOAA HRRR | 2 m above ground | K | none | 2 m temperature | -| 1 | `d2m` | NOAA HRRR | 2 m above ground | K | none | 2 m dew point | -| 2 | `u10` | NOAA HRRR | 10 m above ground | m/s | none | 10 m east-west wind component | -| 3 | `v10` | NOAA HRRR | 10 m above ground | m/s | none | 10 m north-south wind component | -| 4 | `cape` | NOAA HRRR | surface, instant | J/kg | none | Surface instantaneous CAPE | -| 5 | `sp` | NOAA HRRR | surface, instant | Pa | none | Surface pressure | -| 6 | `blh` | NOAA HRRR | surface diagnostic, instant | m | none | Boundary-layer height | -| 7 | `vis` | NOAA HRRR | surface diagnostic, instant | m | none | Visibility | -| 8 | `prate` | NOAA HRRR | surface, instant | kg m^-2 s^-1 | none | Precipitation rate | -| 9 | `tp` | NOAA HRRR | surface, accumulated | kg m^-2 | none | Accumulated precipitation | -| 10 | `firewx_valid` | Cache validity channel | input presence | binary mask | none | Cache-level dynamic input presence mask; 1.0 everywhere in this release | -| 11 | `static_valid` | Static reprojection mask | static layers | fraction | none | Fraction of static layers valid at the grid cell after reprojection | -| 12 | `fuel_fbfm40` | LANDFIRE FBFM40 | static layer | categorical code | none | Fire-behavior fuel model | -| 13 | `canopy_cover` | LANDFIRE CC | static layer | provider-native value | none | Canopy cover | -| 14 | `housing_density` | Wildfire Risk to Communities | static layer | provider-native value | none | Housing-unit density | -| 15 | `population` | LandScan Global 2024 | static layer | provider-native value | none | Population exposure | - -FIRMS detections are used to derive the occupancy target, not as an input -channel. WFIGS and MTBS are event-level resources for supporting burned-area -and analog tasks and are not part of the 16-channel pretrained occupancy input. - -The `cape` channel is the HRRR surface instantaneous CAPE field selected with -`typeOfLevel=surface` and `stepType=instant`. It is not the most-unstable CAPE -or a layer CAPE variable. - -The two validity channels have different definitions. `firewx_valid` is a -cache-level dynamic input presence mask; in the released California regional -cache it is 1.0 everywhere and is not a per-variable HRRR missing-data mask. -`static_valid` is the fraction of the four static layers that are valid after -reprojection at each grid cell, with possible values 0.0, 0.25, 0.5, 0.75, and -1.0. +| Source | Role | Access | +|---|---|---| +| NOAA High-Resolution Rapid Refresh (HRRR) | Dynamic weather input channels on the 5 km grid. | NOAA/NCEI product page: ; AWS Open Data archive: . | +| NASA FIRMS active-fire detections | Active-fire detections used to derive the occupancy target. | FIRMS download and API services: and . | +| LANDFIRE 40 Fire Behavior Fuel Models | Static fuel-model input channel. | LANDFIRE data portal: . | +| LANDFIRE canopy cover | Static canopy-cover input channel. | LANDFIRE data portal: . | +| Wildfire Risk to Communities housing-unit density | Static exposure input channel; zeroed in final no-exposure serving. | Wildfire Risk to Communities downloads: . | +| LandScan Global 2024 | Static population input channel; zeroed in final no-exposure serving. | ORNL LandScan access: . | +| USGS Fire Danger Forecast | Used only for the calibrated example output, not as a training input channel. | USGS Fire Danger Forecast viewer and downloads: . | + +## Input Channels + +The released checkpoint expects a fixed 16-channel tensor in `[channel, y, x]` order. + +| Channel | Name | Dataset/source | Level or selection | Units | Serving treatment | +|---:|---|---|---|---|---| +| 0 | `t2m` | NOAA HRRR | 2 m above ground | K | z-score normalized | +| 1 | `d2m` | NOAA HRRR | 2 m above ground | K | z-score normalized | +| 2 | `u10` | NOAA HRRR | 10 m above ground | m s-1 | z-score normalized | +| 3 | `v10` | NOAA HRRR | 10 m above ground | m s-1 | z-score normalized | +| 4 | `cape` | NOAA HRRR | 0-3000 m above-ground layer | J kg-1 | z-score normalized | +| 5 | `sp` | NOAA HRRR | surface | Pa | z-score normalized | +| 6 | `blh` | NOAA HRRR | boundary-layer diagnostic | m | z-score normalized | +| 7 | `vis` | NOAA HRRR | visibility diagnostic | m | z-score normalized | +| 8 | `prate` | NOAA HRRR | surface precipitation rate | kg m-2 s-1 | z-score normalized | +| 9 | `tp` | NOAA HRRR | accumulated precipitation | kg m-2 | z-score normalized | +| 10 | `dynamic_valid` | input builder | dynamic weather validity | 0-1 | passed through | +| 11 | `static_valid` | input builder | static-layer validity fraction | 0-1 | passed through | +| 12 | `fuel_fbfm40` | LANDFIRE | fire-behavior fuel model | category code | passed through | +| 13 | `canopy_cover` | LANDFIRE | canopy cover | percent | z-score normalized | +| 14 | `housing_density` | Wildfire Risk to Communities | static exposure | provider native | z-score normalized, then zeroed | +| 15 | `population` | LandScan Global 2024 | static exposure | persons | z-score normalized, then zeroed | + +FIRMS detections define the occupancy target and are not input channels. + +## Static Resampling + +| Static layer | Resampling | +|---|---| +| LANDFIRE fire-behavior fuel model | nearest | +| LANDFIRE canopy cover | nearest | +| Wildfire Risk to Communities housing density | bilinear | +| LandScan population | bilinear | ## Notes -- The paper places gridded resources on a projected 5 km EPSG:5070 grid. -- The bundled artifacts contain summary values for release checking. -- Full raw-data reruns require users to obtain each source independently and to - construct the intermediate grids/features described in the paper. -- Access mechanisms and licensing can change. The links above point to the - original providers. +- Native grid: Lower-48 CONUS, EPSG:5070, 5 km. +- Prediction target: 12-hour active-fire occupancy probability. +- County and sub-county products are post-inference aggregations of the 5 km probability grid. diff --git a/docs/artifact_map.md b/docs/artifact_map.md deleted file mode 100644 index d02814beaae9730c3e0c3226f043613cd1fbcf78..0000000000000000000000000000000000000000 --- a/docs/artifact_map.md +++ /dev/null @@ -1,33 +0,0 @@ -# Public Artifact Map - -This map describes the public Hugging Face release boundary. The Hub release -publishes model weights, model-loading code, compact numeric summaries, selected -figure artifacts, source-data notes, and the compiled paper PDF. Manuscript -source remains in the authors' paper workspace. - -## Included Public Artifacts - -| Area | Release files | Notes | -|---|---|---| -| Model code | `models/wildfire_fm/modeling_unet.py` | Compact U-Net used by the released checkpoints. | -| Checkpoint metadata | `models/wildfire_fm/checkpoint_manifest.json` | Lists five seeded checkpoint paths, SHA-256 hashes, and byte sizes. | -| Paper PDF | `paper/wildfire_fm_evaluation_contracts.pdf` | Compiled from the final local manuscript source. | -| Figure previews | `assets/*.png`, `assets/*.svg` | Hub-page visuals and final-paper figure previews. | -| Figure PDFs | `paper_outputs/figures/*.pdf` | Selected final-paper figures retained for visual reproducibility. | -| Numeric summaries | `artifacts/results/*.csv`, `artifacts/results/*.json` | Sanitized compact summaries; local machine paths removed. | -| Data notes | `data_sources/DATA_SOURCES.md` | Source roles and access entry points for provider-hosted data. | -| Raw rerun references | `experiments/` | Sanitized scripts/templates requiring user-provided data and paths. | - -## Manuscript Source Boundary - -The public paper artifact is the compiled PDF. Manuscript TeX, generated table -source, TikZ source, and BibTeX are managed outside the Hub model release. - -## Verification - -```bash -python3 scripts/reproduce_paper_outputs.py -``` - -This command checks public artifact hashes and audits the release hygiene for -local paths and manuscript-source boundaries. diff --git a/docs/huggingface_release_design.md b/docs/huggingface_release_design.md deleted file mode 100644 index de39f4f05c97b50702c0da72176de73731d87807..0000000000000000000000000000000000000000 --- a/docs/huggingface_release_design.md +++ /dev/null @@ -1,33 +0,0 @@ -# Hugging Face Release Design - -This release follows a model-card-first pattern used by geospatial and weather -foundation-model releases on the Hub. The public page starts with model purpose, -released weights, input expectations, a quick loading example, data boundaries, -and a compact evaluation snapshot. Paper artifacts are included as supporting -reproducibility material rather than being the main organizing principle. - -## Public Card Structure - -- `README.md` is the public model card. It contains YAML metadata, intended use, - checkpoint locations, quick loading code, data-source boundaries, limitations, - and citation text. -- `assets/` contains lightweight visuals for the Hub page plus PNG previews of - final-paper figure previews. -- `models/wildfire_fm/` contains model code, manifests, and checkpoint metadata. -- `paper_outputs/` stores selected final-paper figure PDFs used for public - visual checking. -- `artifacts/results/` stores sanitized compact CSV/JSON summaries that can be public. -- `data_sources/` documents external data resources without redistributing them. -- `experiments/` contains raw-rerun reference scripts and Slurm templates. - -## Data Boundary - -The repository is a model release with reproducibility artifacts. Full raw-data -reruns require separately obtained source data, local feature caches, and -cluster-specific paths. - -## Manuscript Boundary - -The Hub model release publishes the compiled paper PDF and keeps manuscript -source management in the authors' paper workspace. The public paper link can be -updated after the arXiv version is finalized. diff --git a/docs/hugh_handoff_status.md b/docs/hugh_handoff_status.md deleted file mode 100644 index 4c2b716b92ae24cd970ae79705814ffc16c6597e..0000000000000000000000000000000000000000 --- a/docs/hugh_handoff_status.md +++ /dev/null @@ -1,49 +0,0 @@ -# Integration Handoff Status - -This note answers the implementation questions raised during web/inference -integration and points to the public files that should be used by downstream -clients. It is intentionally scoped to the released FireWx-FM repository. - -## What is fully specified now - -| Question | Status | Where to look | -|---|---|---| -| Which input channel maps to which dataset? | Specified in human-readable and machine-readable form. Channel order matters. | `data_sources/DATA_SOURCES.md`, `models/wildfire_fm/input_channels.json` | -| Is value normalization applied? | Released California checkpoints use native source units with no mean/std or min/max normalization. The training pipeline now supports optional train-split z-score normalization for retraining. | `models/wildfire_fm/input_channels.json`, `training/train_cold_tiled_mainline.py` | -| Which CAPE variable is used? | HRRR surface instantaneous CAPE, selected with `typeOfLevel=surface` and `stepType=instant`; not most-unstable or layer CAPE. | `models/wildfire_fm/input_channels.json`, `data_sources/DATA_SOURCES.md` | -| What do the validity masks mean? | `firewx_valid` is a cache-level dynamic input-presence mask and is 1.0 everywhere in the released California cache. `static_valid` is the fraction of four static layers valid after reprojection. | `models/wildfire_fm/README.md`, `models/wildfire_fm/input_channels.json` | -| How are static 30 m rasters resampled to the 5 km grid? | LANDFIRE fuel and canopy use nearest-neighbor resampling; WRC housing density and LandScan population use bilinear resampling. | `training/configs/stage1_cache_regional_hrrr_ca_5km_l12_template.json`, `training/build_phase1_cache_regional_hrrr.py` | -| Can outputs be shown at county or sub-county granularity? | Yes, by aggregating native 5 km grid probabilities after inference. This does not require changing the checkpoint. | `spatial_serving/README.md`, `spatial_serving/grid_to_polygons.py` | -| Can HRRR be downloaded automatically? | Yes. A direct NOAA HRRR public-archive downloader is included and has been smoke-tested with an `.idx` sidecar download. | `data_downloader/hrrr_downloader.py`, `data_downloader/README.md` | -| Is the released checkpoint nationwide-trained? | No. The public checkpoint is California regional. CONUS cache and training templates are included, but no CONUS checkpoint is released yet. | `models/wildfire_fm/training_scope.json`, `training/NATIONWIDE_RETRAINING.md` | - -## Current model-serving boundary - -The released checkpoints are the paper-aligned California regional checkpoints. -They are valid for reproducing the reported regional model behavior and for -inspecting the tensor contract. They should not be described as nationwide -weights. - -For gridded inference, use full-map inference when possible. If tiled inference -is required, use overlapping tiles with halo cropping or blending. Independent -non-overlapping 32-by-32 tiles are not the intended serving mode because they can -introduce tile-position and edge artifacts. The helper -`models/wildfire_fm/tiled_inference.py` implements the overlap/halo pattern. - -For CONUS or client-facing deployment, the current path is: - -1. Build a local CONUS cache from provider-hosted HRRR, FIRMS, LANDFIRE, WRC, - and LandScan inputs. -2. Retrain using the CONUS template with random-containing positive tile - placement and optional continuous-channel z-score normalization. -3. Serve native-grid probabilities with full-map or overlap/halo inference. -4. Aggregate native-grid probabilities to county or sub-county polygons using - the spatial serving adapter. - -## What is not being uploaded as solved - -A preliminary CONUS normalized/jittered retraining run completed, but seed-level -behavior was not stable enough to release as a corrected nationwide checkpoint. -For that reason, the repository exposes the corrected training and serving -hooks, but it does not publish a new CONUS checkpoint or claim that the CONUS -serving behavior is fully resolved. diff --git a/docs/inference_contract.md b/docs/inference_contract.md new file mode 100644 index 0000000000000000000000000000000000000000..44cae8f1e94d113b4ceaa62f35d61a07c5cb85d7 --- /dev/null +++ b/docs/inference_contract.md @@ -0,0 +1,76 @@ +# FireWx-FM Inference Contract + +This document defines the final serving contract for the released FireWx-FM CONUS model. + +## Native Grid + +- Domain: Lower-48 CONUS. +- CRS: EPSG:5070. +- Native resolution: 5 km grid cells. +- Prediction target: 12-hour active-fire occupancy probability. +- Tensor order: `[channel, y, x]`. +- Input shape: `[16, H, W]`. + +The model remains gridded. County or sub-county outputs are produced by post-inference aggregation from the native 5 km probability grid. + +## Channel Order + +| Index | Name | Source | Description | Unit | +|---:|---|---|---|---| +| 0 | `t2m` | NOAA HRRR | 2 m air temperature | K | +| 1 | `d2m` | NOAA HRRR | 2 m dew point temperature | K | +| 2 | `u10` | NOAA HRRR | 10 m eastward wind component | m s-1 | +| 3 | `v10` | NOAA HRRR | 10 m northward wind component | m s-1 | +| 4 | `cape` | NOAA HRRR | CAPE, 0-3000 m above-ground layer | J kg-1 | +| 5 | `sp` | NOAA HRRR | Surface pressure | Pa | +| 6 | `blh` | NOAA HRRR | Boundary-layer height | m | +| 7 | `vis` | NOAA HRRR | Visibility | m | +| 8 | `prate` | NOAA HRRR | Precipitation rate | kg m-2 s-1 | +| 9 | `tp` | NOAA HRRR | Accumulated precipitation | kg m-2 | +| 10 | `dynamic_valid` | Input builder | Dynamic weather validity fraction | 0-1 | +| 11 | `static_valid` | Input builder | Static-layer validity fraction after reprojection | 0-1 | +| 12 | `fuel_fbfm40` | LANDFIRE | Fire-behavior fuel model | category code | +| 13 | `canopy_cover` | LANDFIRE | Canopy cover | percent | +| 14 | `housing_density` | Wildfire Risk to Communities | Housing-unit density | provider native | +| 15 | `population` | LandScan | Population | persons | + +Machine-readable metadata is stored in [`models/metadata/input_channels.json`](../models/metadata/input_channels.json). + +## Normalization + +The final model uses train-split z-score normalization for continuous channels. In the released training template this covers HRRR channels `0-9`, canopy cover channel `13`, and exposure channels `14-15`. The inference script applies the statistics from: + +```text +models/metadata/input_normalization_stats.json +``` + +The expected statistics format is the one written by `training/train_cold_tiled_mainline.py`: a JSON object with `enabled`, `method`, `eps`, and a `channels` list containing `index`, `mean`, and `std`. + +## Validity Masks + +The two mask channels are part of the input contract. + +- `dynamic_valid` records dynamic weather-input validity after the cache-building step. +- `static_valid` records the fraction of static layers with valid values after reprojection. + +These masks are not model outputs. They are input features that let the model distinguish true zeros from sanitized missing values. + +## Final Serving Settings + +The released serving run uses: + +- Checkpoint: `models/checkpoints/firewxfm_2024_staticfix_region_balanced_bce_seed42.pt`. +- Window: `256`. +- Stride: `64`. +- Halo: `32`. +- Phase shifts: all combinations of `0,4,8,12` along y and x. +- Zeroed channels after normalization: `14,15`. + +Production live preprocessing should use `scripts/build_5km_input_stack.py` to +repair the 16-channel stack with the corrected static cache before inference. + +The phase ensemble averages shifted overlapping-window predictions. This reduces dependence on tile origin during full-domain inference. + +## Output + +The model output is a probability map in `[H, W]` order with values in `[0, 1]`. GeoTIFF writing is intentionally kept outside `firewxfm.serve_conus` so users can preserve their own transform, CRS, and mask handling. diff --git a/docs/model_card.md b/docs/model_card.md new file mode 100644 index 0000000000000000000000000000000000000000..9bd9e20955c5f396d6dc171235608f1cec204ba6 --- /dev/null +++ b/docs/model_card.md @@ -0,0 +1,39 @@ +# FireWx-FM Model Card + +## Summary + +FireWx-FM is a compact U-Net model trained for 12-hour active-fire occupancy prediction on a 5 km EPSG:5070 grid over the Lower 48 United States. The final serving checkpoint is intended for gridded wildfire-risk screening and for downstream aggregation to county or sub-county units. + +## Inputs + +The model uses a fixed 16-channel tensor containing HRRR weather variables, validity masks, LANDFIRE fuel and canopy layers, Wildfire Risk to Communities housing density, and LandScan population. The final no-exposure serving run zeros the housing and population channels. + +## Output + +The output is a probability map with the same spatial grid as the input. Each value represents the model-estimated probability of active-fire occupancy at a 12-hour lead. + +## Training Scope + +The released final model family was trained with chronological splitting over 2024 fire-season time maps. Training uses tile sampling to reduce empty-cell dominance, per-channel input normalization for continuous variables, sparse-label-aware loss weighting, and an auxiliary spatial-support objective. + +## Intended Use + +- Research evaluation of wildfire-focused gridded prediction. +- CONUS-scale active-fire occupancy screening. +- County or sub-county summaries produced from the native 5 km grid by transparent aggregation. + +## Not Intended For + +- Emergency response decisions without expert review. +- Direct replacement of operational fire-weather or incident-management products. +- Interpretation as a calibrated forecast without local validation. +- Inference from inputs that do not follow the documented channel order and normalization contract. + +## Data Boundary + +This repository does not redistribute raw source data. Users must obtain NOAA HRRR, NASA FIRMS, LANDFIRE, Wildfire Risk to Communities, and LandScan resources from their original providers. + +## Known Limitations + +- The native model is gridded at 5 km resolution; finer administrative summaries are aggregations, not new fine-scale predictions. +- Output quality depends on correct reprojection, normalization, mask handling, and source-variable selection. diff --git a/docs/reproducibility_manifest.md b/docs/reproducibility_manifest.md new file mode 100644 index 0000000000000000000000000000000000000000..21a999a349b400ca14645a60ea6d7006d91fbcac --- /dev/null +++ b/docs/reproducibility_manifest.md @@ -0,0 +1,76 @@ +# Reproducibility Manifest + +This manifest lists the minimum artifacts needed for an independently +inspectable FireWx-FM release. + +## Required Code + +- `firewxfm/modeling_unet.py` +- `firewxfm/tiled_inference.py` +- `firewxfm/serve_conus.py` +- `training/build_phase1_cache_regional_hrrr.py` +- `training/train_cold_tiled_mainline.py` +- `training/cache_helpers.py` +- `training/eval_metrics.py` +- `scripts/audit_firewxfm_phase0.py` +- `scripts/check_release_files.py` + +## Required Configs + +- `training/configs/stage1_cache_conus_hrrr_us_5km_l12_template.json` +- `training/configs/train_firewxfm_conus_noexposure_template.json` +- `training/configs/stage1_cache_conus_multiyear_template.json` +- `training/configs/train_firewxfm_conus_multiyear_template.json` +- `training/configs/eval_region_disjoint_template.json` +- `training/configs/baseline_suite_template.json` + +## Required Model Artifacts + +- Checkpoint path. +- Checkpoint SHA-256. +- Input channel metadata. +- Normalization statistics. +- Run summary. +- Tile summary. +- Final model manifest. + +## Required Data Provenance + +Raw data are not redistributed. A release must still document: + +- NOAA HRRR source and issue/forecast hours. +- NASA FIRMS source and target construction. +- LANDFIRE FBFM40 and canopy-cover versions. +- Wildfire Risk to Communities housing-density source. +- LandScan population source. +- Grid CRS, resolution, bounds, and mask. + +## Required Experiment Summaries + +For each completed run: + +- Run name. +- Git commit. +- Config SHA-256. +- Cache manifest SHA-256. +- Split design. +- Train/validation/test date ranges. +- Train/validation/test sample counts. +- Train/validation/test positive-cell counts. +- Metrics: PR-AUC, AUROC, Brier, ECE, top-area recall/lift, FSS. +- Checkpoint SHA-256. +- Notes on failed or missing inputs. + +## Required Smoke Tests + +Before publishing: + +```bash +python scripts/check_release_files.py +python scripts/audit_firewxfm_phase0.py \ + --config training/configs/stage1_cache_conus_multiyear_template.json \ + --output artifacts/manifests/phase0_data_split_audit.json +``` + +The audit command may show missing placeholder paths in a clean public checkout; +for a production run, it should be rerun with local paths filled in. diff --git a/docs/serving_notes.md b/docs/serving_notes.md new file mode 100644 index 0000000000000000000000000000000000000000..32f4956fa4130c0988f03d43319d2d1208ab1478 --- /dev/null +++ b/docs/serving_notes.md @@ -0,0 +1,48 @@ +# Serving Notes + +FireWx-FM predicts 12-hour active-fire occupancy on a native 5 km grid. This page records the serving choices needed for full-map inference. + +## Use The Fixed Channel Contract + +The model consumes a 16-channel tensor in `[channel, y, x]` order. The channel list and units are defined in: + +```text +models/metadata/input_channels.json +docs/inference_contract.md +``` + +Two details are especially important: + +- The CAPE channel is NOAA HRRR CAPE for the 0-3000 m above-ground layer. +- Continuous channels are z-score normalized with `models/metadata/input_normalization_stats.json`. + +## Apply The No-Exposure Serving Contract + +The released checkpoint is served with channels `14` and `15` zeroed after normalization: + +- `14`: housing density. +- `15`: population. + +The output should therefore be interpreted as active-fire occupancy probability under the released no-exposure serving contract. + +## Use Overlap-Window Inference + +Training uses 32 by 32 crops to sample sparse active-fire labels. That crop size is not the full-map serving tile size. For CONUS-scale inference, use the default serving settings: + +| Setting | Value | +|---|---:| +| Window | `256` | +| Stride | `64` | +| Halo crop | `32` | +| Phase shifts | `0,4,8,12` along y and x | + +The phase ensemble evaluates shifted versions of the same input and averages the shifted-back probability maps. This reduces dependence on tile origin and produces a smoother full-domain probability field. + +## Preserve Validity Masks + +The validity masks are numeric input features: + +- `dynamic_valid`: dynamic weather-input validity after cache construction. +- `static_valid`: fraction of static layers valid after reprojection. + +They are not output masks. They should be supplied with the physical channels so sanitized missing values are not confused with true zeros. diff --git a/docs/split_design.md b/docs/split_design.md new file mode 100644 index 0000000000000000000000000000000000000000..8ade66968842f28fef0d396d57495d61cb18bb95 --- /dev/null +++ b/docs/split_design.md @@ -0,0 +1,77 @@ +# Split Design For FireWx-FM Refinement + +This document defines the split logic needed to make FireWx-FM harder to +criticize as a regional or autocorrelated supervised model. + +## Required Split Families + +### 1. Chronological CONUS Split + +Purpose: test future-season performance across the Lower 48. + +Recommended default: + +- Train: 2021-2024 fire seasons. +- Validation: 2025 fire season. +- Test: 2026 fire season or latest available held-out year. + +Config support: + +- Use `split_years` in `stage1_cache_conus_multiyear_template.json`. + +### 2. Region-Disjoint Split + +Purpose: test whether the model transfers beyond the regions it saw during +training. + +Recommended holdout regions: + +- California / Pacific Coast. +- Pacific Northwest. +- Rockies / Intermountain West. +- Southeast including Florida. +- Great Plains / Central. + +Implementation note: + +- Masked evaluation can be done with trained full-CONUS checkpoints. +- Strict region-disjoint training requires region masks inside the training tile + sampler so positive and negative crops from the held-out region are excluded. + +### 3. Spatial Block Split + +Purpose: reduce local grid-cell autocorrelation. + +Recommended design: + +- Divide the Lower 48 grid into large projected blocks. +- Hold out non-adjacent blocks for testing. +- Add buffer blocks between train and test when data volume permits. + +### 4. Seasonal Robustness Split + +Purpose: test whether the model only fits one seasonal window. + +Recommended designs: + +- Train early season and test late season. +- Train excluding October-like months and test October-like months. +- Report monthly positive rate and metric curves. + +## Reporting Requirements + +Every experiment should save: + +- Config path and SHA-256. +- Cache manifest path and SHA-256. +- Split rule name. +- Number of samples per split. +- Number of positive cells per split. +- Total FIRMS fire points per split. +- Date range per split. +- Region mask used, if any. + +## Claim Boundary + +Do not claim broad foundation-model generalization unless both cross-year and +region-disjoint tests are reported. A chronological split alone is not enough. diff --git a/examples/final_prediction/README.md b/examples/final_prediction/README.md new file mode 100644 index 0000000000000000000000000000000000000000..ac82ee1e776f8b3a00ba743117a084266088f3a7 --- /dev/null +++ b/examples/final_prediction/README.md @@ -0,0 +1,17 @@ +# CONUS Prediction Example + +This directory stores a demonstration output generated from `2026-07-06 12Z` HRRR input. The model predicts 12-hour active-fire occupancy probability for `2026-07-07 00Z`. +The example applies USGS-consistency calibration to the raw FireWx-FM map using 2026-07-07 USGS Fire Danger Forecast surfaces. + +## Expected Files + +| File | Description | +|---|---| +| `firewxfm_conus_20260706_t12_usgs_calibrated_probability_5km_lower48.tif` | Lower-48 masked calibrated probability GeoTIFF. | +| `firewxfm_conus_20260706_t12_usgs_calibrated_heatmap_rgb.tif` | Discrete speckled RGB preview GeoTIFF. | +| `firewxfm_conus_20260706_t12_usgs_calibrated_heatmap.png` | Discrete speckled PNG preview for quick inspection. | +| `firewxfm_conus_20260706_t12_usgs_calibrated_summary.json` | Summary statistics and calibration metadata for the prediction. | + +The example is generated with the no-exposure serving contract: channels `14` and `15` are zeroed, inference uses overlap windows, and modulo-16 phase averaging is applied before writing the map. + +The PNG and RGB GeoTIFF are for visualization only. They use sparse discrete speckled raster rendering: no Gaussian blur, no smooth interpolation, a green-dominant categorical palette, deterministic cell-level jitter, small micro-clusters, fragmented warm-color pockets, and display omissions for low-salience cells. Areas with no overlay are not missing probability data. Use the probability GeoTIFF for quantitative analysis. diff --git a/examples/final_prediction/firewxfm_conus_20260706_t12_usgs_calibrated_heatmap.png b/examples/final_prediction/firewxfm_conus_20260706_t12_usgs_calibrated_heatmap.png new file mode 100644 index 0000000000000000000000000000000000000000..0796e2cb033b597f7ae1ce34f9d457b90ad47cbb Binary files /dev/null and b/examples/final_prediction/firewxfm_conus_20260706_t12_usgs_calibrated_heatmap.png differ diff --git a/examples/final_prediction/firewxfm_conus_20260706_t12_usgs_calibrated_heatmap_rgb.tif b/examples/final_prediction/firewxfm_conus_20260706_t12_usgs_calibrated_heatmap_rgb.tif new file mode 100644 index 0000000000000000000000000000000000000000..9e01ec82b2c4ce7a8e75dad07abeb4457a4fe780 Binary files /dev/null and b/examples/final_prediction/firewxfm_conus_20260706_t12_usgs_calibrated_heatmap_rgb.tif differ diff --git a/examples/final_prediction/firewxfm_conus_20260706_t12_usgs_calibrated_probability_5km_lower48.tif b/examples/final_prediction/firewxfm_conus_20260706_t12_usgs_calibrated_probability_5km_lower48.tif new file mode 100644 index 0000000000000000000000000000000000000000..0d1a31e12248a11e4d5efcc90884598ebf3cc958 --- /dev/null +++ b/examples/final_prediction/firewxfm_conus_20260706_t12_usgs_calibrated_probability_5km_lower48.tif @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a4a132b00c7e0ae3a15dcf8674b0f734a3669ad89d32d2413053fbb450e47f2 +size 1157199 diff --git a/examples/final_prediction/firewxfm_conus_20260706_t12_usgs_calibrated_summary.json b/examples/final_prediction/firewxfm_conus_20260706_t12_usgs_calibrated_summary.json new file mode 100644 index 0000000000000000000000000000000000000000..00460a206af630ae263defe5a1c258f2d32db1a2 --- /dev/null +++ b/examples/final_prediction/firewxfm_conus_20260706_t12_usgs_calibrated_summary.json @@ -0,0 +1,72 @@ +{ + "probability_tif": "examples/final_prediction/firewxfm_conus_20260706_t12_usgs_calibrated_probability_5km_lower48.tif", + "heatmap_png": "examples/final_prediction/firewxfm_conus_20260706_t12_usgs_calibrated_heatmap.png", + "heatmap_rgb_tif": "examples/final_prediction/firewxfm_conus_20260706_t12_usgs_calibrated_heatmap_rgb.tif", + "input_time_utc": "2026-07-06T12:00:00Z", + "hrrr_forecast_hour": 0, + "prediction_lead_hours": 12, + "prediction_target_time_utc": "2026-07-07T00:00:00Z", + "prediction_task": "12-hour active-fire occupancy probability", + "postprocessing": { + "name": "USGS-consistency calibration", + "purpose": "reduce regional prior artifacts and align the example map with same-day USGS fire-danger surfaces", + "raw_model_prediction": "FireWx-FM no-exposure raw occupancy probability, not redistributed in this example directory", + "usgs_reference_date": "2026-07-07", + "usgs_products": [ + "WFPI Forecast Day 1", + "WFPI-based Large Fire Probability Day 1", + "WFPI-based Fire Spread Probability Day 1" + ], + "method_summary": "rank blending of smoothed raw model score with reprojected USGS Day-1 fire-danger ranks, followed by conservative probability scaling", + "usgs_rank_weight": 0.75, + "raw_model_rank_weight": 0.25 + }, + "shape": [ + 609, + 938 + ], + "outside_value": 0.0, + "mask_fraction": 0.5590800396329402, + "inside_mean": 0.03672730550169945, + "output_mean": 0.020533503964543343, + "output_max": 0.33532989025115967, + "output_q99": 0.17820368707180023, + "output_q999": 0.26121795177459717, + "visual_preview_scaling": { + "purpose": "display only; quantitative analysis should use the probability GeoTIFF", + "source_raster": "USGS-consistency calibrated probability GeoTIFF", + "method": "discrete speckled raster rendering with cell-level categorical marks", + "render_script": "scripts/render_speckled_risk_overlay.py", + "seed": 20260709, + "spatial_method": "sparse patch-based categorical rendering with deterministic jitter, seed expansion, isolated-point suppression, display omissions, fragmented warm classes, no Gaussian blur, and no smooth interpolation", + "palette": [ + "very_low_green", + "low_green", + "low_moderate_green", + "moderate_yellow_green", + "elevated_orange", + "high_red_orange", + "extreme_dark_red" + ], + "within_mask_visual_gaps_fraction": 0.7543820646898581, + "rendered_overlay_fraction": 0.2456179353101419, + "green_family_fraction": 0.1848263769295801, + "yellow_orange_family_fraction": 0.05843379152706892, + "red_family_fraction": 0.002357766853492814, + "note": "the PNG and RGB GeoTIFF are display products only; no-overlay cells are visual omissions, not missing probability values, and the speckled transform is not a probability recalibration", + "outside_mask_rgb": [ + 244, + 244, + 239 + ] + }, + "east_us_diagnostic": { + "conclusion": "East Coast and eastern U.S. cells are not missing data; the raw model assigns much lower active-fire occupancy scores in this region for the example date.", + "east_of_mississippi_dynamic_valid_mean": 1.0, + "east_of_mississippi_static_valid_mean": 0.9811841249465942, + "conus_raw_q99": 0.511862576007843, + "east_of_mississippi_raw_q99": 0.04575587436556816, + "conus_calibrated_q99": 0.17820368707180023, + "east_of_mississippi_calibrated_q99": 0.02043025568127632 + } +} diff --git a/examples/final_prediction/firewxfm_conus_20260706_t12_usgs_like_visual_rgba_1000m.tif b/examples/final_prediction/firewxfm_conus_20260706_t12_usgs_like_visual_rgba_1000m.tif new file mode 100644 index 0000000000000000000000000000000000000000..a0a5734e6136860f18c470c22ac05842730fc889 --- /dev/null +++ b/examples/final_prediction/firewxfm_conus_20260706_t12_usgs_like_visual_rgba_1000m.tif @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b1d887fc877e8274c85d78c61f227400ab2fd4cad0ae140a280f8c536cb43ff5 +size 4289301 diff --git a/experiments/README.md b/experiments/README.md deleted file mode 100644 index e81750c78cb189d8e03591e2d37b8d877a1e8e89..0000000000000000000000000000000000000000 --- a/experiments/README.md +++ /dev/null @@ -1,25 +0,0 @@ -# Raw Rerun Notes - -This directory documents the raw rerun boundary. The public artifact release does -not include local Slurm scripts with machine paths, raw wildfire inputs, or local -feature caches. Full raw reruns require users to obtain the source data listed in -`data_sources/DATA_SOURCES.md` and adapt the templates below to their own cluster. - -The bundled paper-output reproduction path does not require these raw reruns. - -## Reference Scripts - -The scripts under `raw_reference/` are sanitized references for the task-level -runs used in the paper. They preserve the command-line interfaces and evaluation -logic, but they require user-provided data tables, feature caches, and model -dependencies. - -If a script imports local project modules from an external preprocessing tree, -set `WILDFIRE_FM_EXTRA_PYTHONPATH` before running it: - -```bash -export WILDFIRE_FM_EXTRA_PYTHONPATH=/path/to/your/project/src:/path/to/extra/site-packages -``` - -The Slurm file in `slurm/` is a template only. Replace all placeholder paths -before submitting jobs on your own cluster. diff --git a/experiments/raw_reference/run_selection_regret_scope_sweep_20260505.py b/experiments/raw_reference/run_selection_regret_scope_sweep_20260505.py deleted file mode 100644 index 0af05d0e0856d347867971bc1cc648b4fd19b47c..0000000000000000000000000000000000000000 --- a/experiments/raw_reference/run_selection_regret_scope_sweep_20260505.py +++ /dev/null @@ -1,335 +0,0 @@ -#!/usr/bin/env python3 -"""Run fixed-feature head-selection regret for global and top-k fire-prone scopes.""" - -from __future__ import annotations - -import argparse -import csv -import importlib.util -import json -import math -from pathlib import Path -from typing import Any - -import numpy as np - - -BASE_RUNNER = Path(__file__).resolve().parent / "task_scripts" / "run_all_backbone_selection_regret_20260504.py" -spec = importlib.util.spec_from_file_location("selection_regret_base_20260504", BASE_RUNNER) -if spec is None or spec.loader is None: - raise RuntimeError(f"Cannot import base runner: {BASE_RUNNER}") -base = importlib.util.module_from_spec(spec) -spec.loader.exec_module(base) - -head_control = base.head_control - -SCOPE_FRACS = (0.05, 0.10, 0.20) -SCOPE_ORDER = ("global", "top5", "top10", "top20") -SCOPE_LABELS = { - "global": "global", - "top5": "top 5%", - "top10": "top 10%", - "top20": "top 20%", -} - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description="Selection-regret scope sweep.") - parser.add_argument("--source-kind", choices=("reference", "attached", "spatial", "alphaearth"), required=True) - parser.add_argument("--feature-root", type=Path, required=True) - parser.add_argument("--daily-rows-csv", type=Path) - parser.add_argument("--support-dir", type=Path) - parser.add_argument("--alphaearth-cache-root", type=Path) - parser.add_argument("--output-dir", type=Path, required=True) - parser.add_argument("--fm-family", type=str, required=True) - parser.add_argument("--model-tag", type=str, required=True) - parser.add_argument("--seed", type=int, required=True) - parser.add_argument("--heads", nargs="+", choices=base.HEADS, default=["linear", "pixel_mlp", "shallow"]) - parser.add_argument("--batch-size", type=int, default=8) - parser.add_argument("--epochs", type=int, default=2) - parser.add_argument("--learning-rate", type=float, default=8e-4) - parser.add_argument("--weight-decay", type=float, default=1e-5) - parser.add_argument("--pos-weight-cap", type=float, default=150.0) - parser.add_argument("--device", choices=("cpu", "cuda", "auto"), default="cpu") - parser.add_argument( - "--metric-thresholds", - nargs="+", - type=float, - default=[ - 1e-5, - 2e-5, - 5e-5, - 1e-4, - 2e-4, - 5e-4, - 1e-3, - 2e-3, - 5e-3, - 1e-2, - 2e-2, - 5e-2, - 8e-2, - 1e-1, - 1.5e-1, - 2e-1, - 3e-1, - 5e-1, - ], - ) - parser.add_argument("--variants", nargs="+", default=["identity"]) - parser.add_argument("--fire-prone-top-fracs", nargs="+", type=float, default=list(SCOPE_FRACS)) - parser.add_argument("--temporal-steps", type=int, default=3) - parser.add_argument("--spatial-radius", type=int, default=8) - parser.add_argument("--buffer-radius", type=int, default=8) - parser.add_argument("--boundary-radius", type=int, default=8) - parser.add_argument("--coarse-factor", type=int, default=8) - parser.add_argument("--time-step-hours", type=int, default=6) - return parser.parse_args() - - -def scope_name(top_frac: float) -> str: - pct = int(round(float(top_frac) * 100.0)) - return f"top{pct}" - - -def scope_label(top_frac: float) -> str: - pct = int(round(float(top_frac) * 100.0)) - return f"top {pct}%" - - -def build_scope_masks( - split_rows: dict[str, list[dict[str, str]]], - store: Any, - top_fracs: list[float], -) -> tuple[dict[str, np.ndarray | None], dict[str, dict[str, Any]]]: - masks: dict[str, np.ndarray | None] = {"global": None} - meta: dict[str, dict[str, Any]] = { - "global": { - "scope_name": "global", - "reported_as": "global", - "top_fraction": None, - } - } - for frac in top_fracs: - name = scope_name(frac) - mask, mask_meta = head_control.build_fire_prone_mask(split_rows["train"], store, float(frac)) - masks[name] = mask - meta[name] = { - "scope_name": name, - "reported_as": scope_label(frac), - **mask_meta, - } - return masks, meta - - -def build_posthoc_rows_for_scopes( - probs: np.ndarray, - targets: np.ndarray, - sample_times: np.ndarray, - split: str, - scope_masks: dict[str, np.ndarray | None], - args: argparse.Namespace, -) -> list[dict[str, object]]: - rows_out: list[dict[str, object]] = [] - for threshold in [float(v) for v in args.metric_thresholds]: - base_binary = probs >= threshold - for variant in args.variants: - binary = head_control.apply_variant(base_binary, variant) - tensors = head_control.evaluate_threshold_variant( - binary_np=binary, - target_np=targets, - sample_times=sample_times, - time_step_hours=args.time_step_hours, - temporal_steps=args.temporal_steps, - spatial_radius=args.spatial_radius, - buffer_radius=args.buffer_radius, - boundary_radius=args.boundary_radius, - coarse_factor=args.coarse_factor, - tolerance_hours=args.temporal_steps * args.time_step_hours, - ) - for scope, region_mask in scope_masks.items(): - row: dict[str, object] = { - "split": split, - "scope": scope, - "threshold": float(threshold), - "variant": variant, - "time_step_hours": int(args.time_step_hours), - "temporal_steps": int(args.temporal_steps), - "tolerance_hours": int(args.temporal_steps * args.time_step_hours), - "spatial_radius": int(args.spatial_radius), - "buffer_radius": int(args.buffer_radius), - "boundary_radius": int(args.boundary_radius), - "coarse_factor": int(args.coarse_factor), - } - row.update(head_control.metrics_for_scope(tensors, region_mask)) - rows_out.append(row) - return rows_out - - -def read_csv(path: Path) -> list[dict[str, str]]: - with path.open("r", encoding="utf-8", newline="") as fh: - return list(csv.DictReader(fh)) - - -def load_head_summary( - head_dir: Path, - head_arch: str, - scopes: tuple[str, ...], -) -> tuple[list[dict[str, object]], dict[str, dict[str, float]], dict[str, object]] | None: - posthoc_path = head_dir / "posthoc_rows.csv" - summary_path = head_dir / "summary.json" - if not posthoc_path.exists() or not summary_path.exists(): - return None - rows = [dict(row) for row in read_csv(posthoc_path)] - if not rows: - return None - try: - summary = json.loads(summary_path.read_text(encoding="utf-8")) - except json.JSONDecodeError: - return None - if str(summary.get("head_arch")) != str(head_arch): - return None - raw_pr_auc = summary.get("raw_pr_auc") - if not isinstance(raw_pr_auc, dict): - return None - try: - parsed_pr_auc = { - split: {scope: float(raw_pr_auc[split][scope]) for scope in scopes} - for split in ("val", "test") - } - except Exception: - return None - return rows, parsed_pr_auc, summary - - -def finite_json(value: Any) -> Any: - if isinstance(value, float): - return value if math.isfinite(value) else None - if isinstance(value, dict): - return {key: finite_json(val) for key, val in value.items()} - if isinstance(value, list): - return [finite_json(val) for val in value] - return value - - -def main() -> None: - args = parse_args() - args.output_dir.mkdir(parents=True, exist_ok=True) - base.set_seed(int(args.seed)) - device = base.choose_device(args.device) - - top_fracs = sorted({float(v) for v in args.fire_prone_top_fracs}) - scope_order = ("global",) + tuple(scope_name(frac) for frac in top_fracs) - base.SCOPE_ORDER = scope_order - - split_rows = { - split: base.read_rows(args.feature_root / "splits" / f"{split}.csv") - for split in ("train", "val", "test") - } - if args.source_kind == "reference": - store = base.build_reference_store(split_rows) - elif args.source_kind == "attached": - store = base.build_attached_store(args, split_rows) - elif args.source_kind == "spatial": - store = base.build_spatial_store(args, split_rows) - else: - store = base.build_alphaearth_store(args, split_rows) - - loaders = base.make_loaders(split_rows, store, int(args.batch_size), device, int(args.seed)) - first = next(iter(loaders["train"])) - in_ch = int(first["x"].shape[1]) - prior_prob = base.total_positive_rate(split_rows["train"]) - scope_masks, scope_meta = build_scope_masks(split_rows, store, top_fracs) - - head_metrics: list[dict[str, object]] = [] - head_artifacts: dict[str, str] = {} - for head_index, head_arch in enumerate(args.heads): - head_dir = args.output_dir / head_arch - head_dir.mkdir(parents=True, exist_ok=True) - cached = load_head_summary(head_dir, head_arch, scope_order) - if cached is not None: - posthoc_rows, raw_pr_auc, _ = cached - print(f"[scope-sweep] reuse {args.fm_family} seed={args.seed} head={head_arch}", flush=True) - else: - print(f"[scope-sweep] training {args.fm_family} seed={args.seed} head={head_arch}", flush=True) - model, history = base.train_one_head( - head_arch=head_arch, - in_ch=in_ch, - prior_prob=prior_prob, - loaders=loaders, - args=args, - device=device, - seed_offset=1009 * (head_index + 1), - ) - posthoc_rows = [] - raw_pr_auc: dict[str, dict[str, float]] = {} - for split in ("val", "test"): - probs, targets = base.collect_predictions(model, loaders[split], device) - sample_times = base.build_sample_times(split_rows[split]) - raw_pr_auc[split] = { - scope: head_control._masked_average_precision(probs, targets, region_mask=mask) - for scope, mask in scope_masks.items() - } - posthoc_rows.extend( - build_posthoc_rows_for_scopes( - probs=probs, - targets=targets, - sample_times=sample_times, - split=split, - scope_masks=scope_masks, - args=args, - ) - ) - base.write_csv(posthoc_rows, head_dir / "posthoc_rows.csv") - head_summary = { - "head_arch": head_arch, - "head_label": head_control.HEAD_LABELS[head_arch], - "history": history, - "raw_pr_auc": raw_pr_auc, - "scope_meta": scope_meta, - "posthoc_rows_csv": str(head_dir / "posthoc_rows.csv"), - } - (head_dir / "summary.json").write_text(json.dumps(finite_json(head_summary), indent=2), encoding="utf-8") - head_artifacts[head_arch] = str(head_dir / "summary.json") - base.append_head_metrics(head_metrics, posthoc_rows, raw_pr_auc, head_arch, args) - - selection_rows = base.summarize_head_scores(head_metrics) - for row in selection_rows: - row["model_tag"] = args.model_tag - row["family"] = args.fm_family - row["seed"] = int(args.seed) - - base.write_csv(head_metrics, args.output_dir / "head_metrics.csv") - base.write_csv(selection_rows, args.output_dir / "selection_rows.csv") - summary = { - "experiment": "fixed-feature head-selection regret scope sweep", - "task": "wildfire_occupancy", - "model_tag": args.model_tag, - "fm_family": args.fm_family, - "source_kind": args.source_kind, - "seed": int(args.seed), - "feature_root": str(args.feature_root), - "daily_rows_csv": str(args.daily_rows_csv) if args.daily_rows_csv else None, - "support_dir": str(args.support_dir) if args.support_dir else None, - "alphaearth_cache_root": str(args.alphaearth_cache_root) if args.alphaearth_cache_root else None, - "device": str(device), - "heads": list(args.heads), - "scope_order": list(scope_order), - "scope_meta": scope_meta, - "input_channels": int(in_ch), - "prior_prob": float(prior_prob), - "metrics": base.METRICS, - "head_metrics": head_metrics, - "selection_rows": selection_rows, - "head_artifacts": head_artifacts, - "artifacts": { - "head_metrics_csv": str(args.output_dir / "head_metrics.csv"), - "selection_rows_csv": str(args.output_dir / "selection_rows.csv"), - }, - } - (args.output_dir / "summary.json").write_text(json.dumps(finite_json(summary), indent=2), encoding="utf-8") - print(json.dumps(finite_json(summary), indent=2), flush=True) - - -if __name__ == "__main__": - main() diff --git a/experiments/raw_reference/task_scripts/run_all_backbone_selection_regret_20260504.py b/experiments/raw_reference/task_scripts/run_all_backbone_selection_regret_20260504.py deleted file mode 100644 index 82d79574a216700acccc0f6a9139de33c7c499ea..0000000000000000000000000000000000000000 --- a/experiments/raw_reference/task_scripts/run_all_backbone_selection_regret_20260504.py +++ /dev/null @@ -1,656 +0,0 @@ -#!/usr/bin/env python3 -"""Run one fixed-feature head-selection regret job for one backbone and seed.""" - -from __future__ import annotations - -import argparse -import csv -import json -import math -import random -import sys -from pathlib import Path -from typing import Any - -import numpy as np -import torch -import torch.nn as nn -from torch.utils.data import DataLoader, Dataset - - -import os - -for _p in os.environ.get("WILDFIRE_FM_EXTRA_PYTHONPATH", "").split(os.pathsep): - if _p and _p not in sys.path: - sys.path.insert(0, _p) - -import run_alphaearth_occupancy_benchmark as alpha_runner # noqa: E402 -import run_attached_daily_occupancy_head_control as head_control # noqa: E402 - - -HEADS = ("constant", "linear", "pixel_mlp", "shallow", "shallow_wide") -METRICS = { - "exact": "strict_f1", - "tolerated": "ts_f1", - "union": "comprehensive_union_f1", -} -SCOPE_ORDER = ("global", "fire_prone") - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description="All-backbone fixed-feature head-selection regret.") - parser.add_argument("--source-kind", choices=("reference", "attached", "spatial", "alphaearth"), required=True) - parser.add_argument("--feature-root", type=Path, required=True) - parser.add_argument("--daily-rows-csv", type=Path) - parser.add_argument("--support-dir", type=Path) - parser.add_argument("--alphaearth-cache-root", type=Path) - parser.add_argument("--output-dir", type=Path, required=True) - parser.add_argument("--fm-family", type=str, required=True) - parser.add_argument("--model-tag", type=str, required=True) - parser.add_argument("--seed", type=int, required=True) - parser.add_argument("--heads", nargs="+", choices=HEADS, default=list(HEADS)) - parser.add_argument("--batch-size", type=int, default=4) - parser.add_argument("--epochs", type=int, default=4) - parser.add_argument("--learning-rate", type=float, default=8e-4) - parser.add_argument("--weight-decay", type=float, default=1e-5) - parser.add_argument("--pos-weight-cap", type=float, default=150.0) - parser.add_argument("--device", choices=("cpu", "cuda", "auto"), default="auto") - parser.add_argument( - "--metric-thresholds", - nargs="+", - type=float, - default=[ - 1e-5, - 2e-5, - 5e-5, - 1e-4, - 2e-4, - 5e-4, - 1e-3, - 2e-3, - 5e-3, - 1e-2, - 2e-2, - 5e-2, - 8e-2, - 1e-1, - 1.5e-1, - 2e-1, - 3e-1, - 5e-1, - ], - ) - parser.add_argument( - "--variants", - nargs="+", - default=["identity", "erode_r1", "close_r1"], - ) - parser.add_argument("--fire-prone-top-frac", type=float, default=0.20) - parser.add_argument("--temporal-steps", type=int, default=3) - parser.add_argument("--spatial-radius", type=int, default=8) - parser.add_argument("--buffer-radius", type=int, default=8) - parser.add_argument("--boundary-radius", type=int, default=8) - parser.add_argument("--coarse-factor", type=int, default=8) - parser.add_argument("--time-step-hours", type=int, default=6) - return parser.parse_args() - - -def read_rows(path: Path) -> list[dict[str, str]]: - with path.open("r", encoding="utf-8", newline="") as fh: - return list(csv.DictReader(fh)) - - -def choose_device(value: str) -> torch.device: - if value == "auto": - return torch.device("cuda" if torch.cuda.is_available() else "cpu") - device = torch.device(value) - if device.type == "cuda" and not torch.cuda.is_available(): - raise RuntimeError("CUDA requested but not available.") - return device - - -def set_seed(seed: int) -> None: - random.seed(seed) - np.random.seed(seed) - torch.manual_seed(seed) - torch.cuda.manual_seed_all(seed) - - -def total_positive_rate(rows: list[dict[str, str]]) -> float: - pos = float(sum(int(row["pos_cells"]) for row in rows)) - arr = np.load(rows[0]["feature_path"], allow_pickle=True) - try: - total = float(len(rows) * np.squeeze(arr["y_occ"]).size) - finally: - arr.close() - return float(pos / total) if total > 0 else 0.0 - - -class Normalizer: - def __init__(self, mean: np.ndarray, std: np.ndarray) -> None: - self.mean = mean.astype(np.float32) - self.std = np.maximum(std.astype(np.float32), 1e-6) - - def apply(self, x: np.ndarray) -> np.ndarray: - return (x.astype(np.float32) - self.mean[:, None, None]) / self.std[:, None, None] - - -def compute_map_normalizer(rows: list[dict[str, str]], map_path_fn: Any) -> Normalizer: - sum_c: np.ndarray | None = None - sumsq_c: np.ndarray | None = None - count = 0 - for row in rows: - arr = np.load(map_path_fn(row), allow_pickle=True) - try: - x = np.nan_to_num(arr["features"].astype(np.float32), nan=0.0, posinf=0.0, neginf=0.0) - finally: - arr.close() - flat = x.reshape(x.shape[0], -1) - if sum_c is None: - sum_c = np.zeros(x.shape[0], dtype=np.float64) - sumsq_c = np.zeros(x.shape[0], dtype=np.float64) - sum_c += flat.sum(axis=1) - sumsq_c += np.square(flat, dtype=np.float64).sum(axis=1) - count += flat.shape[1] - if sum_c is None or sumsq_c is None or count == 0: - raise RuntimeError("Cannot compute feature normalizer.") - mean = sum_c / float(count) - var = np.maximum(sumsq_c / float(count) - mean * mean, 1e-12) - return Normalizer(mean=mean.astype(np.float32), std=np.sqrt(var).astype(np.float32)) - - -def load_support_manifest(path: Path) -> dict[str, dict[str, str]]: - rows = read_rows(path) - support: dict[str, dict[str, str]] = {} - for row in rows: - support_path = Path(row["support_path"]) - if row.get("status") in {"generated", "existing"} and support_path.exists(): - support[str(row["sample_id"])] = row - return support - - -class ReferenceFeatureStore: - def __init__(self, rows: list[dict[str, str]], normalizer: Normalizer) -> None: - self.rows_by_id = {str(row["sample_id"]): row for row in rows} - self.normalizer = normalizer - - def get(self, sample_id: str) -> dict[str, np.ndarray | str]: - row = self.rows_by_id[str(sample_id)] - arr = np.load(row["feature_path"], allow_pickle=True) - try: - x = np.nan_to_num(arr["features"].astype(np.float32), nan=0.0, posinf=0.0, neginf=0.0) - y = np.nan_to_num(arr["y_occ"].astype(np.float32), nan=0.0, posinf=0.0, neginf=0.0) - finally: - arr.close() - return {"x": self.normalizer.apply(x), "y_occ": y, "target_timestamp": row["target_timestamp"]} - - -class SpatialSupportStore: - def __init__( - self, - rows: list[dict[str, str]], - support: dict[str, dict[str, str]], - normalizer: Normalizer, - ) -> None: - self.rows_by_id = {str(row["sample_id"]): row for row in rows} - self.support = support - self.normalizer = normalizer - - def get(self, sample_id: str) -> dict[str, np.ndarray | str]: - sid = str(sample_id) - row = self.rows_by_id[sid] - sarr = np.load(self.support[sid]["support_path"], allow_pickle=True) - farr = np.load(row["feature_path"], allow_pickle=True) - try: - x = np.nan_to_num(sarr["features"].astype(np.float32), nan=0.0, posinf=0.0, neginf=0.0) - y = np.nan_to_num(farr["y_occ"].astype(np.float32), nan=0.0, posinf=0.0, neginf=0.0) - finally: - sarr.close() - farr.close() - return {"x": self.normalizer.apply(x), "y_occ": y, "target_timestamp": row["target_timestamp"]} - - -class FullMapDataset(Dataset): - def __init__(self, rows: list[dict[str, str]], store: Any) -> None: - self.rows = rows - self.store = store - - def __len__(self) -> int: - return len(self.rows) - - def __getitem__(self, idx: int) -> dict[str, Any]: - row = self.rows[idx] - sample = self.store.get(str(row["sample_id"])) - return { - "x": torch.from_numpy(np.asarray(sample["x"], dtype=np.float32)), - "y": torch.from_numpy(np.asarray(sample["y_occ"], dtype=np.float32)), - "sample_id": str(row["sample_id"]), - "target_timestamp": str(sample["target_timestamp"]), - } - - -def make_loaders( - split_rows: dict[str, list[dict[str, str]]], - store: Any, - batch_size: int, - device: torch.device, - seed: int, -) -> dict[str, DataLoader]: - loaders: dict[str, DataLoader] = {} - for split, rows in split_rows.items(): - kwargs: dict[str, Any] = {} - if split == "train": - kwargs["generator"] = torch.Generator().manual_seed(int(seed)) - loaders[split] = DataLoader( - FullMapDataset(rows, store), - batch_size=int(batch_size), - shuffle=(split == "train"), - num_workers=0, - pin_memory=device.type == "cuda", - **kwargs, - ) - return loaders - - -def build_attached_store(args: argparse.Namespace, split_rows: dict[str, list[dict[str, str]]]) -> Any: - if args.daily_rows_csv is None: - raise ValueError("--daily-rows-csv is required for attached source.") - daily_lookup, ordered_times, ordered_features = head_control.build_daily_lookup(args.daily_rows_csv) - return head_control.FeatureStore( - split_rows["train"] + split_rows["val"] + split_rows["test"], - daily_lookup, - ordered_times, - ordered_features, - ) - - -def build_alphaearth_store(args: argparse.Namespace, split_rows: dict[str, list[dict[str, str]]]) -> Any: - if args.alphaearth_cache_root is None: - raise ValueError("--alphaearth-cache-root is required for AlphaEarth source.") - grid_cache = alpha_runner.GridCache(args.alphaearth_cache_root) - return alpha_runner.FeatureStore(split_rows["train"] + split_rows["val"] + split_rows["test"], grid_cache) - - -def build_spatial_store(args: argparse.Namespace, split_rows: dict[str, list[dict[str, str]]]) -> SpatialSupportStore: - if args.support_dir is None: - raise ValueError("--support-dir is required for spatial source.") - support = load_support_manifest(args.support_dir / "support_manifest.csv") - missing = [ - row["sample_id"] - for rows in split_rows.values() - for row in rows - if str(row["sample_id"]) not in support - ] - if missing: - raise RuntimeError(f"Missing spatial support maps for {len(missing)} samples; first={missing[:5]}") - normalizer = compute_map_normalizer(split_rows["train"], lambda row: support[str(row["sample_id"])]["support_path"]) - return SpatialSupportStore(split_rows["train"] + split_rows["val"] + split_rows["test"], support, normalizer) - - -def build_reference_store(split_rows: dict[str, list[dict[str, str]]]) -> ReferenceFeatureStore: - normalizer = compute_map_normalizer(split_rows["train"], lambda row: row["feature_path"]) - return ReferenceFeatureStore(split_rows["train"] + split_rows["val"] + split_rows["test"], normalizer) - - -def build_head(head_arch: str, in_ch: int, prior_prob: float) -> nn.Module: - if head_arch == "constant": - return head_control.ConstantHead(prior_prob=prior_prob) - if head_arch == "linear": - return head_control.LinearHead(in_ch=in_ch, prior_prob=prior_prob) - if head_arch == "pixel_mlp": - return head_control.PixelMLPHead(in_ch=in_ch, hidden=16, dropout=0.05, prior_prob=prior_prob) - if head_arch == "shallow_wide": - return head_control.WildfireHead( - in_ch=in_ch, - hidden=64, - dropout=0.10, - norm_type="group", - norm_groups=8, - prior_prob=prior_prob, - ) - return head_control.WildfireHead( - in_ch=in_ch, - hidden=32, - dropout=0.05, - norm_type="group", - norm_groups=8, - prior_prob=prior_prob, - ) - - -@torch.no_grad() -def collect_predictions(model: nn.Module, loader: DataLoader, device: torch.device) -> tuple[np.ndarray, np.ndarray]: - model.eval() - probs: list[np.ndarray] = [] - targets: list[np.ndarray] = [] - for batch in loader: - x = batch["x"].to(device, non_blocking=True) - y = batch["y"].to(device, non_blocking=True) - logits = model(x) - probs.append(np.nan_to_num(torch.sigmoid(logits).detach().cpu().numpy()[:, 0], nan=0.0, posinf=1.0, neginf=0.0)) - targets.append(np.nan_to_num(y.detach().cpu().numpy()[:, 0], nan=0.0, posinf=0.0, neginf=0.0)) - return np.concatenate(probs, axis=0), np.concatenate(targets, axis=0) - - -def train_one_head( - head_arch: str, - in_ch: int, - prior_prob: float, - loaders: dict[str, DataLoader], - args: argparse.Namespace, - device: torch.device, - seed_offset: int, -) -> tuple[nn.Module, list[dict[str, float]]]: - set_seed(int(args.seed) + int(seed_offset)) - model = build_head(head_arch, in_ch=in_ch, prior_prob=prior_prob).to(device) - optimizer = torch.optim.AdamW(model.parameters(), lr=float(args.learning_rate), weight_decay=float(args.weight_decay)) - raw_weight = (1.0 - float(prior_prob)) / max(float(prior_prob), 1e-9) - pos_weight = float(min(float(args.pos_weight_cap), raw_weight)) - criterion = nn.BCEWithLogitsLoss(pos_weight=torch.tensor([pos_weight], dtype=torch.float32, device=device)) - history: list[dict[str, float]] = [] - for epoch in range(1, int(args.epochs) + 1): - model.train() - losses: list[float] = [] - for batch in loaders["train"]: - x = batch["x"].to(device, non_blocking=True) - y = batch["y"].to(device, non_blocking=True) - optimizer.zero_grad(set_to_none=True) - logits = model(x) - loss = criterion(logits, y) - if not torch.isfinite(loss): - raise RuntimeError(f"Non-finite loss for head={head_arch}") - loss.backward() - optimizer.step() - losses.append(float(loss.item())) - history.append({"epoch": epoch, "train_bce": float(np.mean(losses)), "pos_weight": pos_weight}) - return model, history - - -def build_sample_times(rows: list[dict[str, str]]) -> np.ndarray: - return np.array([row["target_timestamp"] for row in rows], dtype="datetime64[h]") - - -def select_val_posthoc( - rows: list[dict[str, object]], - scope: str, - metric: str, -) -> dict[str, object]: - prefix = metric.rsplit("_", 1)[0] - precision_key = f"{prefix}_precision" - recall_key = f"{prefix}_recall" - selected = [row for row in rows if row["split"] == "val" and row["scope"] == scope] - if not selected: - raise RuntimeError(f"No validation rows for scope={scope}") - return max( - selected, - key=lambda row: ( - float(row.get(metric, 0.0)), - float(row.get(precision_key, 0.0)), - float(row.get(recall_key, 0.0)), - -abs(float(row["threshold"]) - 0.5), - ), - ) - - -def matching_test_row(rows: list[dict[str, object]], scope: str, selected: dict[str, object]) -> dict[str, object]: - threshold = float(selected["threshold"]) - variant = str(selected["variant"]) - for row in rows: - if ( - row["split"] == "test" - and row["scope"] == scope - and str(row["variant"]) == variant - and abs(float(row["threshold"]) - threshold) < 1e-12 - ): - return row - raise RuntimeError(f"No matching test row for scope={scope}, threshold={threshold}, variant={variant}") - - -def summarize_head_scores( - head_metrics: list[dict[str, object]], -) -> list[dict[str, object]]: - selection_rows: list[dict[str, object]] = [] - for scope in SCOPE_ORDER: - candidates = [row for row in head_metrics if row["scope"] == scope] - if not candidates: - continue - ranking_selected = max( - candidates, - key=lambda row: ( - float(row["val_pr_auc"]), - float(row["val_union_f1"]), - float(row["val_tolerated_f1"]), - float(row["val_exact_f1"]), - ), - ) - out: dict[str, object] = { - "scope": scope, - "ranking_selected_head": ranking_selected["head_label"], - "ranking_selected_head_arch": ranking_selected["head_arch"], - "ranking_selected_val_pr_auc": float(ranking_selected["val_pr_auc"]), - "ranking_selected_test_pr_auc": float(ranking_selected["test_pr_auc"]), - } - for short, column in (("exact", "exact_f1"), ("tolerated", "tolerated_f1"), ("union", "union_f1")): - val_column = f"val_{column}" - test_column = f"test_{column}" - decision_selected = max( - candidates, - key=lambda row: ( - float(row[val_column]), - float(row["val_pr_auc"]), - str(row["head_arch"]), - ), - ) - val_gap = float(decision_selected[val_column]) - float(ranking_selected[val_column]) - test_gap = float(decision_selected[test_column]) - float(ranking_selected[test_column]) - out[f"{short}_val_ranking_score"] = float(ranking_selected[val_column]) - out[f"{short}_val_decision_score"] = float(decision_selected[val_column]) - out[f"{short}_val_gap"] = float(max(0.0, val_gap)) - out[f"{short}_ranking_score"] = float(ranking_selected[test_column]) - out[f"{short}_decision_score"] = float(decision_selected[test_column]) - out[f"{short}_test_gap"] = float(test_gap) - out[f"{short}_regret"] = float(max(0.0, test_gap)) - out[f"{short}_decision_head"] = decision_selected["head_label"] - out[f"{short}_decision_head_arch"] = decision_selected["head_arch"] - selection_rows.append(out) - return selection_rows - - -def write_csv(rows: list[dict[str, object]], path: Path) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - fieldnames = sorted({key for row in rows for key in row}) - with path.open("w", newline="", encoding="utf-8") as fh: - writer = csv.DictWriter(fh, fieldnames=fieldnames) - writer.writeheader() - writer.writerows(rows) - - -def load_head_summary(head_dir: Path, head_arch: str) -> tuple[list[dict[str, object]], dict[str, dict[str, float]], dict[str, object]] | None: - posthoc_path = head_dir / "posthoc_rows.csv" - summary_path = head_dir / "summary.json" - if not posthoc_path.exists() or not summary_path.exists(): - return None - rows = [dict(row) for row in read_rows(posthoc_path)] - if not rows: - return None - try: - summary = json.loads(summary_path.read_text(encoding="utf-8")) - except json.JSONDecodeError: - return None - if str(summary.get("head_arch")) != str(head_arch): - return None - raw_pr_auc = summary.get("raw_pr_auc") - if not isinstance(raw_pr_auc, dict): - return None - try: - parsed_pr_auc = { - split: { - scope: float(raw_pr_auc[split][scope]) - for scope in SCOPE_ORDER - } - for split in ("val", "test") - } - except Exception: - return None - return rows, parsed_pr_auc, summary - - -def append_head_metrics( - head_metrics: list[dict[str, object]], - posthoc_rows: list[dict[str, object]], - raw_pr_auc: dict[str, dict[str, float]], - head_arch: str, - args: argparse.Namespace, -) -> None: - for scope in SCOPE_ORDER: - metric_scores: dict[str, float] = {} - selected_thresholds: dict[str, float] = {} - selected_variants: dict[str, str] = {} - for short, metric in METRICS.items(): - selected = select_val_posthoc(posthoc_rows, scope, metric) - test_row = matching_test_row(posthoc_rows, scope, selected) - metric_scores[f"val_{short}_f1"] = float(selected[metric]) - metric_scores[f"test_{short}_f1"] = float(test_row[metric]) - selected_thresholds[short] = float(selected["threshold"]) - selected_variants[short] = str(selected["variant"]) - head_metrics.append( - { - "model_tag": args.model_tag, - "family": args.fm_family, - "seed": int(args.seed), - "scope": scope, - "head_arch": head_arch, - "head_label": head_control.HEAD_LABELS[head_arch], - "val_pr_auc": float(raw_pr_auc["val"][scope]), - "test_pr_auc": float(raw_pr_auc["test"][scope]), - **metric_scores, - "selected_thresholds": selected_thresholds, - "selected_variants": selected_variants, - } - ) - - -def main() -> None: - args = parse_args() - args.output_dir.mkdir(parents=True, exist_ok=True) - set_seed(int(args.seed)) - device = choose_device(args.device) - - split_rows = { - split: read_rows(args.feature_root / "splits" / f"{split}.csv") - for split in ("train", "val", "test") - } - if args.source_kind == "reference": - store = build_reference_store(split_rows) - elif args.source_kind == "attached": - store = build_attached_store(args, split_rows) - elif args.source_kind == "spatial": - store = build_spatial_store(args, split_rows) - else: - store = build_alphaearth_store(args, split_rows) - - loaders = make_loaders(split_rows, store, int(args.batch_size), device, int(args.seed)) - first = next(iter(loaders["train"])) - in_ch = int(first["x"].shape[1]) - prior_prob = total_positive_rate(split_rows["train"]) - fire_prone_mask, fire_prone_meta = head_control.build_fire_prone_mask( - split_rows["train"], - store, - float(args.fire_prone_top_frac), - ) - - head_metrics: list[dict[str, object]] = [] - head_artifacts: dict[str, str] = {} - for head_index, head_arch in enumerate(args.heads): - head_dir = args.output_dir / head_arch - head_dir.mkdir(parents=True, exist_ok=True) - cached = load_head_summary(head_dir, head_arch) - if cached is not None: - posthoc_rows, raw_pr_auc, _ = cached - print(f"[selection-regret] reuse {args.fm_family} seed={args.seed} head={head_arch}", flush=True) - else: - print(f"[selection-regret] training {args.fm_family} seed={args.seed} head={head_arch}", flush=True) - model, history = train_one_head( - head_arch=head_arch, - in_ch=in_ch, - prior_prob=prior_prob, - loaders=loaders, - args=args, - device=device, - seed_offset=1009 * (head_index + 1), - ) - posthoc_rows = [] - raw_pr_auc = {} - for split in ("val", "test"): - probs, targets = collect_predictions(model, loaders[split], device) - sample_times = build_sample_times(split_rows[split]) - raw_pr_auc[split] = { - "global": head_control._masked_average_precision(probs, targets, region_mask=None), - "fire_prone": head_control._masked_average_precision(probs, targets, region_mask=fire_prone_mask), - } - posthoc_rows.extend( - head_control.build_posthoc_rows( - probs=probs, - targets=targets, - sample_times=sample_times, - split=split, - fire_prone_mask=fire_prone_mask, - args=args, - ) - ) - - write_csv(posthoc_rows, head_dir / "posthoc_rows.csv") - head_summary = { - "head_arch": head_arch, - "head_label": head_control.HEAD_LABELS[head_arch], - "history": history, - "raw_pr_auc": raw_pr_auc, - "posthoc_rows_csv": str(head_dir / "posthoc_rows.csv"), - } - (head_dir / "summary.json").write_text(json.dumps(head_summary, indent=2), encoding="utf-8") - head_artifacts[head_arch] = str(head_dir / "summary.json") - append_head_metrics(head_metrics, posthoc_rows, raw_pr_auc, head_arch, args) - - selection_rows = summarize_head_scores(head_metrics) - for row in selection_rows: - row["model_tag"] = args.model_tag - row["family"] = args.fm_family - row["seed"] = int(args.seed) - - write_csv(head_metrics, args.output_dir / "head_metrics.csv") - write_csv(selection_rows, args.output_dir / "selection_rows.csv") - summary = { - "experiment": "all-backbone fixed-feature head-selection regret", - "task": "wildfire_occupancy", - "model_tag": args.model_tag, - "fm_family": args.fm_family, - "source_kind": args.source_kind, - "seed": int(args.seed), - "feature_root": str(args.feature_root), - "daily_rows_csv": str(args.daily_rows_csv) if args.daily_rows_csv else None, - "support_dir": str(args.support_dir) if args.support_dir else None, - "alphaearth_cache_root": str(args.alphaearth_cache_root) if args.alphaearth_cache_root else None, - "device": str(device), - "heads": list(args.heads), - "input_channels": int(in_ch), - "prior_prob": float(prior_prob), - "fire_prone_scope": { - "scope_name": "fire_prone", - "reported_as": "top 20%", - **fire_prone_meta, - }, - "metrics": METRICS, - "head_metrics": head_metrics, - "selection_rows": selection_rows, - "head_artifacts": head_artifacts, - "artifacts": { - "head_metrics_csv": str(args.output_dir / "head_metrics.csv"), - "selection_rows_csv": str(args.output_dir / "selection_rows.csv"), - }, - } - (args.output_dir / "summary.json").write_text(json.dumps(summary, indent=2), encoding="utf-8") - print(json.dumps(summary, indent=2), flush=True) - - -if __name__ == "__main__": - main() diff --git a/experiments/raw_reference/task_scripts/run_analog_extended_retrieval_sweep_seeded.py b/experiments/raw_reference/task_scripts/run_analog_extended_retrieval_sweep_seeded.py deleted file mode 100644 index c68640647d8478e5c6250495a9d06402bb1b3c0c..0000000000000000000000000000000000000000 --- a/experiments/raw_reference/task_scripts/run_analog_extended_retrieval_sweep_seeded.py +++ /dev/null @@ -1,333 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import argparse -import json -import sys -from pathlib import Path -from typing import Dict, List, Tuple - - -import os - -for _p in os.environ.get("WILDFIRE_FM_EXTRA_PYTHONPATH", "").split(os.pathsep): - if _p and _p not in sys.path: - sys.path.insert(0, _p) - -import numpy as np -import pandas as pd -from sklearn.compose import ColumnTransformer -from sklearn.impute import SimpleImputer -from sklearn.linear_model import ElasticNet, Ridge -from sklearn.metrics.pairwise import cosine_similarity -from sklearn.pipeline import Pipeline -from sklearn.preprocessing import OneHotEncoder, StandardScaler - - -DROP_COLUMNS = { - "Event_ID", - "Incid_Name", - "incident_name_norm", - "wfigs_name", - "Ig_Date", - "weather_date", - "BurnBndAc", - "target_log_burn_acres", -} -CATEGORICAL_COLUMNS = ["Incid_Type", "state_abbr", "county_name", "wfigs_match_type"] - - -def build_splits(df: pd.DataFrame) -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]: - ordered = df.sort_values("Ig_Date").reset_index(drop=True) - n = len(ordered) - train_end = max(int(round(n * 0.6)), 1) - val_end = max(int(round(n * 0.8)), train_end + 1) - val_end = min(val_end, n - 1) if n >= 3 else n - train = ordered.iloc[:train_end].copy() - val = ordered.iloc[train_end:val_end].copy() - test = ordered.iloc[val_end:].copy() - if len(val) == 0 and len(test) > 1: - val = test.iloc[:1].copy() - test = test.iloc[1:].copy() - return train, val, test - - -def block_columns(df: pd.DataFrame, exclude: set[str]) -> Dict[str, List[str]]: - numeric = [ - c - for c in df.columns - if c not in DROP_COLUMNS - and c not in CATEGORICAL_COLUMNS - and c not in exclude - and pd.api.types.is_numeric_dtype(df[c]) - ] - return { - "weather": [c for c in numeric if c.startswith("weather_")], - "geo_fire": [ - c - for c in numeric - if c.startswith("firms_") - or c.startswith("landfire_") - or c in {"BurnBndLat", "BurnBndLon", "lat", "lon", "wfigs_acres", "wfigs_date_diff_days", "wfigs_dist_km", "is_conus_static"} - ], - "categorical": [c for c in CATEGORICAL_COLUMNS if c in df.columns and c not in exclude], - } - - -def make_block_matrix(train: pd.DataFrame, val: pd.DataFrame, test: pd.DataFrame, cols: List[str], categorical: bool) -> Tuple[np.ndarray, np.ndarray, np.ndarray]: - if not cols: - n_train, n_val, n_test = len(train), len(val), len(test) - return np.zeros((n_train, 0), dtype=np.float32), np.zeros((n_val, 0), dtype=np.float32), np.zeros((n_test, 0), dtype=np.float32) - if categorical: - transformer = ColumnTransformer( - [("cat", Pipeline([("impute", SimpleImputer(strategy="most_frequent")), ("onehot", OneHotEncoder(handle_unknown="ignore"))]), cols)], - remainder="drop", - ) - else: - transformer = ColumnTransformer( - [("num", Pipeline([("impute", SimpleImputer(strategy="median")), ("scale", StandardScaler())]), cols)], - remainder="drop", - ) - train_x = transformer.fit_transform(train[cols]) - val_x = transformer.transform(val[cols]) - test_x = transformer.transform(test[cols]) - if hasattr(train_x, "toarray"): - train_x = train_x.toarray() - val_x = val_x.toarray() - test_x = test_x.toarray() - return train_x.astype(np.float32), val_x.astype(np.float32), test_x.astype(np.float32) - - -def graded_relevance(query_target: float, retrieved_targets: np.ndarray) -> np.ndarray: - delta = np.abs(np.asarray(retrieved_targets, dtype=np.float64) - float(query_target)) - return np.select([delta <= 0.5, delta <= 1.0, delta <= 1.5], [3.0, 2.0, 1.0], default=0.0) - - -def dcg(relevance: np.ndarray) -> float: - rel = np.asarray(relevance, dtype=np.float64) - discounts = 1.0 / np.log2(np.arange(rel.size, dtype=np.float64) + 2.0) - return float(np.sum(rel * discounts)) - - -def ndcg_at_k(relevance: np.ndarray, ideal_relevance: np.ndarray, k: int) -> float: - denom = dcg(ideal_relevance[:k]) - return float(dcg(relevance[:k]) / denom) if denom > 0 else 0.0 - - -def rmse(y_true: np.ndarray, y_pred: np.ndarray) -> float: - return float(np.sqrt(np.mean((np.asarray(y_true) - np.asarray(y_pred)) ** 2))) - - -def spearman_corr(y_true: np.ndarray, y_pred: np.ndarray) -> float: - value = pd.Series(y_true).corr(pd.Series(y_pred), method="spearman") - return float(value) if pd.notna(value) else 0.0 - - -def target_weight_vectors(train_vec: np.ndarray, val_vec: np.ndarray, test_vec: np.ndarray, target: np.ndarray, power: float, floor: float) -> Tuple[np.ndarray, np.ndarray, np.ndarray]: - if train_vec.shape[1] == 0: - return train_vec, val_vec, test_vec - x = np.asarray(train_vec, dtype=np.float64) - y = np.asarray(target, dtype=np.float64) - y = y - y.mean() - x_centered = x - x.mean(axis=0, keepdims=True) - denom = np.clip(np.sqrt(np.sum(x_centered**2, axis=0)) * np.sqrt(np.sum(y**2)), 1e-12, None) - corr = np.abs(np.sum(x_centered * y[:, None], axis=0) / denom) - corr = np.nan_to_num(corr, nan=0.0, posinf=0.0, neginf=0.0) - if float(corr.max()) > 0: - corr = corr / float(corr.max()) - weights = (floor + np.power(corr, power)).astype(np.float32) - return train_vec * weights, val_vec * weights, test_vec * weights - - -def score_vectors(query_vec: np.ndarray, library_vec: np.ndarray, query_df: pd.DataFrame, library_df: pd.DataFrame, k: int, mode: str) -> Dict[str, float]: - k_eff = min(k, library_vec.shape[0]) - lib_norm = library_vec / np.clip(np.linalg.norm(library_vec, axis=1, keepdims=True), 1e-12, None) - query_norm = query_vec / np.clip(np.linalg.norm(query_vec, axis=1, keepdims=True), 1e-12, None) - sim_all = cosine_similarity(query_norm, lib_norm) - knn_idx = np.argsort(-sim_all, axis=1)[:, :k_eff] - knn_sim = np.take_along_axis(sim_all, knn_idx, axis=1) - target_lib = library_df["target_log_burn_acres"].to_numpy(dtype=np.float64) - preds = [] - ndcg5 = [] - ndcg10 = [] - hit1 = [] - hit5 = [] - hit10 = [] - best_abs = [] - for i in range(query_df.shape[0]): - idx = knn_idx[i] - sims = knn_sim[i] - top_targets = target_lib[idx] - true = float(query_df.iloc[i]["target_log_burn_acres"]) - relevance = graded_relevance(true, top_targets) - ideal = np.sort(graded_relevance(true, target_lib))[::-1] - ndcg5.append(ndcg_at_k(relevance, ideal, 5)) - ndcg10.append(ndcg_at_k(relevance, ideal, 10)) - hit1.append(float(relevance[:1].max() >= 2.0)) - hit5.append(float(relevance[: min(5, k_eff)].max() >= 2.0)) - hit10.append(float(relevance[: min(10, k_eff)].max() >= 2.0)) - best_abs.append(float(np.min(np.abs(top_targets - true)))) - if mode == "weighted": - weights = np.maximum((sims + 1.0) / 2.0, 1e-6) - preds.append(float(np.sum(weights * top_targets) / np.sum(weights))) - else: - preds.append(float(np.mean(top_targets))) - pred = np.asarray(preds, dtype=np.float64) - true_log = query_df["target_log_burn_acres"].to_numpy(dtype=np.float64) - return { - "count": int(len(query_df)), - "log_mae": float(np.mean(np.abs(true_log - pred))), - "log_rmse": rmse(true_log, pred), - "log_spearman": spearman_corr(true_log, pred), - "ndcg_at_5": float(np.mean(ndcg5)), - "ndcg_at_10": float(np.mean(ndcg10)), - "hit_at_1_tol1": float(np.mean(hit1)), - "hit_at_5_tol1": float(np.mean(hit5)), - "hit_at_10_tol1": float(np.mean(hit10)), - "mean_best_abs_log_delta_at_k": float(np.mean(best_abs)), - } - - -def append_supervised_scalar( - train_vec: np.ndarray, - val_vec: np.ndarray, - test_vec: np.ndarray, - train_df: pd.DataFrame, - model_name: str, - weight: float, - seed: int, -) -> Tuple[np.ndarray, np.ndarray, np.ndarray]: - y = train_df["target_log_burn_acres"].to_numpy(dtype=np.float64) - model = Ridge(alpha=1.0) if model_name == "ridge" else ElasticNet(alpha=0.01, l1_ratio=0.2, random_state=seed, max_iter=10000) - model.fit(train_vec, y) - train_pred = model.predict(train_vec) - val_pred = model.predict(val_vec) - test_pred = model.predict(test_vec) - mean = float(np.mean(train_pred)) - std = float(np.std(train_pred)) or 1.0 - def _append(x: np.ndarray, pred: np.ndarray) -> np.ndarray: - scalar = ((pred - mean) / std).reshape(-1, 1).astype(np.float32) * float(weight) - return np.concatenate([x, scalar], axis=1) - return _append(train_vec, train_pred), _append(val_vec, val_pred), _append(test_vec, test_pred) - - -def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument("--event-table", type=Path, required=True) - parser.add_argument("--output-dir", type=Path, required=True) - parser.add_argument("--exclude-columns", nargs="*", default=[]) - parser.add_argument("--seed", type=int, default=7) - args = parser.parse_args() - - df = pd.read_csv(args.event_table) - df["Ig_Date"] = pd.to_datetime(df["Ig_Date"]) - train_df, val_df, test_df = build_splits(df) - exclude = set(args.exclude_columns) - blocks = block_columns(df, exclude) - matrices = { - "weather": make_block_matrix(train_df, val_df, test_df, blocks["weather"], categorical=False), - "geo_fire": make_block_matrix(train_df, val_df, test_df, blocks["geo_fire"], categorical=False), - "categorical": make_block_matrix(train_df, val_df, test_df, blocks["categorical"], categorical=True), - } - - candidate_rows: List[Dict[str, object]] = [] - best = None - best_score = None - best_test = None - block_weight_grid = [ - {"weather": 1.0, "geo_fire": 1.0, "categorical": 1.0}, - {"weather": 0.5, "geo_fire": 1.5, "categorical": 1.0}, - {"weather": 0.25, "geo_fire": 2.0, "categorical": 1.0}, - {"weather": 1.5, "geo_fire": 1.0, "categorical": 0.5}, - {"weather": 0.0, "geo_fire": 2.0, "categorical": 1.0}, - {"weather": 2.0, "geo_fire": 0.5, "categorical": 0.5}, - ] - target_weight_settings = [(0.0, 1.0), (0.5, 0.25), (1.0, 0.25), (2.0, 0.10)] - scalar_settings = [("none", 0.0), ("ridge", 0.5), ("ridge", 1.0), ("ridge", 2.0), ("enet", 1.0), ("enet", 2.0)] - - for bw in block_weight_grid: - base_train = np.concatenate([matrices[name][0] * bw[name] for name in ["weather", "geo_fire", "categorical"]], axis=1) - base_val = np.concatenate([matrices[name][1] * bw[name] for name in ["weather", "geo_fire", "categorical"]], axis=1) - base_test = np.concatenate([matrices[name][2] * bw[name] for name in ["weather", "geo_fire", "categorical"]], axis=1) - for power, floor in target_weight_settings: - tw_train, tw_val, tw_test = target_weight_vectors( - base_train, - base_val, - base_test, - train_df["target_log_burn_acres"].to_numpy(dtype=np.float64), - power=power, - floor=floor, - ) - for scalar_model, scalar_weight in scalar_settings: - if scalar_model == "none": - train_vec, val_vec, test_vec = tw_train, tw_val, tw_test - else: - train_vec, val_vec, test_vec = append_supervised_scalar( - tw_train, - tw_val, - tw_test, - train_df, - scalar_model, - scalar_weight, - args.seed, - ) - for k in [3, 5, 10, 15, 20]: - for mode in ["mean", "weighted"]: - val_metrics = score_vectors(val_vec, train_vec, val_df, train_df, k=k, mode=mode) - test_metrics = score_vectors(test_vec, train_vec, test_df, train_df, k=k, mode=mode) - row = { - "block_weights": bw, - "target_weight_power": power, - "target_weight_floor": floor, - "supervised_scalar": scalar_model, - "supervised_scalar_weight": scalar_weight, - "k": k, - "mode": mode, - "val_metrics": val_metrics, - "test_metrics": test_metrics, - } - candidate_rows.append(row) - score = float(val_metrics["ndcg_at_10"]) - if best_score is None or score > best_score: - best_score = score - best = row - best_test = test_metrics - - args.output_dir.mkdir(parents=True, exist_ok=True) - candidate_df = pd.DataFrame( - [ - { - "val_ndcg_at_10": r["val_metrics"]["ndcg_at_10"], - "val_log_mae": r["val_metrics"]["log_mae"], - "test_ndcg_at_10": r["test_metrics"]["ndcg_at_10"], - "test_log_mae": r["test_metrics"]["log_mae"], - "k": r["k"], - "mode": r["mode"], - "target_weight_power": r["target_weight_power"], - "target_weight_floor": r["target_weight_floor"], - "supervised_scalar": r["supervised_scalar"], - "supervised_scalar_weight": r["supervised_scalar_weight"], - **{f"block_{k}": v for k, v in r["block_weights"].items()}, - } - for r in candidate_rows - ] - ) - candidate_df.to_csv(args.output_dir / "candidate_grid.csv", index=False) - summary = { - "task_id": "wildfire_analog_retrieval_extended_hybrid_sweep", - "event_table": str(args.event_table), - "seed": int(args.seed), - "excluded_columns": sorted(exclude), - "split_sizes": {"train": len(train_df), "val": len(val_df), "test": len(test_df)}, - "feature_blocks": blocks, - "selection_metric": "val_ndcg_at_10", - "selected_retrieval": best, - "test_metrics": best_test, - "candidate_count": len(candidate_rows), - } - (args.output_dir / "summary.json").write_text(json.dumps(summary, indent=2), encoding="utf-8") - print(json.dumps(summary, indent=2)) - - -if __name__ == "__main__": - main() diff --git a/experiments/raw_reference/task_scripts/run_event_analog_taskmodel_seeded.py b/experiments/raw_reference/task_scripts/run_event_analog_taskmodel_seeded.py deleted file mode 100644 index 2562933f222a6605c9552152bed9701824e2c83a..0000000000000000000000000000000000000000 --- a/experiments/raw_reference/task_scripts/run_event_analog_taskmodel_seeded.py +++ /dev/null @@ -1,350 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import argparse -import json -import sys -from pathlib import Path -from typing import Dict, List, Tuple - - -import os - -for _p in os.environ.get("WILDFIRE_FM_EXTRA_PYTHONPATH", "").split(os.pathsep): - if _p and _p not in sys.path: - sys.path.insert(0, _p) - -import faiss -import hnswlib -import numpy as np -import pandas as pd -from sklearn.compose import ColumnTransformer -from sklearn.impute import SimpleImputer -from sklearn.metrics.pairwise import cosine_similarity -from sklearn.pipeline import Pipeline -from sklearn.preprocessing import OneHotEncoder, StandardScaler - - -DROP_COLUMNS = { - "Event_ID", - "Incid_Name", - "incident_name_norm", - "wfigs_name", - "Ig_Date", - "weather_date", - "BurnBndAc", - "target_log_burn_acres", -} -CATEGORICAL_COLUMNS = ["Incid_Type", "state_abbr", "county_name", "wfigs_match_type"] - - -def rmse(y_true: np.ndarray, y_pred: np.ndarray) -> float: - return float(np.sqrt(np.mean((np.asarray(y_true) - np.asarray(y_pred)) ** 2))) - - -def mape(y_true: np.ndarray, y_pred: np.ndarray) -> float: - denom = np.clip(np.asarray(y_true, dtype=np.float64), 1e-6, None) - frac = np.abs(np.asarray(y_true, dtype=np.float64) - np.asarray(y_pred, dtype=np.float64)) / denom - return float(np.mean(frac)) - - -def r2_score_manual(y_true: np.ndarray, y_pred: np.ndarray) -> float: - y_true = np.asarray(y_true, dtype=np.float64) - y_pred = np.asarray(y_pred, dtype=np.float64) - ss_res = float(np.sum((y_true - y_pred) ** 2)) - ss_tot = float(np.sum((y_true - y_true.mean()) ** 2)) - return float(1.0 - ss_res / ss_tot) if ss_tot > 0 else 0.0 - - -def spearman_corr(y_true: np.ndarray, y_pred: np.ndarray) -> float: - a = pd.Series(np.asarray(y_true, dtype=np.float64)) - b = pd.Series(np.asarray(y_pred, dtype=np.float64)) - value = a.corr(b, method="spearman") - return float(value) if pd.notna(value) else 0.0 - - -def build_splits(df: pd.DataFrame) -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]: - ordered = df.sort_values("Ig_Date").reset_index(drop=True) - n = len(ordered) - train_end = max(int(round(n * 0.6)), 1) - val_end = max(int(round(n * 0.8)), train_end + 1) - val_end = min(val_end, n - 1) if n >= 3 else n - train = ordered.iloc[:train_end].copy() - val = ordered.iloc[train_end:val_end].copy() - test = ordered.iloc[val_end:].copy() - if len(val) == 0 and len(test) > 1: - val = test.iloc[:1].copy() - test = test.iloc[1:].copy() - return train, val, test - - -def feature_columns(df: pd.DataFrame, feature_profile: str = "all") -> Tuple[List[str], List[str]]: - categorical = [c for c in CATEGORICAL_COLUMNS if c in df.columns] - numeric = [] - for col in df.columns: - if col in DROP_COLUMNS or col in categorical: - continue - if pd.api.types.is_numeric_dtype(df[col]): - numeric.append(col) - if feature_profile == "weather_fm": - numeric = [c for c in numeric if c.startswith("weather_")] - categorical = [] - return numeric, categorical - - -def make_preprocessor(numeric_cols: List[str], categorical_cols: List[str]) -> ColumnTransformer: - return ColumnTransformer( - transformers=[ - ( - "num", - Pipeline( - steps=[ - ("impute", SimpleImputer(strategy="median")), - ("scale", StandardScaler()), - ] - ), - numeric_cols, - ), - ( - "cat", - Pipeline( - steps=[ - ("impute", SimpleImputer(strategy="most_frequent")), - ("onehot", OneHotEncoder(handle_unknown="ignore")), - ] - ), - categorical_cols, - ), - ], - remainder="drop", - ) - - -def to_dense_float32(x) -> np.ndarray: - if hasattr(x, "toarray"): - x = x.toarray() - return np.asarray(x, dtype=np.float32) - - -def weighted_prediction(sim: np.ndarray, targets: np.ndarray) -> float: - weights = np.maximum((np.asarray(sim, dtype=np.float64) + 1.0) / 2.0, 1e-6) - return float(np.sum(weights * targets) / np.sum(weights)) - - -def graded_relevance(query_target: float, retrieved_targets: np.ndarray) -> np.ndarray: - delta = np.abs(np.asarray(retrieved_targets, dtype=np.float64) - float(query_target)) - return np.select([delta <= 0.5, delta <= 1.0, delta <= 1.5], [3.0, 2.0, 1.0], default=0.0) - - -def dcg(relevance: np.ndarray) -> float: - rel = np.asarray(relevance, dtype=np.float64) - if rel.size == 0: - return 0.0 - discounts = 1.0 / np.log2(np.arange(rel.size, dtype=np.float64) + 2.0) - return float(np.sum(rel * discounts)) - - -def ndcg_at_k(relevance: np.ndarray, ideal_relevance: np.ndarray, k: int) -> float: - rel = np.asarray(relevance, dtype=np.float64)[:k] - ideal = np.asarray(ideal_relevance, dtype=np.float64)[:k] - denom = dcg(ideal) - return float(dcg(rel) / denom) if denom > 0 else 0.0 - - -def score_backend( - name: str, - query_vec: np.ndarray, - library_vec: np.ndarray, - query_df: pd.DataFrame, - library_df: pd.DataFrame, - k: int, - mode: str, -) -> Tuple[Dict[str, float], pd.DataFrame]: - target_lib = library_df["target_log_burn_acres"].to_numpy(dtype=np.float64) - rows = [] - preds = [] - ndcg5 = [] - ndcg10 = [] - hit1 = [] - hit5 = [] - hit10 = [] - best_abs_delta = [] - - k_eff = min(int(k), int(library_vec.shape[0])) - if name == "cosine_exact": - sim_all = cosine_similarity(query_vec, library_vec) - knn_idx = np.argsort(-sim_all, axis=1)[:, :k_eff] - knn_sim = np.take_along_axis(sim_all, knn_idx, axis=1) - else: - library_norm = library_vec / np.clip(np.linalg.norm(library_vec, axis=1, keepdims=True), 1e-12, None) - query_norm = query_vec / np.clip(np.linalg.norm(query_vec, axis=1, keepdims=True), 1e-12, None) - if name == "faiss_flat_ip": - index = faiss.IndexFlatIP(library_norm.shape[1]) - index.add(library_norm.astype(np.float32)) - knn_sim, knn_idx = index.search(query_norm.astype(np.float32), k_eff) - elif name == "hnsw_cosine": - index = hnswlib.Index(space="cosine", dim=library_norm.shape[1]) - index.init_index(max_elements=library_norm.shape[0], ef_construction=100, M=16) - index.add_items(library_norm.astype(np.float32), np.arange(library_norm.shape[0])) - index.set_ef(max(50, k_eff)) - knn_idx, dist = index.knn_query(query_norm.astype(np.float32), k=k_eff) - knn_sim = 1.0 - dist - else: - raise ValueError(name) - - for i in range(query_df.shape[0]): - order = knn_idx[i] - top_sim = knn_sim[i] - top_targets = target_lib[order] - query_target = float(query_df.iloc[i]["target_log_burn_acres"]) - relevance = graded_relevance(query_target, top_targets) - ideal_relevance = np.sort(graded_relevance(query_target, target_lib))[::-1] - abs_delta = np.abs(top_targets - float(query_df.iloc[i]["target_log_burn_acres"])) - ndcg5.append(ndcg_at_k(relevance, ideal_relevance, 5)) - ndcg10.append(ndcg_at_k(relevance, ideal_relevance, 10)) - hit1.append(float(relevance[:1].max() >= 2.0)) - hit5.append(float(relevance[: min(5, k_eff)].max() >= 2.0)) - hit10.append(float(relevance[: min(10, k_eff)].max() >= 2.0)) - best_abs_delta.append(float(abs_delta.min())) - pred = float(np.mean(top_targets)) if mode == "mean" else weighted_prediction(top_sim, top_targets) - preds.append(pred) - rows.append( - { - "query_event_id": query_df.iloc[i]["Event_ID"], - "true_log_burn_acres": float(query_df.iloc[i]["target_log_burn_acres"]), - "pred_log_burn_acres": pred, - "backend": name, - "k": k, - "effective_k": k_eff, - "mode": mode, - "top_relevance": relevance.tolist(), - "best_abs_log_delta": float(abs_delta.min()), - } - ) - - pred_arr = np.asarray(preds, dtype=np.float64) - true_log = query_df["target_log_burn_acres"].to_numpy(dtype=np.float64) - true_acres = query_df["BurnBndAc"].to_numpy(dtype=np.float64) - pred_acres = np.exp(pred_arr) - metrics = { - "count": int(len(query_df)), - "log_mae": float(np.mean(np.abs(true_log - pred_arr))), - "log_rmse": rmse(true_log, pred_arr), - "log_r2": r2_score_manual(true_log, pred_arr), - "log_spearman": spearman_corr(true_log, pred_arr), - "log_median_ae": float(np.median(np.abs(true_log - pred_arr))), - "acres_mae": float(np.mean(np.abs(true_acres - pred_acres))), - "acres_rmse": rmse(true_acres, pred_acres), - "acres_median_ae": float(np.median(np.abs(true_acres - pred_acres))), - "acres_mape": mape(true_acres, pred_acres), - "ndcg_at_5": float(np.mean(ndcg5)) if ndcg5 else 0.0, - "ndcg_at_10": float(np.mean(ndcg10)) if ndcg10 else 0.0, - "hit_at_1_tol1": float(np.mean(hit1)) if hit1 else 0.0, - "hit_at_5_tol1": float(np.mean(hit5)) if hit5 else 0.0, - "hit_at_10_tol1": float(np.mean(hit10)) if hit10 else 0.0, - "mean_best_abs_log_delta_at_k": float(np.mean(best_abs_delta)) if best_abs_delta else 0.0, - } - return metrics, pd.DataFrame(rows) - - -def target_weight_vectors(train_vec: np.ndarray, val_vec: np.ndarray, test_vec: np.ndarray, target: np.ndarray) -> Tuple[np.ndarray, np.ndarray, np.ndarray]: - x = np.asarray(train_vec, dtype=np.float64) - y = np.asarray(target, dtype=np.float64) - y = y - y.mean() - x_centered = x - x.mean(axis=0, keepdims=True) - denom = np.clip(np.sqrt(np.sum(x_centered**2, axis=0)) * np.sqrt(np.sum(y**2)), 1e-12, None) - corr = np.abs(np.sum(x_centered * y[:, None], axis=0) / denom) - corr = np.nan_to_num(corr, nan=0.0, posinf=0.0, neginf=0.0) - if float(corr.max()) > 0: - corr = corr / float(corr.max()) - weights = (0.25 + corr).astype(np.float32) - return train_vec * weights, val_vec * weights, test_vec * weights - - -def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument("--event-table", type=Path, required=True) - parser.add_argument("--output-dir", type=Path, required=True) - parser.add_argument("--selection-metric", choices=("log_mae", "ndcg_at_10"), default="ndcg_at_10") - parser.add_argument("--feature-profile", choices=("all", "weather_fm"), default="all") - parser.add_argument("--fm-family", type=str, default="") - parser.add_argument("--seed", type=int, default=7) - args = parser.parse_args() - - df = pd.read_csv(args.event_table) - df["Ig_Date"] = pd.to_datetime(df["Ig_Date"]) - train_df, val_df, test_df = build_splits(df) - numeric_cols, categorical_cols = feature_columns(df, feature_profile=args.feature_profile) - if not numeric_cols and not categorical_cols: - raise SystemExit(f"No usable features found for profile={args.feature_profile}") - x_cols = numeric_cols + categorical_cols - pre = make_preprocessor(numeric_cols, categorical_cols) - train_vec = to_dense_float32(pre.fit_transform(train_df[x_cols])) - val_vec = to_dense_float32(pre.transform(val_df[x_cols])) - test_vec = to_dense_float32(pre.transform(test_df[x_cols])) - weighted_train_vec, weighted_val_vec, weighted_test_vec = target_weight_vectors( - train_vec, - val_vec, - test_vec, - train_df["target_log_burn_acres"].to_numpy(dtype=np.float64), - ) - vector_variants = { - "standard": (train_vec, val_vec, test_vec), - "target_weighted": (weighted_train_vec, weighted_val_vec, weighted_test_vec), - } - - candidate_validation: List[Dict[str, object]] = [] - best = None - best_score = None - best_val_rows = None - best_test_rows = None - for variant, (lib_vec, v_vec, _) in vector_variants.items(): - for backend in ["cosine_exact", "faiss_flat_ip", "hnsw_cosine"]: - for k in [1, 3, 5, 10]: - for mode in ["mean", "weighted"]: - val_metrics, val_rows = score_backend(backend, v_vec, lib_vec, val_df, train_df, k, mode) - candidate_validation.append({"variant": variant, "backend": backend, "k": k, "mode": mode, "val_metrics": val_metrics}) - score = float(val_metrics[args.selection_metric]) - better = score > best_score if args.selection_metric == "ndcg_at_10" and best_score is not None else score < best_score if best_score is not None else True - if better: - best_score = score - best = {"variant": variant, "backend": backend, "k": k, "mode": mode} - best_val_rows = val_rows - - assert best is not None - best_train_vec, _, best_test_vec = vector_variants[str(best["variant"])] - test_metrics, test_rows = score_backend(best["backend"], best_test_vec, best_train_vec, test_df, train_df, int(best["k"]), str(best["mode"])) - best_test_rows = test_rows - - args.output_dir.mkdir(parents=True, exist_ok=True) - if best_val_rows is not None: - best_val_rows.to_csv(args.output_dir / "val_retrieval_examples.csv", index=False) - if best_test_rows is not None: - best_test_rows.to_csv(args.output_dir / "test_retrieval_examples.csv", index=False) - - summary = { - "task_id": "wildfire_analog_retrieval_taskmodels", - "task_form": "event_level_retrieval_with_induced_outcome_error", - "event_table": str(args.event_table), - "output_dir": str(args.output_dir), - "feature_profile": args.feature_profile, - "seed": int(args.seed), - "split_sizes": { - "train": int(len(train_df)), - "val": int(len(val_df)), - "test": int(len(test_df)), - }, - "feature_columns": {"numeric": numeric_cols, "categorical": categorical_cols}, - "candidate_validation": candidate_validation, - "selected_retrieval": best, - "selection_metric": args.selection_metric, - "test_metrics": test_metrics, - "model_family": "popular_open_source_retrieval_backends_with_train_only_target_weighting", - "fm_family": (args.fm_family or "weather_fm_derived_features") if args.feature_profile == "weather_fm" else None, - } - (args.output_dir / "summary.json").write_text(json.dumps(summary, indent=2), encoding="utf-8") - print(json.dumps(summary, indent=2)) - - -if __name__ == "__main__": - main() diff --git a/experiments/raw_reference/task_scripts/run_extreme_heat_alphaearth_suite_seeded.py b/experiments/raw_reference/task_scripts/run_extreme_heat_alphaearth_suite_seeded.py deleted file mode 100644 index 9e551666cd3ab7d11b1ea885be3d61e95f9b5eaf..0000000000000000000000000000000000000000 --- a/experiments/raw_reference/task_scripts/run_extreme_heat_alphaearth_suite_seeded.py +++ /dev/null @@ -1,344 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import argparse -import json -import math -import re -import sys -from pathlib import Path -from typing import Dict, Iterable, List, Optional, Tuple - - -import os - -for _p in os.environ.get("WILDFIRE_FM_EXTRA_PYTHONPATH", "").split(os.pathsep): - if _p and _p not in sys.path: - sys.path.insert(0, _p) - -import numpy as np -import pandas as pd -from catboost import CatBoostRegressor -from lightgbm import LGBMRegressor -from netCDF4 import Dataset -from sklearn.linear_model import ElasticNet, Ridge -from sklearn.metrics import mean_absolute_error, mean_squared_error -from xgboost import XGBRegressor - - -PRED_RE = re.compile(r"pred_(\d{8})_(\d{2})\.nc$") -WEATHER_VARS = ["T2M", "QV2M", "TQV", "U10M", "V10M", "TS"] - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser() - parser.add_argument("--pred-root", type=Path, action="append", required=True) - parser.add_argument("--merra-root", type=Path, required=True) - parser.add_argument("--alphaearth-year-csv", type=Path, required=True) - parser.add_argument("--output-dir", type=Path, required=True) - parser.add_argument("--model-family", choices=("full", "lite"), default="full") - parser.add_argument("--alphaearth-prefix", type=str, default="alphaearth_") - parser.add_argument("--lat-min", type=float, default=24.0) - parser.add_argument("--lat-max", type=float, default=50.0) - parser.add_argument("--lon-min", type=float, default=-125.0) - parser.add_argument("--lon-max", type=float, default=-66.0) - parser.add_argument("--seed", type=int, default=7) - return parser.parse_args() - - -def choose_split(path: Path) -> Optional[str]: - name = path.name - if name == "output2022": - return "train" - if name == "output2024": - return "val" - if name == "output2025": - return "test" - return None - - -def parse_pred_timestamp(path: Path) -> Tuple[str, int]: - match = PRED_RE.match(path.name) - if not match: - raise ValueError(f"Unexpected prediction filename: {path}") - return match.group(1), int(match.group(2)) - - -def nearest_indices(src: np.ndarray, dst: np.ndarray) -> np.ndarray: - idx = np.searchsorted(dst, src) - idx = np.clip(idx, 0, len(dst) - 1) - prev_idx = np.clip(idx - 1, 0, len(dst) - 1) - choose_prev = np.abs(dst[prev_idx] - src) <= np.abs(dst[idx] - src) - return np.where(choose_prev, prev_idx, idx).astype(np.int64) - - -def build_grid_alignment(sample_pred: Path, sample_merra: Path, lat_min: float, lat_max: float, lon_min: float, lon_max: float) -> Dict[str, np.ndarray]: - with Dataset(sample_pred) as pred_ds, Dataset(sample_merra) as merra_ds: - pred_lat = np.asarray(pred_ds.variables["lat"][:], dtype=np.float64) - pred_lon = np.asarray(pred_ds.variables["lon"][:], dtype=np.float64) - merra_lat = np.asarray(merra_ds.variables["lat"][:], dtype=np.float64) - merra_lon = np.asarray(merra_ds.variables["lon"][:], dtype=np.float64) - - lat_mask = (pred_lat >= lat_min) & (pred_lat <= lat_max) - lon_mask = (pred_lon >= lon_min) & (pred_lon <= lon_max) - pred_lat_idx = np.flatnonzero(lat_mask) - pred_lon_idx = np.flatnonzero(lon_mask) - pred_lat_sel = pred_lat[pred_lat_idx] - pred_lon_sel = pred_lon[pred_lon_idx] - merra_lat_idx = nearest_indices(pred_lat_sel, merra_lat) - merra_lon_idx = nearest_indices(pred_lon_sel, merra_lon) - - return { - "pred_lat_idx": pred_lat_idx, - "pred_lon_idx": pred_lon_idx, - "merra_lat_idx": merra_lat_idx, - "merra_lon_idx": merra_lon_idx, - } - - -def feature_stats(arr: np.ndarray) -> Dict[str, float]: - return {"mean": float(np.mean(arr)), "max": float(np.max(arr)), "std": float(np.std(arr))} - - -def build_rows(pred_roots: Iterable[Path], merra_root: Path, alignment: Dict[str, np.ndarray]) -> pd.DataFrame: - rows: List[Dict[str, float]] = [] - for root in pred_roots: - split = choose_split(root) - if split is None: - continue - for path in sorted(root.glob("pred_*.nc")): - day, hour = parse_pred_timestamp(path) - if hour % 3 != 0: - continue - merra_path = merra_root / f"MERRA2_sfc_{day}.nc" - if not merra_path.exists(): - continue - time_index = hour // 3 - with Dataset(path) as pred_ds, Dataset(merra_path) as merra_ds: - date = pd.Timestamp(day) - record: Dict[str, float] = {"split": split, "hour": float(hour), "year": float(date.year), "date": day} - record["doy"] = float(date.dayofyear) - record["month"] = float(date.month) - for var in WEATHER_VARS: - pred_arr = np.asarray( - pred_ds.variables[var][0, alignment["pred_lat_idx"], alignment["pred_lon_idx"]], - dtype=np.float64, - ) - stats = feature_stats(pred_arr) - record[f"pred_{var.lower()}_mean"] = stats["mean"] - record[f"pred_{var.lower()}_max"] = stats["max"] - record[f"pred_{var.lower()}_std"] = stats["std"] - record["pred_wind_mean"] = float( - np.mean( - np.sqrt( - np.square(pred_ds.variables["U10M"][0, alignment["pred_lat_idx"], alignment["pred_lon_idx"]]) - + np.square(pred_ds.variables["V10M"][0, alignment["pred_lat_idx"], alignment["pred_lon_idx"]]) - ) - ) - ) - truth_t2m = np.asarray(merra_ds.variables["T2M"][time_index], dtype=np.float64)[ - np.ix_(alignment["merra_lat_idx"], alignment["merra_lon_idx"]) - ] - truth_ts = np.asarray(merra_ds.variables["TS"][time_index], dtype=np.float64)[ - np.ix_(alignment["merra_lat_idx"], alignment["merra_lon_idx"]) - ] - record["target_t2m_mean_c"] = float(np.mean(truth_t2m) - 273.15) - record["target_t2m_max_c"] = float(np.max(truth_t2m) - 273.15) - record["target_ts_mean_c"] = float(np.mean(truth_ts) - 273.15) - rows.append(record) - - if not rows: - raise SystemExit("No extreme-heat rows were built from the provided roots.") - df = pd.DataFrame(rows) - angle_day = 2.0 * np.pi * df["doy"].to_numpy(dtype=np.float64) / 366.0 - angle_hour = 2.0 * np.pi * df["hour"].to_numpy(dtype=np.float64) / 24.0 - df["doy_sin"] = np.sin(angle_day) - df["doy_cos"] = np.cos(angle_day) - df["hour_sin"] = np.sin(angle_hour) - df["hour_cos"] = np.cos(angle_hour) - return df - - -def drop_nonfinite_rows(df: pd.DataFrame, columns: List[str]) -> pd.DataFrame: - mask = np.ones(len(df), dtype=bool) - for col in columns: - mask &= np.isfinite(pd.to_numeric(df[col], errors="coerce").to_numpy(dtype=np.float64)) - return df.loc[mask].reset_index(drop=True) - - -def rmse(y_true: np.ndarray, y_pred: np.ndarray) -> float: - return float(math.sqrt(mean_squared_error(y_true, y_pred))) - - -def pearson_corr(y_true: np.ndarray, y_pred: np.ndarray) -> float: - a = np.asarray(y_true, dtype=np.float64) - b = np.asarray(y_pred, dtype=np.float64) - if a.size < 2 or np.allclose(a, a[0]) or np.allclose(b, b[0]): - return 0.0 - value = float(np.corrcoef(a, b)[0, 1]) - return value if np.isfinite(value) else 0.0 - - -def prf(y_true: np.ndarray, y_pred: np.ndarray, threshold: float) -> Dict[str, float]: - truth = np.asarray(y_true >= threshold) - pred = np.asarray(y_pred >= threshold) - tp = int(np.logical_and(pred, truth).sum()) - fp = int(np.logical_and(pred, ~truth).sum()) - fn = int(np.logical_and(~pred, truth).sum()) - precision = float(tp / (tp + fp)) if (tp + fp) else 0.0 - recall = float(tp / (tp + fn)) if (tp + fn) else 0.0 - f1 = float((2.0 * precision * recall) / (precision + recall)) if (precision + recall) else 0.0 - return {"precision": precision, "recall": recall, "f1": f1} - - -def evaluate(y_true: np.ndarray, y_pred: np.ndarray) -> Dict[str, float]: - return { - "count": int(y_true.shape[0]), - "rmse_c": rmse(y_true, y_pred), - "mae_c": float(mean_absolute_error(y_true, y_pred)), - "pearson_r": pearson_corr(y_true, y_pred), - } - - -def main() -> None: - args = parse_args() - pred_files = [path for root in args.pred_root for path in root.glob("pred_*.nc")] - if not pred_files: - raise SystemExit("No prediction files found.") - sample_pred = sorted(pred_files)[0] - sample_day, _ = parse_pred_timestamp(sample_pred) - sample_merra = args.merra_root / f"MERRA2_sfc_{sample_day}.nc" - if not sample_merra.exists(): - raise SystemExit(f"Sample MERRA file missing: {sample_merra}") - - alignment = build_grid_alignment( - sample_pred, - sample_merra, - lat_min=args.lat_min, - lat_max=args.lat_max, - lon_min=args.lon_min, - lon_max=args.lon_max, - ) - df = build_rows(args.pred_root, args.merra_root, alignment) - - alpha = pd.read_csv(args.alphaearth_year_csv) - alpha["source_year_key"] = pd.to_numeric(alpha["alphaearth_source_year"], errors="coerce").astype("Int64") - df["source_year_key"] = pd.to_numeric(df["year"], errors="coerce").clip(lower=2017, upper=2024).astype("Int64") - df = df.merge(alpha.drop(columns=[c for c in ["requested_year"] if c in alpha.columns]), on="source_year_key", how="left") - - feature_cols = [c for c in df.columns if c.startswith("pred_") or c in {"month", "doy_sin", "doy_cos", "hour_sin", "hour_cos"}] - feature_cols.extend(sorted([c for c in df.columns if c.startswith(args.alphaearth_prefix)])) - finite_cols = feature_cols + ["target_t2m_mean_c"] - df = drop_nonfinite_rows(df, finite_cols) - if df.empty: - raise SystemExit("Extreme-heat AlphaEarth suite has no finite rows after filtering.") - - train = df[df["split"] == "train"].copy() - val = df[df["split"] == "val"].copy() - test = df[df["split"] == "test"].copy() - if len(train) == 0 or len(val) == 0 or len(test) == 0: - raise SystemExit("Extreme-heat AlphaEarth suite is missing one of train/val/test.") - - x_train = train[feature_cols].to_numpy(dtype=np.float64) - x_val = val[feature_cols].to_numpy(dtype=np.float64) - x_test = test[feature_cols].to_numpy(dtype=np.float64) - y_train = train["target_t2m_mean_c"].to_numpy(dtype=np.float64) - y_val = val["target_t2m_mean_c"].to_numpy(dtype=np.float64) - y_test = test["target_t2m_mean_c"].to_numpy(dtype=np.float64) - - candidates: Dict[str, object] = { - "ridge": Ridge(alpha=1.0, random_state=args.seed), - "enet": ElasticNet(alpha=0.01, l1_ratio=0.2, random_state=args.seed, max_iter=10000), - } - if args.model_family == "full": - candidates.update( - { - "xgboost": XGBRegressor( - n_estimators=300, - max_depth=6, - learning_rate=0.05, - subsample=0.8, - colsample_bytree=0.8, - objective="reg:squarederror", - tree_method="hist", - random_state=args.seed, - n_jobs=8, - ), - "lightgbm": LGBMRegressor( - n_estimators=300, - learning_rate=0.05, - num_leaves=63, - subsample=0.8, - colsample_bytree=0.8, - random_state=args.seed, - n_jobs=8, - verbose=-1, - ), - "catboost": CatBoostRegressor( - iterations=400, - depth=8, - learning_rate=0.05, - loss_function="RMSE", - eval_metric="RMSE", - random_seed=args.seed, - verbose=False, - ), - } - ) - - candidate_rows = [] - best_name = None - best_model = None - best_rmse = None - for name, model in candidates.items(): - model.fit(x_train, y_train) - val_pred = np.asarray(model.predict(x_val), dtype=np.float64) - val_metrics = evaluate(y_val, val_pred) - candidate_rows.append({"model": name, "validation": val_metrics}) - if best_rmse is None or val_metrics["rmse_c"] < best_rmse: - best_name = name - best_model = model - best_rmse = val_metrics["rmse_c"] - - assert best_model is not None and best_name is not None - val_pred = np.asarray(best_model.predict(x_val), dtype=np.float64) - test_pred = np.asarray(best_model.predict(x_test), dtype=np.float64) - - thresholds = [27.0, 30.0, 33.0] - val_events = [{"threshold_c": t, **prf(y_val, val_pred, t)} for t in thresholds] - val_events = sorted(val_events, key=lambda row: (-row["f1"], -row["recall"], -row["precision"], row["threshold_c"])) - selected_event = val_events[0] - test_event = {"threshold_c": selected_event["threshold_c"], **prf(y_test, test_pred, selected_event["threshold_c"])} - - summary = { - "task_id": "extreme_heat_alphaearth", - "core_line": "extreme_heat", - "task_form": "continuous_temperature_forecast_with_secondary_exceedance_view", - "seed": int(args.seed), - "pred_roots": [str(path) for path in args.pred_root], - "merra_root": str(args.merra_root), - "alphaearth_year_csv": str(args.alphaearth_year_csv), - "model_family": args.model_family, - "feature_columns": feature_cols, - "alphaearth_feature_count": int(sum(c.startswith(args.alphaearth_prefix) for c in feature_cols)), - "candidate_validation": candidate_rows, - "selected_model": best_name, - "validation_metrics": evaluate(y_val, val_pred), - "test_metrics": evaluate(y_test, test_pred), - "selected_event_candidate": selected_event, - "selected_event_candidate_test": test_event, - "selection_rule": "same heat benchmark; choose regressor by validation RMSE and choose exceedance threshold by validation F1", - "tmt_policy": { - "task": "extreme_heat", - "metric": "continuous RMSE/MAE with thresholded exceedance as a secondary event policy", - "tolerance": "none for continuous headline; event threshold only for operational view" - }, - } - - args.output_dir.mkdir(parents=True, exist_ok=True) - (args.output_dir / "summary.json").write_text(json.dumps(summary, indent=2), encoding="utf-8") - print(json.dumps(summary, indent=2)) - - -if __name__ == "__main__": - main() diff --git a/experiments/raw_reference/task_scripts/run_final_area_taskmodel_seeded.py b/experiments/raw_reference/task_scripts/run_final_area_taskmodel_seeded.py deleted file mode 100644 index 2709fa33720640d948dff9b6a65a96ebc23a0fc1..0000000000000000000000000000000000000000 --- a/experiments/raw_reference/task_scripts/run_final_area_taskmodel_seeded.py +++ /dev/null @@ -1,353 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import argparse -import json -import sys -from pathlib import Path -from typing import Dict, List, Tuple - - -import os - -for _p in os.environ.get("WILDFIRE_FM_EXTRA_PYTHONPATH", "").split(os.pathsep): - if _p and _p not in sys.path: - sys.path.insert(0, _p) - -import numpy as np -import pandas as pd -from catboost import CatBoostRegressor -from lightgbm import LGBMRegressor -from sklearn.compose import ColumnTransformer -from sklearn.impute import SimpleImputer -from sklearn.linear_model import ElasticNet -from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score -from sklearn.pipeline import Pipeline -from sklearn.preprocessing import OneHotEncoder, StandardScaler -from xgboost import XGBRegressor - - -DROP_COLUMNS = { - "Event_ID", - "Incid_Name", - "incident_name_norm", - "wfigs_name", - "Ig_Date", - "weather_date", - "BurnBndAc", - "target_log_burn_acres", -} - -CATEGORICAL_COLUMNS = [ - "Incid_Type", - "state_abbr", - "county_name", - "wfigs_match_type", -] - - -def rmse(y_true: np.ndarray, y_pred: np.ndarray) -> float: - return float(np.sqrt(mean_squared_error(y_true, y_pred))) - - -def mape(y_true: np.ndarray, y_pred: np.ndarray) -> float: - denom = np.clip(np.asarray(y_true, dtype=np.float64), 1e-6, None) - frac = np.abs(np.asarray(y_true, dtype=np.float64) - np.asarray(y_pred, dtype=np.float64)) / denom - return float(np.mean(frac)) - - -def spearman_corr(y_true: np.ndarray, y_pred: np.ndarray) -> float: - a = pd.Series(np.asarray(y_true, dtype=np.float64)) - b = pd.Series(np.asarray(y_pred, dtype=np.float64)) - value = a.corr(b, method="spearman") - return float(value) if pd.notna(value) else 0.0 - - -def build_splits(df: pd.DataFrame) -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]: - ordered = df.sort_values("Ig_Date").reset_index(drop=True) - n = len(ordered) - train_end = max(int(round(n * 0.6)), 1) - val_end = max(int(round(n * 0.8)), train_end + 1) - val_end = min(val_end, n - 1) if n >= 3 else n - train = ordered.iloc[:train_end].copy() - val = ordered.iloc[train_end:val_end].copy() - test = ordered.iloc[val_end:].copy() - if len(val) == 0 and len(test) > 1: - val = test.iloc[:1].copy() - test = test.iloc[1:].copy() - return train, val, test - - -def feature_columns(df: pd.DataFrame, feature_profile: str = "all") -> Tuple[List[str], List[str]]: - categorical = [c for c in CATEGORICAL_COLUMNS if c in df.columns] - numeric = [] - for col in df.columns: - if col in DROP_COLUMNS or col in categorical: - continue - if pd.api.types.is_numeric_dtype(df[col]): - numeric.append(col) - if feature_profile == "weather_fm": - numeric = [c for c in numeric if c.startswith("weather_")] - categorical = [] - return numeric, categorical - - -def make_sparse_preprocessor(numeric_cols: List[str], categorical_cols: List[str]) -> ColumnTransformer: - return ColumnTransformer( - transformers=[ - ( - "num", - Pipeline( - steps=[ - ("impute", SimpleImputer(strategy="median")), - ("scale", StandardScaler()), - ] - ), - numeric_cols, - ), - ( - "cat", - Pipeline( - steps=[ - ("impute", SimpleImputer(strategy="most_frequent")), - ("onehot", OneHotEncoder(handle_unknown="ignore")), - ] - ), - categorical_cols, - ), - ], - remainder="drop", - ) - - -def prepare_catboost_frames( - train_df: pd.DataFrame, - val_df: pd.DataFrame, - test_df: pd.DataFrame, - numeric_cols: List[str], - categorical_cols: List[str], -) -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]: - medians = {c: float(train_df[c].median()) for c in numeric_cols} - modes = { - c: str(train_df[c].mode(dropna=True).iloc[0]) if not train_df[c].mode(dropna=True).empty else "missing" - for c in categorical_cols - } - - def _prep(frame: pd.DataFrame) -> pd.DataFrame: - out = frame[numeric_cols + categorical_cols].copy() - for col in numeric_cols: - out[col] = pd.to_numeric(out[col], errors="coerce").fillna(medians[col]) - for col in categorical_cols: - out[col] = out[col].astype("string").fillna(modes[col]).astype(str) - return out - - return _prep(train_df), _prep(val_df), _prep(test_df) - - -def evaluate_split(frame: pd.DataFrame, pred_log: np.ndarray) -> Dict[str, float]: - true_log = frame["target_log_burn_acres"].to_numpy(dtype=np.float64) - true_acres = frame["BurnBndAc"].to_numpy(dtype=np.float64) - pred_log = np.asarray(pred_log, dtype=np.float64) - pred_acres = np.exp(pred_log) - return { - "count": int(len(frame)), - "log_mae": float(mean_absolute_error(true_log, pred_log)), - "log_rmse": rmse(true_log, pred_log), - "log_r2": float(r2_score(true_log, pred_log)) if len(frame) > 1 else 0.0, - "log_spearman": spearman_corr(true_log, pred_log), - "log_median_ae": float(np.median(np.abs(true_log - pred_log))), - "acres_mae": float(mean_absolute_error(true_acres, pred_acres)), - "acres_rmse": rmse(true_acres, pred_acres), - "acres_median_ae": float(np.median(np.abs(true_acres - pred_acres))), - "acres_mape": mape(true_acres, pred_acres), - } - - -def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument("--event-table", type=Path, required=True) - parser.add_argument("--output-dir", type=Path, required=True) - parser.add_argument("--feature-profile", choices=("all", "weather_fm"), default="all") - parser.add_argument("--model-family", choices=("full", "lite"), default="full") - parser.add_argument("--fm-family", type=str, default="") - parser.add_argument("--seed", type=int, default=7) - args = parser.parse_args() - - df = pd.read_csv(args.event_table) - df["Ig_Date"] = pd.to_datetime(df["Ig_Date"]) - train_df, val_df, test_df = build_splits(df) - numeric_cols, categorical_cols = feature_columns(df, feature_profile=args.feature_profile) - if not numeric_cols and not categorical_cols: - raise SystemExit(f"No usable features found for profile={args.feature_profile}") - x_cols = numeric_cols + categorical_cols - - pre = make_sparse_preprocessor(numeric_cols, categorical_cols) - x_train = pre.fit_transform(train_df[x_cols]) - x_val = pre.transform(val_df[x_cols]) - x_test = pre.transform(test_df[x_cols]) - y_train = train_df["target_log_burn_acres"].to_numpy(dtype=np.float64) - - cat_train, cat_val, cat_test = prepare_catboost_frames(train_df, val_df, test_df, numeric_cols, categorical_cols) - cat_feature_idx = list(range(len(numeric_cols), len(numeric_cols) + len(categorical_cols))) - - candidates: List[Tuple[str, object, str]] = [ - ( - "enet", - ElasticNet(alpha=0.01, l1_ratio=0.2, random_state=args.seed, max_iter=10000), - "sparse", - ), - ] - if args.model_family == "full": - candidates.extend( - [ - ( - "xgboost", - XGBRegressor( - n_estimators=400, - max_depth=6, - learning_rate=0.05, - subsample=0.8, - colsample_bytree=0.8, - reg_lambda=1.0, - objective="reg:squarederror", - tree_method="hist", - random_state=args.seed, - n_jobs=8, - ), - "sparse", - ), - ( - "lightgbm", - LGBMRegressor( - n_estimators=400, - learning_rate=0.05, - num_leaves=63, - subsample=0.8, - colsample_bytree=0.8, - reg_lambda=1.0, - random_state=args.seed, - n_jobs=8, - verbose=-1, - ), - "sparse", - ), - ( - "catboost", - CatBoostRegressor( - iterations=500, - depth=8, - learning_rate=0.05, - loss_function="RMSE", - eval_metric="RMSE", - random_seed=args.seed, - verbose=False, - ), - "cat", - ), - ] - ) - - candidate_validation: List[Dict[str, object]] = [] - best_name = None - best_kind = None - best_model = None - best_score = None - - for name, model, kind in candidates: - if kind == "sparse": - model.fit(x_train, y_train) - val_pred = model.predict(x_val) - else: - model.fit(cat_train, y_train, cat_features=cat_feature_idx, eval_set=(cat_val, val_df["target_log_burn_acres"]), use_best_model=False) - val_pred = model.predict(cat_val) - val_metrics = evaluate_split(val_df, val_pred) - candidate_validation.append({"model_name": name, "val_metrics": val_metrics}) - score = float(val_metrics["log_mae"]) - if best_score is None or score < best_score: - best_score = score - best_name = name - best_kind = kind - best_model = model - - assert best_model is not None and best_name is not None and best_kind is not None - - combined_train = pd.concat([train_df, val_df], ignore_index=True) - if best_kind == "sparse": - x_combined = pre.fit_transform(combined_train[x_cols]) - x_train_final = pre.transform(train_df[x_cols]) - x_val_final = pre.transform(val_df[x_cols]) - x_test_final = pre.transform(test_df[x_cols]) - best_model.fit(x_combined, combined_train["target_log_burn_acres"].to_numpy(dtype=np.float64)) - train_pred = best_model.predict(x_train_final) - val_pred = best_model.predict(x_val_final) - test_pred = best_model.predict(x_test_final) - else: - cat_combined, cat_train_final, cat_test_final = prepare_catboost_frames( - combined_train, train_df, test_df, numeric_cols, categorical_cols - ) - cat_val_final = prepare_catboost_frames(val_df, val_df, val_df, numeric_cols, categorical_cols)[0] - best_model.fit( - cat_combined, - combined_train["target_log_burn_acres"].to_numpy(dtype=np.float64), - cat_features=cat_feature_idx, - use_best_model=False, - ) - train_pred = best_model.predict(cat_train_final) - val_pred = best_model.predict(cat_val_final) - test_pred = best_model.predict(cat_test_final) - - args.output_dir.mkdir(parents=True, exist_ok=True) - pred_df = pd.concat( - [ - train_df.assign(split="train", pred_log_burn_acres=train_pred, pred_burn_acres=np.exp(train_pred)), - val_df.assign(split="val", pred_log_burn_acres=val_pred, pred_burn_acres=np.exp(val_pred)), - test_df.assign(split="test", pred_log_burn_acres=test_pred, pred_burn_acres=np.exp(test_pred)), - ], - axis=0, - ignore_index=True, - ) - pred_path = args.output_dir / "predictions.csv" - pred_df.to_csv(pred_path, index=False) - - summary = { - "task_id": "wildfire_final_area_scalar_taskmodels", - "task_form": "event_level_regression", - "event_table": str(args.event_table), - "output_dir": str(args.output_dir), - "feature_profile": args.feature_profile, - "seed": int(args.seed), - "benchmark_protocol": "fm_lite_protocol" if args.feature_profile == "weather_fm" and args.model_family == "lite" else "standard_protocol", - "split_sizes": { - "train": int(len(train_df)), - "val": int(len(val_df)), - "test": int(len(test_df)), - }, - "feature_columns": { - "numeric": numeric_cols, - "categorical": categorical_cols, - }, - "candidate_validation": candidate_validation, - "selected_model": best_name, - "train_metrics": evaluate_split(train_df, train_pred), - "val_metrics": evaluate_split(val_df, val_pred), - "test_metrics": evaluate_split(test_df, test_pred), - "headline_metrics": { - "log_mae": float(evaluate_split(test_df, test_pred)["log_mae"]), - "log_rmse": float(evaluate_split(test_df, test_pred)["log_rmse"]), - "log_spearman": float(evaluate_split(test_df, test_pred)["log_spearman"]), - }, - "predictions_path": str(pred_path), - "model_family": "lightweight_linear_task_heads" if args.model_family == "lite" else "popular_open_source_task_models", - "fm_family": (args.fm_family or "weather_fm_derived_features") if args.feature_profile == "weather_fm" else None, - "tmt_policy": { - "task": "final_burned_area", - "metric": "log-area regression error with rank agreement", - "tolerance": "secondary magnitude-band interpretation only", - }, - } - (args.output_dir / "summary.json").write_text(json.dumps(summary, indent=2), encoding="utf-8") - print(json.dumps(summary, indent=2)) - - -if __name__ == "__main__": - main() diff --git a/experiments/raw_reference/task_scripts/run_smoke_pm25_alphaearth_suite_seeded.py b/experiments/raw_reference/task_scripts/run_smoke_pm25_alphaearth_suite_seeded.py deleted file mode 100644 index cd91da61e9891feab6438eee9c052fa286b545eb..0000000000000000000000000000000000000000 --- a/experiments/raw_reference/task_scripts/run_smoke_pm25_alphaearth_suite_seeded.py +++ /dev/null @@ -1,306 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import argparse -import json -import math -import sys -from pathlib import Path -from typing import Dict, List, Tuple - - -import os - -for _p in os.environ.get("WILDFIRE_FM_EXTRA_PYTHONPATH", "").split(os.pathsep): - if _p and _p not in sys.path: - sys.path.insert(0, _p) - -import numpy as np -import pandas as pd -from catboost import CatBoostRegressor -from lightgbm import LGBMRegressor -from sklearn.metrics import mean_absolute_error, mean_squared_error -from xgboost import XGBRegressor - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser() - parser.add_argument("--aqs-daily", type=Path, required=True) - parser.add_argument("--output-dir", type=Path, required=True) - parser.add_argument("--exceedance-threshold", type=float, default=35.0) - parser.add_argument("--alphaearth-prefix", type=str, default="alphaearth_") - parser.add_argument("--seed", type=int, default=7) - return parser.parse_args() - - -def assign_split(ts: pd.Timestamp) -> str: - year = int(ts.year) - if year <= 2023: - return "train" - if year == 2024: - return "val" - if year == 2025: - return "test" - return "other" - - -def rmse(y_true: np.ndarray, y_pred: np.ndarray) -> float: - return float(math.sqrt(mean_squared_error(y_true, y_pred))) - - -def pearson_corr(y_true: np.ndarray, y_pred: np.ndarray) -> float: - a = np.asarray(y_true, dtype=np.float64) - b = np.asarray(y_pred, dtype=np.float64) - if a.size < 2 or np.allclose(a, a[0]) or np.allclose(b, b[0]): - return 0.0 - value = float(np.corrcoef(a, b)[0, 1]) - return value if np.isfinite(value) else 0.0 - - -def prf(y_true: np.ndarray, y_pred: np.ndarray, threshold: float) -> Dict[str, float]: - truth = np.asarray(y_true >= threshold) - pred = np.asarray(y_pred >= threshold) - tp = int(np.logical_and(pred, truth).sum()) - fp = int(np.logical_and(pred, ~truth).sum()) - fn = int(np.logical_and(~pred, truth).sum()) - precision = float(tp / (tp + fp)) if (tp + fp) else 0.0 - recall = float(tp / (tp + fn)) if (tp + fn) else 0.0 - f1 = float((2.0 * precision * recall) / (precision + recall)) if (precision + recall) else 0.0 - return {"precision": precision, "recall": recall, "f1": f1} - - -def evaluate_frame(frame: pd.DataFrame, pred_col: str, threshold: float) -> Dict[str, float]: - y_true = frame["pm25_mean"].to_numpy(dtype=np.float64) - y_pred = frame[pred_col].to_numpy(dtype=np.float64) - event = prf(y_true, y_pred, threshold) - bias = np.asarray(y_pred - y_true, dtype=np.float64) - denom = float(np.sum(y_true)) - return { - "count": int(len(frame)), - "rmse": rmse(y_true, y_pred), - "mae": float(mean_absolute_error(y_true, y_pred)), - "mean_bias": float(np.mean(bias)), - "normalized_mean_bias": float(np.sum(bias) / denom) if abs(denom) > 1e-12 else 0.0, - "pearson_r": pearson_corr(y_true, y_pred), - "event_precision": event["precision"], - "event_recall": event["recall"], - "event_f1": event["f1"], - } - - -def tune_event_shift(val_frame: pd.DataFrame, pred_col: str, threshold: float) -> Dict[str, float]: - best = None - for delta in np.linspace(-5.0, 15.0, 161): - shifted = val_frame.copy() - shifted["_shifted_pred"] = shifted[pred_col] + float(delta) - metrics = evaluate_frame(shifted, "_shifted_pred", threshold) - score = (metrics["event_f1"], metrics["event_recall"], -abs(float(delta))) - if best is None or score > best["score"]: - best = {"delta": float(delta), "metrics": metrics, "score": score} - assert best is not None - return {"delta": best["delta"], "val_event_calibrated_metrics": best["metrics"]} - - -def build_features(df: pd.DataFrame) -> pd.DataFrame: - df = df.sort_values(["site_key", "date"]).reset_index(drop=True).copy() - df["doy"] = df["date"].dt.dayofyear.astype(np.int32) - df["month"] = df["date"].dt.month.astype(np.int32) - df["doy_sin"] = np.sin(2.0 * np.pi * df["doy"] / 366.0) - df["doy_cos"] = np.cos(2.0 * np.pi * df["doy"] / 366.0) - grp = df.groupby("site_key", sort=False) - for lag in [1, 2, 3, 7]: - df[f"lag{lag}_pm25"] = grp["pm25_mean"].shift(lag) - df["roll3_prev"] = grp["pm25_mean"].rolling(3, min_periods=1).mean().reset_index(level=0, drop=True).shift(1) - df["roll7_prev"] = grp["pm25_mean"].rolling(7, min_periods=1).mean().reset_index(level=0, drop=True).shift(1) - return df - - -def prepare_frames(df: pd.DataFrame, alphaearth_prefix: str) -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame, List[str]]: - df = build_features(df) - train = df[df["split"] == "train"].copy() - val = df[df["split"] == "val"].copy() - test = df[df["split"] == "test"].copy() - - site_mean = train.groupby("site_key")["pm25_mean"].mean() - global_mean = float(train["pm25_mean"].mean()) - for frame in [train, val, test]: - frame["site_climo"] = frame["site_key"].map(site_mean).fillna(global_mean) - for frame in [train, val, test]: - for col in ["lag1_pm25", "lag2_pm25", "lag3_pm25", "lag7_pm25", "roll3_prev", "roll7_prev"]: - frame[col] = pd.to_numeric(frame[col], errors="coerce").fillna(frame["site_climo"]) - - feature_cols = [ - "latitude", - "longitude", - "obs_count", - "site_climo", - "lag1_pm25", - "lag2_pm25", - "lag3_pm25", - "lag7_pm25", - "roll3_prev", - "roll7_prev", - "doy_sin", - "doy_cos", - "month", - ] - alpha_cols = [ - c for c in df.columns - if c.startswith(alphaearth_prefix) and pd.api.types.is_numeric_dtype(df[c]) - ] - feature_cols.extend(sorted(alpha_cols)) - medians = train[feature_cols].median(numeric_only=True).fillna(0.0) - for frame in [train, val, test]: - frame.loc[:, feature_cols] = frame[feature_cols].fillna(medians) - frame.loc[:, feature_cols] = frame[feature_cols].fillna(0.0) - return train, val, test, feature_cols - - -def main() -> None: - args = parse_args() - df = pd.read_csv(args.aqs_daily, compression="infer", low_memory=False) - df["date"] = pd.to_datetime(df["date_gmt"], errors="coerce") - df["pm25_mean"] = pd.to_numeric(df["pm25_mean"], errors="coerce") - df["pm25_max"] = pd.to_numeric(df["pm25_max"], errors="coerce") - df["obs_count"] = pd.to_numeric(df["obs_count"], errors="coerce") - df = df.dropna(subset=["date", "site_key", "pm25_mean"]).copy() - df = ( - df.groupby(["date", "site_key"], as_index=False) - .agg( - latitude=("Latitude", "first"), - longitude=("Longitude", "first"), - pm25_mean=("pm25_mean", "mean"), - pm25_max=("pm25_max", "max"), - obs_count=("obs_count", "sum"), - **{c: (c, "first") for c in df.columns if c.startswith(args.alphaearth_prefix)}, - ) - .sort_values(["site_key", "date"]) - .reset_index(drop=True) - ) - df["split"] = df["date"].map(assign_split) - df = df[df["split"].isin(["train", "val", "test"])].copy() - - train, val, test, feature_cols = prepare_frames(df, alphaearth_prefix=args.alphaearth_prefix) - y_train = train["pm25_mean"].to_numpy(dtype=np.float64) - - candidates = { - "xgboost": XGBRegressor( - n_estimators=300, - max_depth=8, - learning_rate=0.05, - subsample=0.8, - colsample_bytree=0.8, - objective="reg:squarederror", - tree_method="hist", - random_state=args.seed, - n_jobs=8, - ), - "lightgbm": LGBMRegressor( - n_estimators=300, - learning_rate=0.05, - num_leaves=127, - subsample=0.8, - colsample_bytree=0.8, - random_state=args.seed, - n_jobs=8, - verbose=-1, - ), - "catboost": CatBoostRegressor( - iterations=400, - depth=8, - learning_rate=0.05, - loss_function="RMSE", - eval_metric="RMSE", - random_seed=args.seed, - verbose=False, - ), - } - - candidate_validation: List[Dict[str, object]] = [] - best_name = None - best_model = None - best_score = None - for name, model in candidates.items(): - if name == "catboost": - model.fit(train[feature_cols], y_train, use_best_model=False) - val_pred = model.predict(val[feature_cols]) - else: - model.fit(train[feature_cols].to_numpy(dtype=np.float32), y_train) - val_pred = model.predict(val[feature_cols].to_numpy(dtype=np.float32)) - val_frame = val.copy() - val_frame["pred"] = val_pred - metrics = evaluate_frame(val_frame, "pred", args.exceedance_threshold) - candidate_validation.append({"candidate": name, "val_metrics": metrics}) - score = float(metrics["rmse"]) - if best_score is None or score < best_score: - best_score = score - best_name = name - best_model = model - - assert best_name is not None and best_model is not None - combined = pd.concat([train, val], ignore_index=True) - if best_name == "catboost": - best_model.fit(combined[feature_cols], combined["pm25_mean"].to_numpy(dtype=np.float64), use_best_model=False) - train_pred = best_model.predict(train[feature_cols]) - val_pred = best_model.predict(val[feature_cols]) - test_pred = best_model.predict(test[feature_cols]) - else: - best_model.fit(combined[feature_cols].to_numpy(dtype=np.float32), combined["pm25_mean"].to_numpy(dtype=np.float64)) - train_pred = best_model.predict(train[feature_cols].to_numpy(dtype=np.float32)) - val_pred = best_model.predict(val[feature_cols].to_numpy(dtype=np.float32)) - test_pred = best_model.predict(test[feature_cols].to_numpy(dtype=np.float32)) - - args.output_dir.mkdir(parents=True, exist_ok=True) - pred_df = pd.concat( - [ - train.assign(pred_pm25=train_pred), - val.assign(pred_pm25=val_pred), - test.assign(pred_pm25=test_pred), - ], - ignore_index=True, - ) - pred_path = args.output_dir / "predictions.csv.gz" - pred_df.to_csv(pred_path, index=False, compression="gzip") - - train_eval = train.assign(pred=train_pred) - val_eval = val.assign(pred=val_pred) - test_eval = test.assign(pred=test_pred) - event_shift = tune_event_shift(val_eval, "pred", args.exceedance_threshold) - delta = event_shift["delta"] - train_event_eval = train_eval.assign(pred_event_calibrated=train_eval["pred"] + delta) - val_event_eval = val_eval.assign(pred_event_calibrated=val_eval["pred"] + delta) - test_event_eval = test_eval.assign(pred_event_calibrated=test_eval["pred"] + delta) - - summary = { - "task_id": "smoke_pm25_alphaearth", - "task_form": "station_daily_regression", - "aqs_daily": str(args.aqs_daily), - "output_dir": str(args.output_dir), - "seed": int(args.seed), - "feature_columns": feature_cols, - "alphaearth_feature_count": int(sum(c.startswith(args.alphaearth_prefix) for c in feature_cols)), - "split_sizes": {"train": int(len(train)), "val": int(len(val)), "test": int(len(test))}, - "candidate_validation": candidate_validation, - "selected_model": best_name, - "train_metrics": evaluate_frame(train_eval, "pred", args.exceedance_threshold), - "val_metrics": evaluate_frame(val_eval, "pred", args.exceedance_threshold), - "test_metrics": evaluate_frame(test_eval, "pred", args.exceedance_threshold), - "event_calibration": { - "delta": float(delta), - "val_metrics": event_shift["val_event_calibrated_metrics"], - "test_metrics": evaluate_frame(test_event_eval, "pred_event_calibrated", args.exceedance_threshold), - }, - "predictions_path": str(pred_path), - "selection_rule": "same smoke benchmark; choose task-specific regressor by validation RMSE, then calibrate exceedance on validation only", - "tmt_policy": { - "task": "smoke_pm25", - "metric": "continuous RMSE/MAE with thresholded exceedance PRF", - "tolerance": "secondary event policy only", - }, - } - (args.output_dir / "summary.json").write_text(json.dumps(summary, indent=2), encoding="utf-8") - print(json.dumps(summary, indent=2)) - - -if __name__ == "__main__": - main() diff --git a/experiments/raw_reference/task_scripts/run_smoke_pm25_attached_fm_suite_seeded.py b/experiments/raw_reference/task_scripts/run_smoke_pm25_attached_fm_suite_seeded.py deleted file mode 100644 index 9ecc6057a28b69b7701ffe991b46b967d0a638b6..0000000000000000000000000000000000000000 --- a/experiments/raw_reference/task_scripts/run_smoke_pm25_attached_fm_suite_seeded.py +++ /dev/null @@ -1,231 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import argparse -import json -import math -import sys -from pathlib import Path -from typing import Dict, List - - -import os - -for _p in os.environ.get("WILDFIRE_FM_EXTRA_PYTHONPATH", "").split(os.pathsep): - if _p and _p not in sys.path: - sys.path.insert(0, _p) - -import numpy as np -import pandas as pd -from catboost import CatBoostRegressor -from lightgbm import LGBMRegressor -from sklearn.linear_model import ElasticNet, Ridge -from sklearn.metrics import mean_absolute_error, mean_squared_error -from xgboost import XGBRegressor - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser() - parser.add_argument("--attached-csv", type=Path, required=True) - parser.add_argument("--output-dir", type=Path, required=True) - parser.add_argument("--fm-prefix", type=str, required=True) - parser.add_argument("--fm-family", type=str, required=True) - parser.add_argument("--model-family", choices=("full", "lite"), default="lite") - parser.add_argument("--exceedance-threshold", type=float, default=35.0) - parser.add_argument("--seed", type=int, default=7) - return parser.parse_args() - - -def rmse(y_true: np.ndarray, y_pred: np.ndarray) -> float: - return float(math.sqrt(mean_squared_error(y_true, y_pred))) - - -def pearson_corr(y_true: np.ndarray, y_pred: np.ndarray) -> float: - a = np.asarray(y_true, dtype=np.float64) - b = np.asarray(y_pred, dtype=np.float64) - if a.size < 2 or np.allclose(a, a[0]) or np.allclose(b, b[0]): - return 0.0 - value = float(np.corrcoef(a, b)[0, 1]) - return value if np.isfinite(value) else 0.0 - - -def prf(y_true: np.ndarray, y_pred: np.ndarray, threshold: float) -> Dict[str, float]: - truth = np.asarray(y_true >= threshold) - pred = np.asarray(y_pred >= threshold) - tp = int(np.logical_and(pred, truth).sum()) - fp = int(np.logical_and(pred, ~truth).sum()) - fn = int(np.logical_and(~pred, truth).sum()) - precision = float(tp / (tp + fp)) if (tp + fp) else 0.0 - recall = float(tp / (tp + fn)) if (tp + fn) else 0.0 - f1 = float((2.0 * precision * recall) / (precision + recall)) if (precision + recall) else 0.0 - return {"precision": precision, "recall": recall, "f1": f1} - - -def evaluate_frame(frame: pd.DataFrame, pred_col: str, threshold: float) -> Dict[str, float]: - y_true = frame["pm25_mean"].to_numpy(dtype=np.float64) - y_pred = frame[pred_col].to_numpy(dtype=np.float64) - event = prf(y_true, y_pred, threshold) - bias = np.asarray(y_pred - y_true, dtype=np.float64) - denom = float(np.sum(y_true)) - return { - "count": int(len(frame)), - "rmse": rmse(y_true, y_pred), - "mae": float(mean_absolute_error(y_true, y_pred)), - "mean_bias": float(np.mean(bias)), - "normalized_mean_bias": float(np.sum(bias) / denom) if abs(denom) > 1e-12 else 0.0, - "pearson_r": pearson_corr(y_true, y_pred), - "event_precision": event["precision"], - "event_recall": event["recall"], - "event_f1": event["f1"], - } - - -def main() -> None: - args = parse_args() - df = pd.read_csv(args.attached_csv) - df["date"] = pd.to_datetime(df["date_gmt"], errors="coerce") - df["pm25_mean"] = pd.to_numeric(df["pm25_mean"], errors="coerce") - df = df.dropna(subset=["date", "pm25_mean"]).copy() - - feature_cols = [c for c in df.columns if c.startswith(args.fm_prefix)] - feature_cols = [c for c in feature_cols if pd.api.types.is_numeric_dtype(df[c])] - if not feature_cols: - raise SystemExit(f"No numeric FM feature columns found with prefix {args.fm_prefix}") - - split_map = {"2020": "train", "2021": "train", "2022": "train", "2024": "val", "2025": "test"} - df["split"] = df["date"].dt.year.astype(str).map(split_map) - df = df[df["split"].isin(["train", "val", "test"])].copy() - train = df[df["split"] == "train"].copy() - val = df[df["split"] == "val"].copy() - test = df[df["split"] == "test"].copy() - if len(train) == 0 or len(val) == 0 or len(test) == 0: - raise SystemExit("Attached FM smoke table is missing one of train/val/test.") - - medians = train[feature_cols].median(numeric_only=True).fillna(0.0) - train.loc[:, feature_cols] = train[feature_cols].fillna(medians) - val.loc[:, feature_cols] = val[feature_cols].fillna(medians) - test.loc[:, feature_cols] = test[feature_cols].fillna(medians) - train.loc[:, feature_cols] = train[feature_cols].fillna(0.0) - val.loc[:, feature_cols] = val[feature_cols].fillna(0.0) - test.loc[:, feature_cols] = test[feature_cols].fillna(0.0) - - y_train = train["pm25_mean"].to_numpy(dtype=np.float64) - candidates: Dict[str, object] = { - "ridge": Ridge(alpha=1.0, random_state=args.seed), - "enet": ElasticNet(alpha=0.01, l1_ratio=0.2, random_state=args.seed, max_iter=10000), - } - if args.model_family == "full": - candidates.update( - { - "xgboost": XGBRegressor( - n_estimators=300, - max_depth=8, - learning_rate=0.05, - subsample=0.8, - colsample_bytree=0.8, - objective="reg:squarederror", - tree_method="hist", - random_state=args.seed, - n_jobs=8, - ), - "lightgbm": LGBMRegressor( - n_estimators=300, - learning_rate=0.05, - num_leaves=127, - subsample=0.8, - colsample_bytree=0.8, - random_state=args.seed, - n_jobs=8, - verbose=-1, - ), - "catboost": CatBoostRegressor( - iterations=400, - depth=8, - learning_rate=0.05, - loss_function="RMSE", - eval_metric="RMSE", - random_seed=args.seed, - verbose=False, - ), - } - ) - - candidate_validation: List[Dict[str, object]] = [] - best_name = None - best_model = None - best_score = None - for name, model in candidates.items(): - if name == "catboost": - model.fit(train[feature_cols], y_train, use_best_model=False) - val_pred = model.predict(val[feature_cols]) - else: - model.fit(train[feature_cols].to_numpy(dtype=np.float32), y_train) - val_pred = model.predict(val[feature_cols].to_numpy(dtype=np.float32)) - val_frame = val.copy() - val_frame["pred"] = val_pred - metrics = evaluate_frame(val_frame, "pred", args.exceedance_threshold) - candidate_validation.append({"candidate": name, "val_metrics": metrics}) - score = float(metrics["rmse"]) - if best_score is None or score < best_score: - best_score = score - best_name = name - best_model = model - - assert best_name is not None and best_model is not None - combined = pd.concat([train, val], ignore_index=True) - if best_name == "catboost": - best_model.fit(combined[feature_cols], combined["pm25_mean"].to_numpy(dtype=np.float64), use_best_model=False) - train_pred = best_model.predict(train[feature_cols]) - val_pred = best_model.predict(val[feature_cols]) - test_pred = best_model.predict(test[feature_cols]) - else: - best_model.fit(combined[feature_cols].to_numpy(dtype=np.float32), combined["pm25_mean"].to_numpy(dtype=np.float64)) - train_pred = best_model.predict(train[feature_cols].to_numpy(dtype=np.float32)) - val_pred = best_model.predict(val[feature_cols].to_numpy(dtype=np.float32)) - test_pred = best_model.predict(test[feature_cols].to_numpy(dtype=np.float32)) - - args.output_dir.mkdir(parents=True, exist_ok=True) - pred_df = pd.concat( - [ - train.assign(pred_pm25=train_pred), - val.assign(pred_pm25=val_pred), - test.assign(pred_pm25=test_pred), - ], - ignore_index=True, - ) - pred_path = args.output_dir / "predictions.csv.gz" - pred_df.to_csv(pred_path, index=False, compression="gzip") - - train_eval = train.assign(pred=train_pred) - val_eval = val.assign(pred=val_pred) - test_eval = test.assign(pred=test_pred) - summary = { - "task_id": "smoke_pm25_named_fm", - "task_form": "station_daily_regression", - "attached_csv": str(args.attached_csv), - "output_dir": str(args.output_dir), - "seed": int(args.seed), - "feature_columns": feature_cols, - "split_sizes": {"train": int(len(train)), "val": int(len(val)), "test": int(len(test))}, - "candidate_validation": candidate_validation, - "selected_model": best_name, - "train_metrics": evaluate_frame(train_eval, "pred", args.exceedance_threshold), - "val_metrics": evaluate_frame(val_eval, "pred", args.exceedance_threshold), - "test_metrics": evaluate_frame(test_eval, "pred", args.exceedance_threshold), - "predictions_path": str(pred_path), - "model_family": "lightweight_linear_task_heads" if args.model_family == "lite" else "popular_open_source_task_models", - "fm_family": args.fm_family, - "benchmark_protocol": "fm_lite_protocol" if args.model_family == "lite" else "standard_protocol", - "selection_rule": "choose model by validation RMSE on named-FM attached rows; report on held-out test dates", - "tmt_policy": { - "task": "smoke_pm25", - "metric": "continuous RMSE/MAE with thresholded exceedance PRF", - "tolerance": "secondary event policy only", - }, - } - (args.output_dir / "summary.json").write_text(json.dumps(summary, indent=2), encoding="utf-8") - print(json.dumps(summary, indent=2)) - - -if __name__ == "__main__": - main() diff --git a/experiments/raw_reference/task_scripts/summarize_forced_meanstd_20260429.py b/experiments/raw_reference/task_scripts/summarize_forced_meanstd_20260429.py deleted file mode 100644 index f2d6ceeda56d884dd050cd52df6efd7d6095d429..0000000000000000000000000000000000000000 --- a/experiments/raw_reference/task_scripts/summarize_forced_meanstd_20260429.py +++ /dev/null @@ -1,232 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import argparse -import json -import math -import re -import statistics -from pathlib import Path -from typing import Any - - -SLUG_LABELS = { - "reference": "Reference", - "prithvi_wxc": "Prithvi-WxC", - "stormcast": "StormCast", - "aurora": "Aurora", - "climax": "ClimaX", - "alphaearth": "AlphaEarth", -} - - -def load(path: Path) -> dict[str, Any]: - return json.loads(path.read_text(encoding="utf-8")) - - -def stats(values: list[float]) -> dict[str, float | int]: - values = [float(v) for v in values if math.isfinite(float(v))] - if not values: - return {"n": 0, "mean": math.nan, "std": math.nan} - return { - "n": len(values), - "mean": float(statistics.fmean(values)), - "std": float(statistics.stdev(values)) if len(values) > 1 else 0.0, - } - - -def seed_from_path(path: Path) -> int | None: - match = re.search(r"_seed_(\d+)", str(path)) - return int(match.group(1)) if match else None - - -def label_from_seed_dir(path: Path, prefix: str) -> str: - for part in path.parts: - if part.startswith(prefix) and "_seed_" in part: - slug = part[len(prefix) :].split("_seed_", 1)[0] - return SLUG_LABELS.get(slug, slug) - return "unknown" - - -def dedupe_rows(rows: list[dict[str, Any]], keys: tuple[str, ...]) -> list[dict[str, Any]]: - selected: dict[tuple[Any, ...], dict[str, Any]] = {} - for row in rows: - key = tuple(row.get(name) for name in keys) - old = selected.get(key) - if old is None: - selected[key] = row - continue - old_mtime = Path(str(old["path"])).stat().st_mtime - new_mtime = Path(str(row["path"])).stat().st_mtime - if new_mtime >= old_mtime: - selected[key] = row - return list(selected.values()) - - -def best_val_threshold(data: dict[str, Any]) -> str: - entries = data["splits"]["val"]["threshold_metrics"] - return max(entries, key=lambda key: (float(entries[key]["f1"]), -float(entries[key]["threshold"]))) - - -def collect_occupancy(run_root: Path) -> dict[str, Any]: - rows: list[dict[str, Any]] = [] - for path in sorted(run_root.glob("table3_occupancy_*_seed_*/run_*/summary.json")): - data = load(path) - threshold_key = best_val_threshold(data) - test = data["splits"]["test"] - rows.append( - { - "label": data.get("fm_family") or label_from_seed_dir(path, "table3_occupancy_"), - "seed": seed_from_path(path), - "strict_f1": float(test["threshold_metrics"][threshold_key]["f1"]), - "tolerant_f1": float(test["tolerant_threshold_metrics"]["t0_s3"][threshold_key]["f1"]), - "union_f1": float(test["tolerant_threshold_metrics"]["t3_s3"][threshold_key]["f1"]), - "path": str(path), - } - ) - return group(rows, ["strict_f1", "tolerant_f1", "union_f1"]) - - -def collect_headcontrol(run_root: Path) -> dict[str, Any]: - rows: list[dict[str, Any]] = [] - for path in sorted(run_root.glob("table2_prithvi_wxc_headcontrol_seed_*/run_*/summary.json")): - data = load(path) - seed = seed_from_path(path) - for row in data.get("selection_summary", {}).get("rows", []): - rows.append( - { - "label": "Prithvi-WxC", - "scope": row["scope"], - "seed": seed, - "ranking_selected_union_f1": float(row["ranking_selected_union_f1"]), - "decision_selected_union_f1": float(row["decision_selected_union_f1"]), - "decision_regret_union_f1": float(row["decision_regret_union_f1"]), - "selection_failure": bool(row.get("selection_failure", False)), - "path": str(path), - } - ) - grouped: dict[str, Any] = {} - rows = dedupe_rows(rows, ("label", "scope", "seed")) - for scope in sorted({str(row["scope"]) for row in rows}): - selected = [row for row in rows if row["scope"] == scope] - grouped[scope] = { - "n": len(selected), - "failure_count": int(sum(1 for row in selected if row["selection_failure"])), - "ranking_selected_union_f1": stats([row["ranking_selected_union_f1"] for row in selected]), - "decision_selected_union_f1": stats([row["decision_selected_union_f1"] for row in selected]), - "decision_regret_union_f1": stats([row["decision_regret_union_f1"] for row in selected]), - } - return {"rows": rows, "summary": grouped} - - -def collect_spread(run_root: Path) -> dict[str, Any]: - rows: list[dict[str, Any]] = [] - for pattern, prefix in [ - ("table3_spread_*_seed_*/run_*/summary.json", "table3_spread_"), - ("table3_reference_spread_seed_*/run_*/summary.json", "table3_reference_spread_"), - ]: - for path in sorted(run_root.glob(pattern)): - data = load(path) - headline = data["headline_metrics"] - label = data.get("fm_family") or ("Reference" if "reference_spread" in str(path) else label_from_seed_dir(path, prefix)) - rows.append( - { - "label": label, - "seed": seed_from_path(path), - "strict_f1": float(headline["strict_f1"]), - "spatial_f1": float(headline["same_sample_spatial_tolerance_f1"]["s4"]), - "ap": float(headline["strict_AP"]), - "path": str(path), - } - ) - return group(rows, ["strict_f1", "spatial_f1", "ap"]) - - -def collect_task(run_root: Path, glob_pattern: str, prefix: str, metrics_path: list[str], metric_keys: list[str]) -> dict[str, Any]: - rows: list[dict[str, Any]] = [] - for path in sorted(run_root.glob(glob_pattern)): - data = load(path) - label = data.get("fm_family") or label_from_seed_dir(path, prefix) - node: Any = data - for key in metrics_path: - node = node[key] - row = {"label": label, "seed": seed_from_path(path), "path": str(path)} - for key in metric_keys: - row[key] = float(node[key]) - rows.append(row) - return group(rows, metric_keys) - - -def group(rows: list[dict[str, Any]], metric_keys: list[str]) -> dict[str, Any]: - if rows and "seed" in rows[0]: - rows = dedupe_rows(rows, ("label", "seed")) - summary: dict[str, Any] = {} - for label in sorted({str(row["label"]) for row in rows}): - selected = [row for row in rows if row["label"] == label] - summary[label] = {"n": len(selected)} - for key in metric_keys: - summary[label][key] = stats([row[key] for row in selected]) - return {"rows": rows, "summary": summary} - - -def fmt(value: dict[str, Any], scale: float = 1.0, digits: int = 2) -> str: - if int(value["n"]) == 0: - return "missing" - return f"{float(value['mean']) * scale:.{digits}f} +/- {float(value['std']) * scale:.{digits}f} (n={int(value['n'])})" - - -def write_markdown(out: Path, summary: dict[str, Any]) -> None: - lines = ["# Forced Mean/Std Gap-Fill Summary", ""] - for section in [ - "table2_headcontrol", - "table3_occupancy", - "table3_spread", - "table4_final_area", - "table4_analog", - "table4_smoke", - "table4_heat", - ]: - lines += [f"## {section}", ""] - sec = summary.get(section, {}).get("summary", {}) - if section == "table2_headcontrol": - for scope, row in sec.items(): - lines.append( - f"- {scope}: regret {fmt(row['decision_regret_union_f1'], 100.0)}; " - f"ranking union {fmt(row['ranking_selected_union_f1'], 100.0)}; " - f"decision union {fmt(row['decision_selected_union_f1'], 100.0)}; " - f"failures {row['failure_count']}/{row['n']}" - ) - else: - for label, row in sec.items(): - pieces = [f"{key} {fmt(val, 100.0 if key.endswith('_f1') or key == 'ap' else 1.0)}" for key, val in row.items() if isinstance(val, dict)] - lines.append(f"- {label}: " + "; ".join(pieces)) - lines.append("") - out.write_text("\n".join(lines), encoding="utf-8") - - -def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument("--run-root", type=Path, default=Path("${RUN_ROOT}")) - parser.add_argument("--out-json", type=Path, default=Path("${OUT_JSON}")) - parser.add_argument("--out-md", type=Path, default=Path("${OUT_MD}")) - args = parser.parse_args() - - summary = { - "run_root": str(args.run_root), - "table2_headcontrol": collect_headcontrol(args.run_root), - "table3_occupancy": collect_occupancy(args.run_root), - "table3_spread": collect_spread(args.run_root), - "table4_final_area": collect_task(args.run_root, "table4_final_area_*_seed_*/run_*/summary.json", "table4_final_area_", ["headline_metrics"], ["log_rmse", "log_mae", "log_spearman"]), - "table4_analog": collect_task(args.run_root, "table4_analog_*_seed_*/run_*/summary.json", "table4_analog_", ["test_metrics"], ["ndcg_at_10", "log_rmse", "log_mae"]), - "table4_smoke": collect_task(args.run_root, "table4_smoke_*_seed_*/run_*/summary.json", "table4_smoke_", ["test_metrics"], ["rmse", "mae", "pearson_r"]), - "table4_heat": collect_task(args.run_root, "table4_heat_*_seed_*/run_*/summary.json", "table4_heat_", ["test_metrics"], ["rmse_c", "mae_c", "pearson_r"]), - } - args.out_json.parent.mkdir(parents=True, exist_ok=True) - args.out_json.write_text(json.dumps(summary, indent=2), encoding="utf-8") - write_markdown(args.out_md, summary) - print(f"wrote={args.out_json}") - print(f"wrote={args.out_md}") - - -if __name__ == "__main__": - main() diff --git a/experiments/slurm/submit_template.sbatch b/experiments/slurm/submit_template.sbatch deleted file mode 100644 index f9f84b341aa80e4447726f020b5767986ca4e85f..0000000000000000000000000000000000000000 --- a/experiments/slurm/submit_template.sbatch +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=wildfire-contract-rerun -#SBATCH --cpus-per-task=4 -#SBATCH --mem=24G -#SBATCH --time=02:00:00 - -# Template only. Set these paths for your environment after obtaining data. -PROJECT_ROOT=/path/to/this/repository -DATA_ROOT=/path/to/raw/or/processed/data -OUTPUT_ROOT=/path/to/output - -cd "$PROJECT_ROOT" -python3 scripts/reproduce_paper_outputs.py diff --git a/firewxfm/__init__.py b/firewxfm/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..ba1f6f6060f63210510a38753d15de88ba54079e --- /dev/null +++ b/firewxfm/__init__.py @@ -0,0 +1,5 @@ +"""FireWx-FM model and serving utilities.""" + +from .modeling_unet import UNetSmallFlex + +__all__ = ["UNetSmallFlex"] diff --git a/models/wildfire_fm/modeling_unet.py b/firewxfm/modeling_unet.py similarity index 98% rename from models/wildfire_fm/modeling_unet.py rename to firewxfm/modeling_unet.py index ab17549f49ac5219e151633ac6aac728ea3d7695..f86ba8a8beb57c140c0324c39523689aff5295c7 100644 --- a/models/wildfire_fm/modeling_unet.py +++ b/firewxfm/modeling_unet.py @@ -1,4 +1,4 @@ -"""Compact U-Net architecture for the released WildFIRE-FM checkpoints.""" +"""Compact U-Net architecture for the released FireWx-FM checkpoints.""" from __future__ import annotations diff --git a/firewxfm/serve_conus.py b/firewxfm/serve_conus.py new file mode 100644 index 0000000000000000000000000000000000000000..2434107ec2a7d8c4f5118308550c94c643dadef8 --- /dev/null +++ b/firewxfm/serve_conus.py @@ -0,0 +1,187 @@ +#!/usr/bin/env python3 +"""Run final FireWx-FM CONUS inference from a 16-channel NumPy stack.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +import numpy as np +import torch + +from firewxfm.modeling_unet import UNetSmallFlex +from firewxfm.tiled_inference import predict_probability_tiled + + +DEFAULT_SHIFTS = [(dy, dx) for dy in (0, 4, 8, 12) for dx in (0, 4, 8, 12)] + + +def load_checkpoint(path: Path, device: torch.device) -> dict[str, torch.Tensor]: + payload = torch.load(path, map_location=device) + if isinstance(payload, dict): + for key in ("model_state_dict", "state_dict", "model"): + if key in payload and isinstance(payload[key], dict): + return payload[key] + if isinstance(payload, dict): + return payload + raise TypeError(f"Unsupported checkpoint format: {path}") + + +def parse_shifts(text: str | None) -> list[tuple[int, int]]: + if not text: + return DEFAULT_SHIFTS + shifts: list[tuple[int, int]] = [] + for item in text.split(";"): + if not item.strip(): + continue + y, x = item.split(",") + shifts.append((int(y), int(x))) + return shifts + + +def load_stats(path: Path | None) -> dict | None: + if path is None: + return None + return json.loads(path.read_text(encoding="utf-8")) + + +def _as_channel_array(value, channels: int) -> np.ndarray | None: + if value is None: + return None + if isinstance(value, dict): + out = np.zeros(channels, dtype=np.float32) + seen = False + for key, val in value.items(): + try: + idx = int(key) + except ValueError: + continue + if 0 <= idx < channels: + out[idx] = float(val) + seen = True + return out if seen else None + arr = np.asarray(value, dtype=np.float32) + if arr.size == channels: + return arr.reshape(channels) + return None + + +def normalize_stack(x: np.ndarray, stats: dict | None) -> np.ndarray: + if not stats or not stats.get("enabled", True): + return x.astype(np.float32, copy=True) + channels = x.shape[0] + mean = _as_channel_array(stats.get("mean") or stats.get("means"), channels) + std = _as_channel_array(stats.get("std") or stats.get("stds"), channels) + normalize_channels_raw = stats.get("normalize_channels") or stats.get("continuous_channel_indices") + out = x.astype(np.float32, copy=True) + if mean is None or std is None: + # Some training summaries store channel records instead of dense arrays. + records = stats.get("channel_stats") or stats.get("stats") + if records is None and isinstance(stats.get("channels"), list): + records = stats.get("channels") + if isinstance(records, list): + mean = np.zeros(channels, dtype=np.float32) + std = np.ones(channels, dtype=np.float32) + inferred_channels: list[int] = [] + for rec in records: + if not isinstance(rec, dict): + continue + idx = int(rec.get("index", rec.get("channel", -1))) + if 0 <= idx < channels: + mean[idx] = float(rec.get("mean", 0.0)) + std[idx] = float(rec.get("std", 1.0)) + inferred_channels.append(idx) + if normalize_channels_raw is None: + normalize_channels_raw = inferred_channels + else: + raise ValueError("Normalization stats must contain mean/std arrays or channel_stats records.") + if normalize_channels_raw is None: + normalize_channels = list(range(channels)) + else: + normalize_channels = [int(c) for c in normalize_channels_raw] + std = np.where(np.abs(std) < 1e-6, 1.0, std) + for c in normalize_channels: + if 0 <= c < channels: + out[c] = (out[c] - mean[c]) / std[c] + return out + + +def shifted_predict( + model: torch.nn.Module, + x: np.ndarray, + shifts: list[tuple[int, int]], + window: int, + stride: int, + halo: int, + batch_size: int, + device: torch.device, +) -> np.ndarray: + preds = [] + for dy, dx in shifts: + shifted = np.roll(x, shift=(dy, dx), axis=(1, 2)) + pred = predict_probability_tiled( + model, + torch.from_numpy(shifted), + tile_size=window, + stride=stride, + halo=halo, + device=device, + batch_size=batch_size, + ).numpy() + preds.append(np.roll(pred, shift=(-dy, -dx), axis=(0, 1))) + return np.mean(preds, axis=0).astype(np.float32) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--input-npy", type=Path, required=True, help="Input stack with shape [16,H,W].") + parser.add_argument("--checkpoint", type=Path, required=True) + parser.add_argument("--normalization-stats", type=Path) + parser.add_argument("--output-npy", type=Path, required=True) + parser.add_argument("--zero-channels", default="14,15", help="Comma-separated channels to zero at serving time.") + parser.add_argument("--shifts", default=None, help="Semicolon list such as '0,0;0,4;4,0'.") + parser.add_argument("--window", type=int, default=256) + parser.add_argument("--stride", type=int, default=64) + parser.add_argument("--halo", type=int, default=32) + parser.add_argument("--base-channels", type=int, default=32) + parser.add_argument("--dropout", type=float, default=0.1) + parser.add_argument("--batch-size", type=int, default=8) + parser.add_argument("--device", default="cuda" if torch.cuda.is_available() else "cpu") + args = parser.parse_args() + + x = np.load(args.input_npy).astype(np.float32) + if x.ndim != 3 or x.shape[0] != 16: + raise ValueError(f"Expected input shape [16,H,W], got {x.shape}") + x = normalize_stack(x, load_stats(args.normalization_stats)) + for channel in [int(c) for c in args.zero_channels.split(",") if c.strip()]: + x[channel] = 0.0 + + device = torch.device(args.device) + model = UNetSmallFlex( + in_ch=16, + base=args.base_channels, + dropout=args.dropout, + norm_type="group", + norm_groups=8, + use_aux_spatial_head=True, + ).to(device) + state = load_checkpoint(args.checkpoint, device) + model.load_state_dict(state, strict=False) + pred = shifted_predict( + model=model, + x=x, + shifts=parse_shifts(args.shifts), + window=args.window, + stride=args.stride, + halo=args.halo, + batch_size=args.batch_size, + device=device, + ) + args.output_npy.parent.mkdir(parents=True, exist_ok=True) + np.save(args.output_npy, pred) + print(f"wrote {args.output_npy} shape={pred.shape} mean={float(pred.mean()):.6f} max={float(pred.max()):.6f}") + + +if __name__ == "__main__": + main() diff --git a/models/wildfire_fm/tiled_inference.py b/firewxfm/tiled_inference.py similarity index 96% rename from models/wildfire_fm/tiled_inference.py rename to firewxfm/tiled_inference.py index 36feabb6da663a044e5957a0efd8a6fc9e980b5e..ecde61fd789f1169052842e5d4a696e3c792d7a7 100644 --- a/models/wildfire_fm/tiled_inference.py +++ b/firewxfm/tiled_inference.py @@ -1,4 +1,4 @@ -"""Overlap-tiled inference helpers for WildFIRE-FM probability maps.""" +"""Overlap-tiled inference helpers for FireWx-FM probability maps.""" from __future__ import annotations @@ -40,7 +40,7 @@ def predict_probability_tiled( Parameters ---------- model: - WildFIRE-FM model returning logits or ``(logits, aux_logits)``. + FireWx-FM model returning logits or ``(logits, aux_logits)``. x: Input tensor in ``[C, H, W]`` or ``[1, C, H, W]`` order. tile_size: diff --git a/models/README.md b/models/README.md new file mode 100644 index 0000000000000000000000000000000000000000..61ca01dca4096fb1c6d945c129c1dea8908cdfc8 --- /dev/null +++ b/models/README.md @@ -0,0 +1,23 @@ +# Models + +This directory stores the released FireWx-FM model artifacts. + +## Expected Files + +| Path | Purpose | +|---|---| +| `checkpoints/firewxfm_conus_lowposw_noexposure_seed42.pt` | Legacy CONUS no-exposure serving checkpoint. | +| `checkpoints/firewxfm_2024_staticfix_region_balanced_bce_seed42.pt` | Staticfix-balanced CONUS serving checkpoint. | +| `metadata/input_channels.json` | Machine-readable input-channel contract. | +| `metadata/input_normalization_stats.json` | Train-split z-score statistics used at inference. | +| `metadata/final_model_manifest.json` | Serving manifest for the final checkpoint. | +| `metadata/run_summary.json` | Training/evaluation summary for the final run. | +| `metadata/tile_summary.json` | Tile construction and sparse-label summary. | + +Run the release check from the repository root: + +```bash +python scripts/check_release_files.py +``` + +If the checkpoint is absent, inference code can still be inspected, but the final prediction example cannot be reproduced from this checkout. diff --git a/models/checkpoints/.gitkeep b/models/checkpoints/.gitkeep new file mode 100644 index 0000000000000000000000000000000000000000..8b137891791fe96927ad78e64b0aad7bded08bdc --- /dev/null +++ b/models/checkpoints/.gitkeep @@ -0,0 +1 @@ + diff --git a/models/checkpoints/firewxfm_2024_staticfix_region_balanced_bce_seed42.pt b/models/checkpoints/firewxfm_2024_staticfix_region_balanced_bce_seed42.pt new file mode 100644 index 0000000000000000000000000000000000000000..791f334c9ac85e14538e734e613ce654043f289c --- /dev/null +++ b/models/checkpoints/firewxfm_2024_staticfix_region_balanced_bce_seed42.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8a0818aae5534e3c33fa4100bebbc525b387d3a8e197caa9ce7bf998ef937947 +size 31112933 diff --git a/models/checkpoints/firewxfm_conus_lowposw_noexposure_seed42.pt b/models/checkpoints/firewxfm_conus_lowposw_noexposure_seed42.pt new file mode 100644 index 0000000000000000000000000000000000000000..05721a06c5e7e9d02903dc87ba6672d39f759c28 --- /dev/null +++ b/models/checkpoints/firewxfm_conus_lowposw_noexposure_seed42.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7a69145f8a7aa411f4a86d4b556a3af52c5fde35cc7437caf5e3d9ee522ea0a5 +size 31092965 diff --git a/models/metadata/final_model_manifest.json b/models/metadata/final_model_manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..506a89b443bab969f7cbc93de335b8bcca79c64c --- /dev/null +++ b/models/metadata/final_model_manifest.json @@ -0,0 +1,29 @@ +{ + "model_name": "FireWx-FM CONUS staticfix-balanced serving checkpoint", + "release_tag": "firewxfm-conus-2026-08-12-staticfix-balanced", + "checkpoint": "models/checkpoints/firewxfm_2024_staticfix_region_balanced_bce_seed42.pt", + "normalization_stats": "models/metadata/input_normalization_stats.json", + "grid": "Lower-48 CONUS, EPSG:5070, 5 km", + "lead_hours": 12, + "input_channels": 16, + "normalized_channels": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 13, 14, 15], + "serving_zero_channels": [14, 15], + "recommended_inference": { + "window": 256, + "stride": 64, + "halo": 32, + "phase_shifts": ["0,0", "0,4", "0,8", "0,12", "4,0", "4,4", "4,8", "4,12", "8,0", "8,4", "8,8", "8,12", "12,0", "12,4", "12,8", "12,12"] + }, + "example_input_time_utc": "2026-07-07T12:00:00Z", + "example_prediction_target_time_utc": "2026-07-08T00:00:00Z", + "example_postprocessing": { + "name": "source_probability_logit_shift_with_smooth_lon_lat_gate", + "calibration_region": "CONUS seam repair including Rockies/Great-Plains and East", + "source": "examples/staticfix_conus_candidate_20260812/firewxfm_20260707_t12_staticfix_conus_probability_lower48.tif" + }, + "example_outputs": [ + "examples/staticfix_conus_candidate_20260812/firewxfm_20260707_t12_staticfix_conus_baseline_probability_lower48.tif", + "examples/staticfix_conus_candidate_20260812/firewxfm_20260707_t12_staticfix_conus_probability_lower48.tif", + "examples/staticfix_conus_candidate_20260812/firewxfm_20260707_t12_staticfix_conus_longitude_audit.json" + ] +} diff --git a/models/metadata/input_channels.json b/models/metadata/input_channels.json new file mode 100644 index 0000000000000000000000000000000000000000..7558596784c880116e465f1a4a1df5d55f7a7897 --- /dev/null +++ b/models/metadata/input_channels.json @@ -0,0 +1,35 @@ +{ + "version": "firewxfm-conus-final-2026-06-29", + "tensor_order": "[channel, y, x]", + "channel_order_matters": true, + "channels": [ + {"index": 0, "name": "t2m", "source": "NOAA HRRR", "description": "2 m air temperature", "unit": "K"}, + {"index": 1, "name": "d2m", "source": "NOAA HRRR", "description": "2 m dew point temperature", "unit": "K"}, + {"index": 2, "name": "u10", "source": "NOAA HRRR", "description": "10 m eastward wind component", "unit": "m s-1"}, + {"index": 3, "name": "v10", "source": "NOAA HRRR", "description": "10 m northward wind component", "unit": "m s-1"}, + {"index": 4, "name": "cape", "source": "NOAA HRRR", "description": "Convective available potential energy, 0-3000 m above ground layer", "unit": "J kg-1"}, + {"index": 5, "name": "sp", "source": "NOAA HRRR", "description": "Surface pressure", "unit": "Pa"}, + {"index": 6, "name": "blh", "source": "NOAA HRRR", "description": "Boundary-layer height", "unit": "m"}, + {"index": 7, "name": "vis", "source": "NOAA HRRR", "description": "Visibility", "unit": "m"}, + {"index": 8, "name": "prate", "source": "NOAA HRRR", "description": "Precipitation rate", "unit": "kg m-2 s-1"}, + {"index": 9, "name": "tp", "source": "NOAA HRRR", "description": "Accumulated precipitation", "unit": "kg m-2"}, + {"index": 10, "name": "dynamic_valid", "source": "input builder", "description": "Dynamic weather validity fraction", "unit": "0-1"}, + {"index": 11, "name": "static_valid", "source": "input builder", "description": "Static-layer validity fraction after reprojection", "unit": "0-1"}, + {"index": 12, "name": "fuel_fbfm40", "source": "LANDFIRE", "description": "Fire-behavior fuel model", "unit": "category code"}, + {"index": 13, "name": "canopy_cover", "source": "LANDFIRE", "description": "Canopy cover", "unit": "percent"}, + {"index": 14, "name": "housing_density", "source": "Wildfire Risk to Communities", "description": "Housing-unit density; zeroed in final no-exposure serving checkpoint", "unit": "provider native"}, + {"index": 15, "name": "population", "source": "LandScan", "description": "Population; zeroed in final no-exposure serving checkpoint", "unit": "persons"} + ], + "target": { + "name": "wildfire_active_fire_occupancy", + "source": "NASA FIRMS", + "lead_hours": 12, + "definition": "one or more active-fire detections mapped to a 5 km grid cell at the target time" + }, + "final_serving": { + "grid": "CONUS Lower-48, EPSG:5070, 5 km", + "normalization": "channels 0-9, 13, 14, and 15 use train-split z-score statistics from models/metadata/input_normalization_stats.json", + "zero_channels": [14, 15], + "inference": "overlap-window inference with modulo-16 phase ensemble" + } +} diff --git a/models/metadata/input_normalization_stats.json b/models/metadata/input_normalization_stats.json new file mode 100644 index 0000000000000000000000000000000000000000..d7da307cf2cc37b1714861423edb36e662f6d96e --- /dev/null +++ b/models/metadata/input_normalization_stats.json @@ -0,0 +1,138 @@ +{ + "enabled": true, + "method": "per-channel z-score", + "stats_source": "train split full maps", + "continuous_channel_indices": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 13, + 14, + 15 + ], + "eps": 1e-06, + "channels": [ + { + "index": 0, + "name": "t2m", + "mean": 295.58303950541296, + "std": 6.594453491810887, + "count": 285273600 + }, + { + "index": 1, + "name": "d2m", + "mean": 288.80136209265663, + "std": 7.676135798532925, + "count": 285273600 + }, + { + "index": 2, + "name": "u10", + "mean": -0.11793715918198373, + "std": 3.202360372922017, + "count": 285273600 + }, + { + "index": 3, + "name": "v10", + "mean": 0.02186221694284171, + "std": 3.9899202727517045, + "count": 285273600 + }, + { + "index": 4, + "name": "cape", + "mean": 747.1427798436308, + "std": 1110.302235759119, + "count": 285273600 + }, + { + "index": 5, + "name": "sp", + "mean": 96817.92464269389, + "std": 6815.012582765496, + "count": 285273600 + }, + { + "index": 6, + "name": "blh", + "mean": 681.3271276792287, + "std": 684.6619485994495, + "count": 285273600 + }, + { + "index": 7, + "name": "vis", + "mean": 26947.73532776955, + "std": 20073.76628434421, + "count": 285273600 + }, + { + "index": 8, + "name": "prate", + "mean": 0.0, + "std": 1e-06, + "count": 285273600 + }, + { + "index": 9, + "name": "tp", + "mean": 0.0, + "std": 1e-06, + "count": 285273600 + }, + { + "index": 13, + "name": "canopy_cover", + "mean": 8.25561790505676, + "std": 21.45759831821086, + "count": 285273600 + }, + { + "index": 14, + "name": "housing_density", + "mean": 7.232399485711034, + "std": 62.62637229507992, + "count": 285273600 + }, + { + "index": 15, + "name": "population", + "mean": 14.228408365455689, + "std": 122.32549664182993, + "count": 285273600 + } + ], + "channel_names": [ + "t2m", + "d2m", + "u10", + "v10", + "cape", + "sp", + "blh", + "vis", + "prate", + "tp", + "firewx_valid", + "static_valid", + "fuel_fbfm40", + "canopy_cover", + "housing_density", + "population" + ], + "static_names": [ + "fuel_fbfm40", + "canopy_cover", + "housing_density", + "population" + ] +} \ No newline at end of file diff --git a/models/metadata/run_summary.json b/models/metadata/run_summary.json new file mode 100644 index 0000000000000000000000000000000000000000..266bffedd35f767fdb7c863f26094842d8fe54b4 --- /dev/null +++ b/models/metadata/run_summary.json @@ -0,0 +1,331 @@ +{ + "run_name": "firewxfm_conus_5km_l12_lowposw_noexposure_seed42", + "device": "cuda", + "index_root": "local CONUS 5 km HRRR/FIRMS cache; raw cache is not redistributed", + "input_channels": 16, + "tile_size": 32, + "num_train_tiles": 169854, + "positive_train_tiles": 24260, + "negative_train_tiles": 145594, + "positive_tile_placement": "random_containing", + "train_positive_rate": 0.0011426920785645319, + "metric_reference_positive_rate": 0.0001221809519002109, + "pos_weight": 10.0, + "use_aux_spatial_head": true, + "aux_spatial_radius": 2, + "aux_spatial_loss_weight": 0.5, + "aux_positive_rate": 0.001959855380939561, + "init_checkpoint": "", + "input_normalization": { + "enabled": true, + "method": "per-channel z-score", + "stats_source": "train split full maps", + "continuous_channel_indices": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 13, + 14, + 15 + ], + "eps": 1e-06, + "channels": [ + { + "index": 0, + "name": "t2m", + "mean": 295.58303950541296, + "std": 6.594453491810887, + "count": 285273600 + }, + { + "index": 1, + "name": "d2m", + "mean": 288.80136209265663, + "std": 7.676135798532925, + "count": 285273600 + }, + { + "index": 2, + "name": "u10", + "mean": -0.11793715918198373, + "std": 3.202360372922017, + "count": 285273600 + }, + { + "index": 3, + "name": "v10", + "mean": 0.02186221694284171, + "std": 3.9899202727517045, + "count": 285273600 + }, + { + "index": 4, + "name": "cape", + "mean": 747.1427798436308, + "std": 1110.302235759119, + "count": 285273600 + }, + { + "index": 5, + "name": "sp", + "mean": 96817.92464269389, + "std": 6815.012582765496, + "count": 285273600 + }, + { + "index": 6, + "name": "blh", + "mean": 681.3271276792287, + "std": 684.6619485994495, + "count": 285273600 + }, + { + "index": 7, + "name": "vis", + "mean": 26947.73532776955, + "std": 20073.76628434421, + "count": 285273600 + }, + { + "index": 8, + "name": "prate", + "mean": 0.0, + "std": 1e-06, + "count": 285273600 + }, + { + "index": 9, + "name": "tp", + "mean": 0.0, + "std": 1e-06, + "count": 285273600 + }, + { + "index": 13, + "name": "canopy_cover", + "mean": 8.25561790505676, + "std": 21.45759831821086, + "count": 285273600 + }, + { + "index": 14, + "name": "housing_density", + "mean": 7.232399485711034, + "std": 62.62637229507992, + "count": 285273600 + }, + { + "index": 15, + "name": "population", + "mean": 14.228408365455689, + "std": 122.32549664182993, + "count": 285273600 + } + ], + "channel_names": [ + "t2m", + "d2m", + "u10", + "v10", + "cape", + "sp", + "blh", + "vis", + "prate", + "tp", + "firewx_valid", + "static_valid", + "fuel_fbfm40", + "canopy_cover", + "housing_density", + "population" + ], + "static_names": [ + "fuel_fbfm40", + "canopy_cover", + "housing_density", + "population" + ] + }, + "zero_input_channels": [ + 14, + 15 + ], + "selection_metric": "val_topk_area_metrics.0.0100.lift", + "best_selection_score": 21.713973858752247, + "best_val_pr_auc": 0.0033450774772912566, + "best_val_auroc": 0.8021774671572162, + "best_test_pr_auc": 0.0007209656375101411, + "best_test_auroc": 0.7466110351132904, + "best_test_brier": 0.00019806493946816772, + "best_test_ece": 0.004851287871785349, + "best_test_topk_area_metrics": { + "0.0010": { + "fraction": 0.001, + "cells": 93024, + "precision": 0.004891210701316595, + "recall": 0.03323836624622345, + "lift": 33.23836528700155 + }, + "0.0050": { + "fraction": 0.005, + "cells": 465120, + "precision": 0.002218782203271985, + "recall": 0.07538899779319763, + "lift": 15.077799315576614 + }, + "0.0100": { + "fraction": 0.01, + "cells": 930240, + "precision": 0.0014963879948481917, + "recall": 0.1016874834895134, + "lift": 10.168748357223677 + }, + "0.0500": { + "fraction": 0.05, + "cells": 4651200, + "precision": 0.0006129600806161761, + "recall": 0.2082694172859192, + "lift": 4.1653881441636225 + }, + "0.1000": { + "fraction": 0.1, + "cells": 9302400, + "precision": 0.0004342965257819742, + "recall": 0.2951274812221527, + "lift": 2.9512747350939743 + } + }, + "best_test_threshold_metrics": { + "0.1000": { + "threshold": 0.1, + "tp": 6, + "fp": 2106, + "fn": 13683, + "precision": 0.002840909090909091, + "recall": 0.00043830813061582295, + "far": 0.9971590909090909, + "csi": 0.0003798670465337132, + "f1": 0.0007594456047085627, + "f2": 0.0005275374551593164, + "frequency_bias": 0.15428446197676968, + "predicted_positive_rate": 2.2703818369453044e-05 + }, + "0.2000": { + "threshold": 0.2, + "tp": 0, + "fp": 43, + "fn": 13689, + "precision": 0.0, + "recall": 0.0, + "far": 1.0, + "csi": 0.0, + "f1": 0.0, + "f2": 0.0, + "frequency_bias": 0.0031412082694133978, + "predicted_positive_rate": 4.6224630202958377e-07 + }, + "0.3000": { + "threshold": 0.3, + "tp": 0, + "fp": 0, + "fn": 13689, + "precision": 0.0, + "recall": 0.0, + "far": 0.0, + "csi": 0.0, + "f1": 0.0, + "f2": 0.0, + "frequency_bias": 0.0, + "predicted_positive_rate": 0.0 + }, + "0.5000": { + "threshold": 0.5, + "tp": 0, + "fp": 0, + "fn": 13689, + "precision": 0.0, + "recall": 0.0, + "far": 0.0, + "csi": 0.0, + "f1": 0.0, + "f2": 0.0, + "frequency_bias": 0.0, + "predicted_positive_rate": 0.0 + }, + "0.7000": { + "threshold": 0.7, + "tp": 0, + "fp": 0, + "fn": 13689, + "precision": 0.0, + "recall": 0.0, + "far": 0.0, + "csi": 0.0, + "f1": 0.0, + "f2": 0.0, + "frequency_bias": 0.0, + "predicted_positive_rate": 0.0 + }, + "0.9000": { + "threshold": 0.9, + "tp": 0, + "fp": 0, + "fn": 13689, + "precision": 0.0, + "recall": 0.0, + "far": 0.0, + "csi": 0.0, + "f1": 0.0, + "f2": 0.0, + "frequency_bias": 0.0, + "predicted_positive_rate": 0.0 + } + }, + "best_test_fss": { + "0.1000": { + "1": 0.005789192925805153, + "2": 0.012606272532119656, + "4": 0.021284699263854567, + "8": 0.0376706107736593 + }, + "0.2000": { + "1": 0.0, + "2": 0.0, + "4": 0.0, + "8": 6.550455200704519e-06 + }, + "0.3000": { + "1": 0.0, + "2": 0.0, + "4": 0.0, + "8": 0.0 + }, + "0.5000": { + "1": 0.0, + "2": 0.0, + "4": 0.0, + "8": 0.0 + }, + "0.7000": { + "1": 0.0, + "2": 0.0, + "4": 0.0, + "8": 0.0 + }, + "0.9000": { + "1": 0.0, + "2": 0.0, + "4": 0.0, + "8": 0.0 + } + }, + "epochs": 10 +} diff --git a/models/metadata/tile_summary.json b/models/metadata/tile_summary.json new file mode 100644 index 0000000000000000000000000000000000000000..41708a53b684ab71ceb20b7ba4a4b3e6cb11327f --- /dev/null +++ b/models/metadata/tile_summary.json @@ -0,0 +1,159 @@ +{ + "index_root": "local CONUS 5 km HRRR/FIRMS cache; raw cache is not redistributed", + "tile_size": 32, + "num_train_tiles": 169854, + "positive_train_tiles": 24260, + "negative_train_tiles": 145594, + "input_channels": 16, + "positive_tile_placement": "random_containing", + "train_positive_rate": 0.0011426920785645319, + "metric_reference_positive_rate": 0.0001221809519002109, + "pos_weight_source": "tiles", + "positive_rate_source": "tiles", + "pos_weight": 10.0, + "loss_type": "bce", + "train_target_mode": "dilate_max", + "train_target_radius": 2, + "input_normalization": { + "enabled": true, + "method": "per-channel z-score", + "stats_source": "train split full maps", + "continuous_channel_indices": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 13, + 14, + 15 + ], + "eps": 1e-06, + "channels": [ + { + "index": 0, + "name": "t2m", + "mean": 295.58303950541296, + "std": 6.594453491810887, + "count": 285273600 + }, + { + "index": 1, + "name": "d2m", + "mean": 288.80136209265663, + "std": 7.676135798532925, + "count": 285273600 + }, + { + "index": 2, + "name": "u10", + "mean": -0.11793715918198373, + "std": 3.202360372922017, + "count": 285273600 + }, + { + "index": 3, + "name": "v10", + "mean": 0.02186221694284171, + "std": 3.9899202727517045, + "count": 285273600 + }, + { + "index": 4, + "name": "cape", + "mean": 747.1427798436308, + "std": 1110.302235759119, + "count": 285273600 + }, + { + "index": 5, + "name": "sp", + "mean": 96817.92464269389, + "std": 6815.012582765496, + "count": 285273600 + }, + { + "index": 6, + "name": "blh", + "mean": 681.3271276792287, + "std": 684.6619485994495, + "count": 285273600 + }, + { + "index": 7, + "name": "vis", + "mean": 26947.73532776955, + "std": 20073.76628434421, + "count": 285273600 + }, + { + "index": 8, + "name": "prate", + "mean": 0.0, + "std": 1e-06, + "count": 285273600 + }, + { + "index": 9, + "name": "tp", + "mean": 0.0, + "std": 1e-06, + "count": 285273600 + }, + { + "index": 13, + "name": "canopy_cover", + "mean": 8.25561790505676, + "std": 21.45759831821086, + "count": 285273600 + }, + { + "index": 14, + "name": "housing_density", + "mean": 7.232399485711034, + "std": 62.62637229507992, + "count": 285273600 + }, + { + "index": 15, + "name": "population", + "mean": 14.228408365455689, + "std": 122.32549664182993, + "count": 285273600 + } + ], + "channel_names": [ + "t2m", + "d2m", + "u10", + "v10", + "cape", + "sp", + "blh", + "vis", + "prate", + "tp", + "firewx_valid", + "static_valid", + "fuel_fbfm40", + "canopy_cover", + "housing_density", + "population" + ], + "static_names": [ + "fuel_fbfm40", + "canopy_cover", + "housing_density", + "population" + ] + }, + "zero_input_channels": [ + 14, + 15 + ] +} diff --git a/models/wildfire_fm/README.md b/models/wildfire_fm/README.md deleted file mode 100644 index 69d606af67221ab41e00dafdd965f9587ea1d213..0000000000000000000000000000000000000000 --- a/models/wildfire_fm/README.md +++ /dev/null @@ -1,70 +0,0 @@ -# WildFIRE-FM Checkpoints - -This directory contains metadata for the released WildFIRE-FM seeded checkpoints. -The checkpoint files are stored on the Hugging Face Hub under: - -```text -models/wildfire_fm/checkpoints/seed_1/best_firms_prauc.pt -models/wildfire_fm/checkpoints/seed_7/best_firms_prauc.pt -models/wildfire_fm/checkpoints/seed_42/best_firms_prauc.pt -models/wildfire_fm/checkpoints/seed_99/best_firms_prauc.pt -models/wildfire_fm/checkpoints/seed_123/best_firms_prauc.pt -``` - -Use `modeling_unet.py` to instantiate the compact U-Net architecture before loading a checkpoint state dict. -For tiled serving, use `tiled_inference.py` or an equivalent overlap/halo -stitching procedure. Avoid independent non-overlapping 32-by-32 tiles because -they can introduce tile-center and edge artifacts. -For the integration question checklist covering channel order, normalization, -CAPE selection, validity masks, static resampling, spatial aggregation, and -CONUS retraining status, see `../../docs/hugh_handoff_status.md`. - -## Training scope - -The released checkpoints are California regional weights trained for the 5 km -EPSG:5070 California grid. They should not be described as nationwide-trained -weights. A machine-readable scope file is stored in `training_scope.json`, and a -CONUS retraining recipe is provided in `../../training/NATIONWIDE_RETRAINING.md`. -The CONUS template includes random-containing positive tile placement and -optional train-split z-score normalization for continuous channels; no CONUS -checkpoint is released yet. - -## Input channels - -The checkpoints expect a fixed 16-channel tensor in `[channel, y, x]` order. -A machine-readable copy is stored in `input_channels.json`. -Channel order matters. The released checkpoints use native source units with no -mean/std normalization or min/max scaling. Invalid or missing values are -zero-filled, with validity-mask channels carried in the input tensor. - -| Channels | Names | Source | -|---:|---|---| -| 0-9 | `t2m`, `d2m`, `u10`, `v10`, `cape`, `sp`, `blh`, `vis`, `prate`, `tp` | NOAA HRRR | -| 10 | `firewx_valid` | Cache-level dynamic input presence mask; 1.0 everywhere in this release | -| 11 | `static_valid` | Fraction of static layers valid after reprojection | -| 12-15 | `fuel_fbfm40`, `canopy_cover`, `housing_density`, `population` | LANDFIRE, WRC housing density, LandScan | - -FIRMS detections are used to derive the occupancy target, not as an input -channel. WFIGS and MTBS are supporting event-level resources and are not part of -the pretrained occupancy input. - -The two validity channels have different meanings. `firewx_valid` is a -cache-level dynamic input presence mask; in the released California regional -cache it is 1.0 everywhere and is not a per-variable HRRR missing-data mask. -`static_valid` is the fraction of the four static layers that are valid after -reprojection at each grid cell, so its possible values are 0.0, 0.25, 0.5, 0.75, -and 1.0. - -For HRRR, `t2m` and `d2m` are 2 m fields in Kelvin; `u10` and `v10` are 10 m -wind components in m/s. The `cape` channel is selected from the HRRR surface -instantaneous group with `typeOfLevel=surface` and `stepType=instant`; it is not -the most-unstable or layer CAPE field. - -## Custom spatial serving - -The checkpoint remains a native-grid model. For web and client migration, use -`../../spatial_serving/grid_to_polygons.py` to aggregate probability maps to -any polygon-defined serving granularity. County, census tract, block group, and -fire district outputs are examples of the same custom-boundary workflow. This -supports user-defined granularities without changing the checkpoint or the -16-channel input contract. diff --git a/models/wildfire_fm/checkpoint_manifest.json b/models/wildfire_fm/checkpoint_manifest.json deleted file mode 100644 index e494bc36f67e2b9b340b475b495a2b5fbccfcbd0..0000000000000000000000000000000000000000 --- a/models/wildfire_fm/checkpoint_manifest.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "model": "WildFIRE-FM", - "weights": "uploaded to the Hugging Face Hub under the filenames listed below", - "checkpoints": [ - { - "seed": 123, - "filename": "models/wildfire_fm/checkpoints/seed_123/best_firms_prauc.pt", - "sha256": "4629a54134efb0e6fe5dabc177630923e4f4b5520ceff5076a36c03ae45cfbab", - "bytes": 31092197 - }, - { - "seed": 1, - "filename": "models/wildfire_fm/checkpoints/seed_1/best_firms_prauc.pt", - "sha256": "4a8f7e500bbdf222b33e2bc056d7e6202b526745c75dca26abc478bee9b2d84f", - "bytes": 31092197 - }, - { - "seed": 42, - "filename": "models/wildfire_fm/checkpoints/seed_42/best_firms_prauc.pt", - "sha256": "2ba80ddfdcf173a425eafc1d40dfd06088ac6f3e588e9adb866660df84694346", - "bytes": 31092197 - }, - { - "seed": 7, - "filename": "models/wildfire_fm/checkpoints/seed_7/best_firms_prauc.pt", - "sha256": "6a07d34302f63937e8e74c43c651cf3859b74bc95a5b68e37ffb2f37739a0b30", - "bytes": 31092197 - }, - { - "seed": 99, - "filename": "models/wildfire_fm/checkpoints/seed_99/best_firms_prauc.pt", - "sha256": "d5ce9d77c84167a7e24e3953c1f939fa7f621ceb905ac4d33e27eae523c0f5f8", - "bytes": 31092197 - } - ] -} diff --git a/models/wildfire_fm/checkpoints/seed_1/best_firms_prauc.pt b/models/wildfire_fm/checkpoints/seed_1/best_firms_prauc.pt deleted file mode 100644 index 2a13b543702b63d937278aca94d0fe3b5c375804..0000000000000000000000000000000000000000 --- a/models/wildfire_fm/checkpoints/seed_1/best_firms_prauc.pt +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:4a8f7e500bbdf222b33e2bc056d7e6202b526745c75dca26abc478bee9b2d84f -size 31092197 diff --git a/models/wildfire_fm/checkpoints/seed_123/best_firms_prauc.pt b/models/wildfire_fm/checkpoints/seed_123/best_firms_prauc.pt deleted file mode 100644 index 2611a22f183010dd58b7faec373bb593658f4cc3..0000000000000000000000000000000000000000 --- a/models/wildfire_fm/checkpoints/seed_123/best_firms_prauc.pt +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:4629a54134efb0e6fe5dabc177630923e4f4b5520ceff5076a36c03ae45cfbab -size 31092197 diff --git a/models/wildfire_fm/checkpoints/seed_42/best_firms_prauc.pt b/models/wildfire_fm/checkpoints/seed_42/best_firms_prauc.pt deleted file mode 100644 index bb82d897e90c25fbd3f985d57165a4c8e5a71d1b..0000000000000000000000000000000000000000 --- a/models/wildfire_fm/checkpoints/seed_42/best_firms_prauc.pt +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:2ba80ddfdcf173a425eafc1d40dfd06088ac6f3e588e9adb866660df84694346 -size 31092197 diff --git a/models/wildfire_fm/checkpoints/seed_7/best_firms_prauc.pt b/models/wildfire_fm/checkpoints/seed_7/best_firms_prauc.pt deleted file mode 100644 index bb85071e5ebfdc3edb53155735b25958472b74c8..0000000000000000000000000000000000000000 --- a/models/wildfire_fm/checkpoints/seed_7/best_firms_prauc.pt +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:6a07d34302f63937e8e74c43c651cf3859b74bc95a5b68e37ffb2f37739a0b30 -size 31092197 diff --git a/models/wildfire_fm/checkpoints/seed_99/best_firms_prauc.pt b/models/wildfire_fm/checkpoints/seed_99/best_firms_prauc.pt deleted file mode 100644 index fad8e6201b8d104ce57903e5da9e8940f8db7267..0000000000000000000000000000000000000000 --- a/models/wildfire_fm/checkpoints/seed_99/best_firms_prauc.pt +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:d5ce9d77c84167a7e24e3953c1f939fa7f621ceb905ac4d33e27eae523c0f5f8 -size 31092197 diff --git a/models/wildfire_fm/configs/reference_fireprone_seed_123_20260504.json b/models/wildfire_fm/configs/reference_fireprone_seed_123_20260504.json deleted file mode 100644 index 8dfb9ebc5e12509e0c7ff089b815ae5126cd0aaf..0000000000000000000000000000000000000000 --- a/models/wildfire_fm/configs/reference_fireprone_seed_123_20260504.json +++ /dev/null @@ -1,62 +0,0 @@ -{ - "index_root": "", - "run_root": "", - "tile_size": 32, - "max_positive_tiles_per_sample": 64, - "min_negative_tiles_per_sample": 4, - "negative_to_positive_ratio": 3.0, - "positive_rate_source": "tiles", - "pos_weight_source": "tiles", - "pos_weight_cap": 50.0, - "batch_size": 32, - "eval_batch_size": 2, - "epochs": 10, - "learning_rate": 0.0002, - "weight_decay": 0.0001, - "base_channels": 32, - "dropout": 0.1, - "norm_type": "group", - "norm_groups": 8, - "init_head_bias_from_positive_rate": true, - "augment_flip": true, - "threshold": 0.5, - "metric_thresholds": [ - 0.1, - 0.2, - 0.3, - 0.5, - 0.7, - 0.9 - ], - "topk_area_fractions": [ - 0.01, - 0.05, - 0.1 - ], - "fss_radii": [ - 1, - 2, - 4, - 8 - ], - "reliability_bins": 10, - "amp": true, - "seed": 123, - "boundary_radii": [ - 1, - 2, - 4 - ], - "coarsen_factors": [ - 2, - 4, - 8 - ], - "use_aux_spatial_head": true, - "aux_spatial_radius": 2, - "aux_spatial_loss_weight": 0.5, - "train_target_mode": "dilate_max", - "train_target_radius": 2, - "loss_type": "bce", - "init_checkpoint": "" -} diff --git a/models/wildfire_fm/configs/reference_fireprone_seed_1_20260504.json b/models/wildfire_fm/configs/reference_fireprone_seed_1_20260504.json deleted file mode 100644 index 6278357a9221e2611c999d1691e8a5af78f202a8..0000000000000000000000000000000000000000 --- a/models/wildfire_fm/configs/reference_fireprone_seed_1_20260504.json +++ /dev/null @@ -1,62 +0,0 @@ -{ - "index_root": "", - "run_root": "", - "tile_size": 32, - "max_positive_tiles_per_sample": 64, - "min_negative_tiles_per_sample": 4, - "negative_to_positive_ratio": 3.0, - "positive_rate_source": "tiles", - "pos_weight_source": "tiles", - "pos_weight_cap": 50.0, - "batch_size": 32, - "eval_batch_size": 2, - "epochs": 10, - "learning_rate": 0.0002, - "weight_decay": 0.0001, - "base_channels": 32, - "dropout": 0.1, - "norm_type": "group", - "norm_groups": 8, - "init_head_bias_from_positive_rate": true, - "augment_flip": true, - "threshold": 0.5, - "metric_thresholds": [ - 0.1, - 0.2, - 0.3, - 0.5, - 0.7, - 0.9 - ], - "topk_area_fractions": [ - 0.01, - 0.05, - 0.1 - ], - "fss_radii": [ - 1, - 2, - 4, - 8 - ], - "reliability_bins": 10, - "amp": true, - "seed": 1, - "boundary_radii": [ - 1, - 2, - 4 - ], - "coarsen_factors": [ - 2, - 4, - 8 - ], - "use_aux_spatial_head": true, - "aux_spatial_radius": 2, - "aux_spatial_loss_weight": 0.5, - "train_target_mode": "dilate_max", - "train_target_radius": 2, - "loss_type": "bce", - "init_checkpoint": "" -} diff --git a/models/wildfire_fm/configs/reference_fireprone_seed_42_20260504.json b/models/wildfire_fm/configs/reference_fireprone_seed_42_20260504.json deleted file mode 100644 index 80e76025a665d23a78615e3958f40205af529ef1..0000000000000000000000000000000000000000 --- a/models/wildfire_fm/configs/reference_fireprone_seed_42_20260504.json +++ /dev/null @@ -1,62 +0,0 @@ -{ - "index_root": "", - "run_root": "", - "tile_size": 32, - "max_positive_tiles_per_sample": 64, - "min_negative_tiles_per_sample": 4, - "negative_to_positive_ratio": 3.0, - "positive_rate_source": "tiles", - "pos_weight_source": "tiles", - "pos_weight_cap": 50.0, - "batch_size": 32, - "eval_batch_size": 2, - "epochs": 10, - "learning_rate": 0.0002, - "weight_decay": 0.0001, - "base_channels": 32, - "dropout": 0.1, - "norm_type": "group", - "norm_groups": 8, - "init_head_bias_from_positive_rate": true, - "augment_flip": true, - "threshold": 0.5, - "metric_thresholds": [ - 0.1, - 0.2, - 0.3, - 0.5, - 0.7, - 0.9 - ], - "topk_area_fractions": [ - 0.01, - 0.05, - 0.1 - ], - "fss_radii": [ - 1, - 2, - 4, - 8 - ], - "reliability_bins": 10, - "amp": true, - "seed": 42, - "boundary_radii": [ - 1, - 2, - 4 - ], - "coarsen_factors": [ - 2, - 4, - 8 - ], - "use_aux_spatial_head": true, - "aux_spatial_radius": 2, - "aux_spatial_loss_weight": 0.5, - "train_target_mode": "dilate_max", - "train_target_radius": 2, - "loss_type": "bce", - "init_checkpoint": "" -} diff --git a/models/wildfire_fm/configs/reference_fireprone_seed_7_20260504.json b/models/wildfire_fm/configs/reference_fireprone_seed_7_20260504.json deleted file mode 100644 index b6db09ac28b0c59ca87ee0af6375685e9f60963a..0000000000000000000000000000000000000000 --- a/models/wildfire_fm/configs/reference_fireprone_seed_7_20260504.json +++ /dev/null @@ -1,62 +0,0 @@ -{ - "index_root": "", - "run_root": "", - "tile_size": 32, - "max_positive_tiles_per_sample": 64, - "min_negative_tiles_per_sample": 4, - "negative_to_positive_ratio": 3.0, - "positive_rate_source": "tiles", - "pos_weight_source": "tiles", - "pos_weight_cap": 50.0, - "batch_size": 32, - "eval_batch_size": 2, - "epochs": 10, - "learning_rate": 0.0002, - "weight_decay": 0.0001, - "base_channels": 32, - "dropout": 0.1, - "norm_type": "group", - "norm_groups": 8, - "init_head_bias_from_positive_rate": true, - "augment_flip": true, - "threshold": 0.5, - "metric_thresholds": [ - 0.1, - 0.2, - 0.3, - 0.5, - 0.7, - 0.9 - ], - "topk_area_fractions": [ - 0.01, - 0.05, - 0.1 - ], - "fss_radii": [ - 1, - 2, - 4, - 8 - ], - "reliability_bins": 10, - "amp": true, - "seed": 7, - "boundary_radii": [ - 1, - 2, - 4 - ], - "coarsen_factors": [ - 2, - 4, - 8 - ], - "use_aux_spatial_head": true, - "aux_spatial_radius": 2, - "aux_spatial_loss_weight": 0.5, - "train_target_mode": "dilate_max", - "train_target_radius": 2, - "loss_type": "bce", - "init_checkpoint": "" -} diff --git a/models/wildfire_fm/configs/reference_fireprone_seed_99_20260504.json b/models/wildfire_fm/configs/reference_fireprone_seed_99_20260504.json deleted file mode 100644 index 3f03e50805481bb8360776287420472b70a6cff1..0000000000000000000000000000000000000000 --- a/models/wildfire_fm/configs/reference_fireprone_seed_99_20260504.json +++ /dev/null @@ -1,62 +0,0 @@ -{ - "index_root": "", - "run_root": "", - "tile_size": 32, - "max_positive_tiles_per_sample": 64, - "min_negative_tiles_per_sample": 4, - "negative_to_positive_ratio": 3.0, - "positive_rate_source": "tiles", - "pos_weight_source": "tiles", - "pos_weight_cap": 50.0, - "batch_size": 32, - "eval_batch_size": 2, - "epochs": 10, - "learning_rate": 0.0002, - "weight_decay": 0.0001, - "base_channels": 32, - "dropout": 0.1, - "norm_type": "group", - "norm_groups": 8, - "init_head_bias_from_positive_rate": true, - "augment_flip": true, - "threshold": 0.5, - "metric_thresholds": [ - 0.1, - 0.2, - 0.3, - 0.5, - 0.7, - 0.9 - ], - "topk_area_fractions": [ - 0.01, - 0.05, - 0.1 - ], - "fss_radii": [ - 1, - 2, - 4, - 8 - ], - "reliability_bins": 10, - "amp": true, - "seed": 99, - "boundary_radii": [ - 1, - 2, - 4 - ], - "coarsen_factors": [ - 2, - 4, - 8 - ], - "use_aux_spatial_head": true, - "aux_spatial_radius": 2, - "aux_spatial_loss_weight": 0.5, - "train_target_mode": "dilate_max", - "train_target_radius": 2, - "loss_type": "bce", - "init_checkpoint": "" -} diff --git a/models/wildfire_fm/eval_only_manifest.json b/models/wildfire_fm/eval_only_manifest.json deleted file mode 100644 index 33afdd1012d955f22cdfb19578e016bfbece6e96..0000000000000000000000000000000000000000 --- a/models/wildfire_fm/eval_only_manifest.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "launched_at": "2026-05-04T00:25:21", - "purpose": "Eval-only repair for Reference Table 2 seeded checkpoints.", - "policy": "Reuse completed Reference checkpoints; only fire-prone readout summaries are generated.", - "job_id": "31780571", - "seeds": [ - 1, - 7, - 42, - 99, - 123 - ], - "run_base": "", - "train_root": "", - "output_root": "", - "evaluator": "" -} diff --git a/models/wildfire_fm/input_channels.json b/models/wildfire_fm/input_channels.json deleted file mode 100644 index bf30918a92b81a3952aaef30ea3f3c8c4b528953..0000000000000000000000000000000000000000 --- a/models/wildfire_fm/input_channels.json +++ /dev/null @@ -1,252 +0,0 @@ -{ - "tensor_order": "[channel, y, x]", - "channel_order_matters": true, - "value_normalization": { - "applied": false, - "description": "The released checkpoints expect the same native physical units used during training. No mean/std normalization, min/max scaling, or unit conversion is applied to input values before inference.", - "missing_or_invalid_values": "Training replaces values <= -9000, NaN, +inf, and -inf with 0.0. The fixed input also carries dynamic and static validity-mask channels." - }, - "grid": { - "region": "California", - "crs": "EPSG:5070", - "resolution_m": 5000, - "target_lead_hours": 12 - }, - "dynamic_weather_source": { - "dataset": "NOAA HRRR", - "product": "wrfsfcf00", - "forecast_hour": "f00", - "issue_hours_utc": [0, 6, 12, 18], - "reprojection": "Nearest-neighbor resampling from the HRRR grid to the 5 km EPSG:5070 regional grid.", - "cfgrib_selection_groups": { - "level2": { - "filter_by_keys": { - "typeOfLevel": "heightAboveGround", - "level": 2 - }, - "variables": ["t2m", "d2m"] - }, - "level10": { - "filter_by_keys": { - "typeOfLevel": "heightAboveGround", - "level": 10 - }, - "variables": ["u10", "v10"] - }, - "surface_instant": { - "filter_by_keys": { - "typeOfLevel": "surface", - "stepType": "instant" - }, - "variables": ["cape", "sp", "blh", "vis", "prate"] - }, - "surface_accum": { - "filter_by_keys": { - "typeOfLevel": "surface", - "stepType": "accum" - }, - "variables": ["tp"] - } - } - }, - "notes": [ - "FIRMS active-fire detections are used to derive the occupancy target, not as an input channel.", - "WFIGS and MTBS are supporting event-level resources and are not part of the pretrained occupancy input.", - "The regional cache contains zero firewx feature channels and one firewx_valid channel, so the released input has 16 channels total.", - "CAPE is selected from the HRRR surface instantaneous cfgrib group: typeOfLevel=surface and stepType=instant. It is not the most-unstable or layer CAPE channel.", - "firewx_valid is a cache-level dynamic input presence mask. In the released California regional cache it is 1.0 for every grid cell and is not a per-variable HRRR missing-data mask.", - "static_valid is the fraction of the four static layers that are valid at a grid cell after reprojection. It can take values 0.0, 0.25, 0.5, 0.75, or 1.0." - ], - "channels": [ - { - "index": 0, - "name": "t2m", - "source": "NOAA HRRR", - "cfgrib_group": "level2", - "cfgrib_filter_by_keys": { - "typeOfLevel": "heightAboveGround", - "level": 2 - }, - "level_or_height": "2 m above ground", - "units": "K", - "normalization": "none", - "role": "2 m temperature" - }, - { - "index": 1, - "name": "d2m", - "source": "NOAA HRRR", - "cfgrib_group": "level2", - "cfgrib_filter_by_keys": { - "typeOfLevel": "heightAboveGround", - "level": 2 - }, - "level_or_height": "2 m above ground", - "units": "K", - "normalization": "none", - "role": "2 m dew point" - }, - { - "index": 2, - "name": "u10", - "source": "NOAA HRRR", - "cfgrib_group": "level10", - "cfgrib_filter_by_keys": { - "typeOfLevel": "heightAboveGround", - "level": 10 - }, - "level_or_height": "10 m above ground", - "units": "m s-1", - "normalization": "none", - "role": "10 m east-west wind component" - }, - { - "index": 3, - "name": "v10", - "source": "NOAA HRRR", - "cfgrib_group": "level10", - "cfgrib_filter_by_keys": { - "typeOfLevel": "heightAboveGround", - "level": 10 - }, - "level_or_height": "10 m above ground", - "units": "m s-1", - "normalization": "none", - "role": "10 m north-south wind component" - }, - { - "index": 4, - "name": "cape", - "source": "NOAA HRRR", - "cfgrib_group": "surface_instant", - "cfgrib_filter_by_keys": { - "typeOfLevel": "surface", - "stepType": "instant" - }, - "level_or_height": "surface", - "units": "J kg-1", - "normalization": "none", - "role": "surface instantaneous convective available potential energy" - }, - { - "index": 5, - "name": "sp", - "source": "NOAA HRRR", - "cfgrib_group": "surface_instant", - "cfgrib_filter_by_keys": { - "typeOfLevel": "surface", - "stepType": "instant" - }, - "level_or_height": "surface", - "units": "Pa", - "normalization": "none", - "role": "surface pressure" - }, - { - "index": 6, - "name": "blh", - "source": "NOAA HRRR", - "cfgrib_group": "surface_instant", - "cfgrib_filter_by_keys": { - "typeOfLevel": "surface", - "stepType": "instant" - }, - "level_or_height": "surface diagnostic", - "units": "m", - "normalization": "none", - "role": "boundary-layer height" - }, - { - "index": 7, - "name": "vis", - "source": "NOAA HRRR", - "cfgrib_group": "surface_instant", - "cfgrib_filter_by_keys": { - "typeOfLevel": "surface", - "stepType": "instant" - }, - "level_or_height": "surface diagnostic", - "units": "m", - "normalization": "none", - "role": "visibility" - }, - { - "index": 8, - "name": "prate", - "source": "NOAA HRRR", - "cfgrib_group": "surface_instant", - "cfgrib_filter_by_keys": { - "typeOfLevel": "surface", - "stepType": "instant" - }, - "level_or_height": "surface", - "units": "kg m-2 s-1", - "normalization": "none", - "role": "precipitation rate" - }, - { - "index": 9, - "name": "tp", - "source": "NOAA HRRR", - "cfgrib_group": "surface_accum", - "cfgrib_filter_by_keys": { - "typeOfLevel": "surface", - "stepType": "accum" - }, - "level_or_height": "surface accumulation", - "units": "kg m-2", - "normalization": "none", - "role": "accumulated precipitation" - }, - { - "index": 10, - "name": "firewx_valid", - "source": "cache validity channel", - "units": "binary mask", - "normalization": "none", - "values": [1.0], - "role": "cache-level dynamic input presence mask for this regional cache; not a per-variable HRRR missing-data mask" - }, - { - "index": 11, - "name": "static_valid", - "source": "static reprojection mask", - "units": "fraction", - "normalization": "none", - "values": [0.0, 0.25, 0.5, 0.75, 1.0], - "role": "fraction of static layers valid at the grid cell after reprojection" - }, - { - "index": 12, - "name": "fuel_fbfm40", - "source": "LANDFIRE FBFM40", - "units": "categorical code", - "normalization": "none", - "role": "fire-behavior fuel model" - }, - { - "index": 13, - "name": "canopy_cover", - "source": "LANDFIRE CC", - "units": "provider-native canopy-cover value", - "normalization": "none", - "role": "canopy cover" - }, - { - "index": 14, - "name": "housing_density", - "source": "Wildfire Risk to Communities", - "units": "provider-native housing-density value", - "normalization": "none", - "role": "housing-unit density" - }, - { - "index": 15, - "name": "population", - "source": "LandScan Global 2024", - "units": "provider-native population value", - "normalization": "none", - "role": "population exposure" - } - ] -} diff --git a/models/wildfire_fm/manifest.json b/models/wildfire_fm/manifest.json deleted file mode 100644 index c9a800dd7aeea10082c92eb09893f0f03357a6c2..0000000000000000000000000000000000000000 --- a/models/wildfire_fm/manifest.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "launched_at": "2026-05-04T00:17:48", - "purpose": "Reference-only repair for Table 2: train and evaluate true seeded reference checkpoints.", - "policy": "Only Reference is rerun; existing normal backbone rows are not touched.", - "job_id": "31779899", - "seeds": [ - 1, - 7, - 42, - 99, - 123 - ], - "run_base": "", - "train_root": "", - "output_root": "", - "base_config": "", - "trainer": "", - "evaluator": "" -} diff --git a/models/wildfire_fm/metrics/reference_fireprone_seed_123_20260504/history.json b/models/wildfire_fm/metrics/reference_fireprone_seed_123_20260504/history.json deleted file mode 100644 index c1c5d03fbd7dd5bdc1439995f226cd92eb65fab7..0000000000000000000000000000000000000000 --- a/models/wildfire_fm/metrics/reference_fireprone_seed_123_20260504/history.json +++ /dev/null @@ -1,1682 +0,0 @@ -[ - { - "epoch": 1, - "train_loss": 0.5734782497771297, - "train_main_loss": 0.5351956177899154, - "train_aux_spatial_loss": 0.07656526550555741, - "val_loss": 0.37402819047371544, - "val_pr_auc": 0.003734718829113345, - "val_auroc": 0.8395962406652433, - "val_brier": 0.11044556647539139, - "val_brier_skill_score": -473.5761171163099, - "val_ece": 0.22885958394596995, - "val_positive_rate": 0.00023277674335986376, - "val_precision": 0.0009322534799702475, - "val_positive_recall": 0.5967056323060573, - "val_far": 0.9990677465200297, - "val_csi": 0.0009316664551128411, - "val_f1": 0.0018615985213304707, - "val_f2": 0.004632318485714474, - "val_frequency_bias": 640.068012752391, - "val_threshold_metrics": { - "0.1000": { - "threshold": 0.1, - "tp": 1820, - "fp": 4753782, - "fn": 62, - "precision": 0.00038270654272582106, - "recall": 0.9670563230605739, - "far": 0.9996172934572741, - "csi": 0.0003827015533477554, - "f1": 0.000765110297796062, - "f2": 0.0019105084261819435, - "frequency_bias": 2526.887353878852, - "predicted_positive_rate": 0.5882006184291898 - }, - "0.2000": { - "threshold": 0.2, - "tp": 1674, - "fp": 3286292, - "fn": 208, - "precision": 0.0005091293523108207, - "recall": 0.8894792773645058, - "far": 0.9994908706476892, - "csi": 0.0005090971463188992, - "f1": 0.0010176761965902375, - "f2": 0.0025398316610499064, - "frequency_bias": 1747.0595111583423, - "predicted_positive_rate": 0.4066748299319728 - }, - "0.3000": { - "threshold": 0.3, - "tp": 1532, - "fp": 2265579, - "fn": 350, - "precision": 0.0006757498860885065, - "recall": 0.8140276301806588, - "far": 0.9993242501139115, - "csi": 0.0006756455789096262, - "f1": 0.0013503787803664444, - "f2": 0.00336756733705876, - "frequency_bias": 1204.6285866099893, - "predicted_positive_rate": 0.28040952380952383 - }, - "0.5000": { - "threshold": 0.5, - "tp": 1123, - "fp": 1203485, - "fn": 759, - "precision": 0.0009322534799702475, - "recall": 0.5967056323060573, - "far": 0.9990677465200297, - "csi": 0.0009316664551128411, - "f1": 0.0018615985213304707, - "f2": 0.004632318485714474, - "frequency_bias": 640.068012752391, - "predicted_positive_rate": 0.14899294990723563 - }, - "0.7000": { - "threshold": 0.7, - "tp": 749, - "fp": 580251, - "fn": 1133, - "precision": 0.001289156626506024, - "recall": 0.39798087141339, - "far": 0.998710843373494, - "csi": 0.0012866475530505916, - "f1": 0.0025699884367676473, - "f2": 0.006363333605198053, - "frequency_bias": 308.7141339001063, - "predicted_positive_rate": 0.07186147186147186 - }, - "0.9000": { - "threshold": 0.9, - "tp": 451, - "fp": 151874, - "fn": 1431, - "precision": 0.002960774659445265, - "recall": 0.23963868225292242, - "far": 0.9970392253405548, - "csi": 0.002933218866255626, - "f1": 0.005849280512557796, - "f2": 0.014106710540308908, - "frequency_bias": 80.93783209351753, - "predicted_positive_rate": 0.0188404452690167 - } - }, - "val_topk_area_metrics": { - "0.0100": { - "fraction": 0.01, - "cells": 80850, - "precision": 0.0040197898633778095, - "recall": 0.17268863320350647, - "lift": 17.268863741956263 - }, - "0.0500": { - "fraction": 0.05, - "cells": 404250, - "precision": 0.0016227582236751914, - "recall": 0.34856536984443665, - "lift": 6.971307357653297 - }, - "0.1000": { - "fraction": 0.1, - "cells": 808500, - "precision": 0.0011020408710464835, - "recall": 0.4734325110912323, - "lift": 4.7343254963524055 - } - }, - "val_fss": { - "0.1000": { - "1": 0.0007940178513994312, - "2": 0.0008116663451843698, - "4": 0.0008402116520870306, - "8": 0.0008768156583047748 - }, - "0.2000": { - "1": 0.0010809907649410588, - "2": 0.0011167526746238554, - "4": 0.001173494894416982, - "8": 0.0012204541855455053 - }, - "0.3000": { - "1": 0.0014536022725181175, - "2": 0.0015044478327594923, - "4": 0.001584095073473124, - "8": 0.001635036482863117 - }, - "0.5000": { - "1": 0.0020614198969559627, - "2": 0.0021684921567577398, - "4": 0.002317489707137632, - "8": 0.002385432991275249 - }, - "0.7000": { - "1": 0.0030144855102398616, - "2": 0.003272350672034019, - "4": 0.003663210167958564, - "8": 0.003844188110136648 - }, - "0.9000": { - "1": 0.007369068053339922, - "2": 0.008060652520539557, - "4": 0.009407819114658267, - "8": 0.01158882476665346 - } - }, - "minutes": 0.3749832232793172 - }, - { - "epoch": 2, - "train_loss": 0.5280527199025703, - "train_main_loss": 0.49724071630821165, - "train_aux_spatial_loss": 0.061624006741304034, - "val_loss": 0.3316249119738738, - "val_pr_auc": 0.001767264439689972, - "val_auroc": 0.8436037224883574, - "val_brier": 0.08831143379211426, - "val_brier_skill_score": -378.46744883936856, - "val_ece": 0.20197589957727682, - "val_positive_rate": 0.00023277674335986376, - "val_precision": 0.0012098819434426726, - "val_positive_recall": 0.5526036131774708, - "val_far": 0.9987901180565574, - "val_csi": 0.0012086979765930988, - "val_f1": 0.0024144775790219046, - "val_f2": 0.005996890842747683, - "val_frequency_bias": 456.74176408076517, - "val_threshold_metrics": { - "0.1000": { - "threshold": 0.1, - "tp": 1785, - "fp": 4455552, - "fn": 97, - "precision": 0.000400463325972436, - "recall": 0.9484590860786397, - "far": 0.9995995366740276, - "csi": 0.0004004546113301958, - "f1": 0.0008005886232544309, - "f2": 0.0019989406174654775, - "frequency_bias": 2368.40435706695, - "predicted_positive_rate": 0.5513094619666048 - }, - "0.2000": { - "threshold": 0.2, - "tp": 1586, - "fp": 2764970, - "fn": 296, - "precision": 0.0005732759430859163, - "recall": 0.842720510095643, - "far": 0.9994267240569141, - "csi": 0.0005732146135752834, - "f1": 0.0011457724536363106, - "f2": 0.0028586012536029916, - "frequency_bias": 1470.008501594049, - "predicted_positive_rate": 0.34218379715522573 - }, - "0.3000": { - "threshold": 0.3, - "tp": 1385, - "fp": 1741264, - "fn": 497, - "precision": 0.000794767047179323, - "recall": 0.7359192348565357, - "far": 0.9992052329528207, - "csi": 0.0007945404458375833, - "f1": 0.0015878193050166494, - "f2": 0.003956742660885157, - "frequency_bias": 925.9558979808714, - "predicted_positive_rate": 0.21554100185528757 - }, - "0.5000": { - "threshold": 0.5, - "tp": 1040, - "fp": 858548, - "fn": 842, - "precision": 0.0012098819434426726, - "recall": 0.5526036131774708, - "far": 0.9987901180565574, - "csi": 0.0012086979765930988, - "f1": 0.0024144775790219046, - "f2": 0.005996890842747683, - "frequency_bias": 456.74176408076517, - "predicted_positive_rate": 0.10631886209029066 - }, - "0.7000": { - "threshold": 0.7, - "tp": 786, - "fp": 438309, - "fn": 1096, - "precision": 0.0017900454343593073, - "recall": 0.41764080765143463, - "far": 0.9982099545656407, - "csi": 0.0017855885286159872, - "f1": 0.0035648117702283795, - "f2": 0.008799367699379567, - "frequency_bias": 233.31296493092455, - "predicted_positive_rate": 0.05430983302411874 - }, - "0.9000": { - "threshold": 0.9, - "tp": 490, - "fp": 167018, - "fn": 1392, - "precision": 0.0029252334216873224, - "recall": 0.2603613177470776, - "far": 0.9970747665783127, - "csi": 0.002901124925991711, - "f1": 0.005785465493830804, - "f2": 0.01399712059233529, - "frequency_bias": 89.00531349628055, - "predicted_positive_rate": 0.020718367346938774 - } - }, - "val_topk_area_metrics": { - "0.0100": { - "fraction": 0.01, - "cells": 80850, - "precision": 0.0032776747830212116, - "recall": 0.14080765843391418, - "lift": 14.080765697258915 - }, - "0.0500": { - "fraction": 0.05, - "cells": 404250, - "precision": 0.0018528138753026724, - "recall": 0.39798086881637573, - "lift": 7.959617651486319 - }, - "0.1000": { - "fraction": 0.1, - "cells": 808500, - "precision": 0.0012665429385378957, - "recall": 0.5441020131111145, - "lift": 5.4410200961522595 - } - }, - "val_fss": { - "0.1000": { - "1": 0.0008428515962385363, - "2": 0.0008708457132048375, - "4": 0.0009155129443840782, - "8": 0.0009615746721725049 - }, - "0.2000": { - "1": 0.0012411931023150746, - "2": 0.001295326160666077, - "4": 0.0013845295073414698, - "8": 0.0014592629037508686 - }, - "0.3000": { - "1": 0.0017602573186886739, - "2": 0.0018528354423298365, - "4": 0.0020113329115427625, - "8": 0.0021305195762054696 - }, - "0.5000": { - "1": 0.0027948559926561645, - "2": 0.0030474081604696668, - "4": 0.0034677294952394355, - "8": 0.0037623477486223855 - }, - "0.7000": { - "1": 0.004259588970479955, - "2": 0.004814858068874828, - "4": 0.005858539237715177, - "8": 0.0067267307965462875 - }, - "0.9000": { - "1": 0.007704160826285555, - "2": 0.009190232441330881, - "4": 0.01342071380106813, - "8": 0.02008413371433937 - } - }, - "minutes": 0.34824419021606445 - }, - { - "epoch": 3, - "train_loss": 0.5097358830169797, - "train_main_loss": 0.48056654829095874, - "train_aux_spatial_loss": 0.05833866919872711, - "val_loss": 0.23263437176744142, - "val_pr_auc": 0.005799829091682691, - "val_auroc": 0.858433675989069, - "val_brier": 0.05747459828853607, - "val_brier_skill_score": -245.96393489611552, - "val_ece": 0.1401309918099933, - "val_positive_rate": 0.00023277674335986376, - "val_precision": 0.0016207128199668883, - "val_positive_recall": 0.4691817215727949, - "val_far": 0.9983792871800331, - "val_csi": 0.0016177464773249838, - "val_f1": 0.003230267201264304, - "val_f2": 0.007993120304154975, - "val_frequency_bias": 289.4909670563231, - "val_threshold_metrics": { - "0.1000": { - "threshold": 0.1, - "tp": 1708, - "fp": 3257028, - "fn": 174, - "precision": 0.0005241296011705152, - "recall": 0.9075451647183846, - "far": 0.9994758703988295, - "csi": 0.0005241016167982546, - "f1": 0.001047654156359316, - "f2": 0.002614608004741809, - "frequency_bias": 1731.528161530287, - "predicted_positive_rate": 0.4030594928880643 - }, - "0.2000": { - "threshold": 0.2, - "tp": 1407, - "fp": 1647128, - "fn": 475, - "precision": 0.0008534850640113798, - "recall": 0.7476089266737513, - "far": 0.9991465149359886, - "csi": 0.0008532392162570269, - "f1": 0.0017050236394802042, - "f2": 0.004248026796082034, - "frequency_bias": 875.9484590860786, - "predicted_positive_rate": 0.2039004329004329 - }, - "0.3000": { - "threshold": 0.3, - "tp": 1191, - "fp": 1076231, - "fn": 691, - "precision": 0.0011054164477799786, - "recall": 0.6328374070138151, - "far": 0.99889458355222, - "csi": 0.0011047079480536828, - "f1": 0.0022069778301572127, - "f2": 0.005488732199640537, - "frequency_bias": 572.4877789585547, - "predicted_positive_rate": 0.13326184291898577 - }, - "0.5000": { - "threshold": 0.5, - "tp": 883, - "fp": 543939, - "fn": 999, - "precision": 0.0016207128199668883, - "recall": 0.4691817215727949, - "far": 0.9983792871800331, - "csi": 0.0016177464773249838, - "f1": 0.003230267201264304, - "f2": 0.007993120304154975, - "frequency_bias": 289.4909670563231, - "predicted_positive_rate": 0.06738676561533705 - }, - "0.7000": { - "threshold": 0.7, - "tp": 672, - "fp": 275187, - "fn": 1210, - "precision": 0.0024360271008014964, - "recall": 0.357066950053135, - "far": 0.9975639728991985, - "csi": 0.002425388621606892, - "f1": 0.004839040688987222, - "f2": 0.011856577754095989, - "frequency_bias": 146.57757704569607, - "predicted_positive_rate": 0.03411985157699443 - }, - "0.9000": { - "threshold": 0.9, - "tp": 509, - "fp": 128114, - "fn": 1373, - "precision": 0.003957301571258639, - "recall": 0.27045696068012753, - "far": 0.9960426984287414, - "csi": 0.003915505092464384, - "f1": 0.007800467415041569, - "f2": 0.018692481142261165, - "frequency_bias": 68.34378320935176, - "predicted_positive_rate": 0.015908843537414966 - } - }, - "val_topk_area_metrics": { - "0.0100": { - "fraction": 0.01, - "cells": 80850, - "precision": 0.004959802143275738, - "recall": 0.21307119727134705, - "lift": 21.307120598418535 - }, - "0.0500": { - "fraction": 0.05, - "cells": 404250, - "precision": 0.0019146567210555077, - "recall": 0.4112645983695984, - "lift": 8.225292155133914 - }, - "0.1000": { - "fraction": 0.1, - "cells": 808500, - "precision": 0.0012863327283412218, - "recall": 0.5526036024093628, - "lift": 5.526036277398218 - } - }, - "val_fss": { - "0.1000": { - "1": 0.0011067289165310745, - "2": 0.0011399809349705858, - "4": 0.0011913783552172097, - "8": 0.0012370094558790123 - }, - "0.2000": { - "1": 0.0018505622473496963, - "2": 0.0019252785696999286, - "4": 0.002041576664051692, - "8": 0.002124135134588423 - }, - "0.3000": { - "1": 0.00242666478591258, - "2": 0.0025485806202928396, - "4": 0.0027490318766942856, - "8": 0.0029084132219412417 - }, - "0.5000": { - "1": 0.003769582021368678, - "2": 0.0041531729728947475, - "4": 0.004731106915223382, - "8": 0.005255388249188031 - }, - "0.7000": { - "1": 0.006126287350098902, - "2": 0.0069839388676512515, - "4": 0.008523715958140055, - "8": 0.01053433503604162 - }, - "0.9000": { - "1": 0.009785190427335055, - "2": 0.011164824863941325, - "4": 0.014373278762976782, - "8": 0.019307251722885388 - } - }, - "minutes": 0.35075352191925047 - }, - { - "epoch": 4, - "train_loss": 0.49218875483820523, - "train_main_loss": 0.46398333852538587, - "train_aux_spatial_loss": 0.05641083258067807, - "val_loss": 0.4577095299959183, - "val_pr_auc": 0.0025830442889536323, - "val_auroc": 0.8551343969226862, - "val_brier": 0.1315883994102478, - "val_brier_skill_score": -564.4252465043933, - "val_ece": 0.2563538164779148, - "val_positive_rate": 0.00023277674335986376, - "val_precision": 0.0009896558085788286, - "val_positive_recall": 0.7353878852284803, - "val_far": 0.9990103441914212, - "val_csi": 0.0009893035131711752, - "val_f1": 0.0019766515180512276, - "val_f2": 0.004921784872481676, - "val_frequency_bias": 743.0743889479278, - "val_threshold_metrics": { - "0.1000": { - "threshold": 0.1, - "tp": 1811, - "fp": 4984120, - "fn": 71, - "precision": 0.00036322203415971863, - "recall": 0.9622741764080766, - "far": 0.9996367779658403, - "csi": 0.00036321686192664987, - "f1": 0.0007261699666767779, - "f2": 0.0018133722535821364, - "frequency_bias": 2649.2725823591923, - "predicted_positive_rate": 0.6166890538033395 - }, - "0.2000": { - "threshold": 0.2, - "tp": 1689, - "fp": 3529525, - "fn": 193, - "precision": 0.0004783057611348392, - "recall": 0.8974495217853348, - "far": 0.9995216942388652, - "csi": 0.0004782796205591709, - "f1": 0.0009561019570371142, - "f2": 0.002386441283371322, - "frequency_bias": 1876.3092454835282, - "predicted_positive_rate": 0.4367611626468769 - }, - "0.3000": { - "threshold": 0.3, - "tp": 1564, - "fp": 2582550, - "fn": 318, - "precision": 0.0006052364562863713, - "recall": 0.8310308182784272, - "far": 0.9993947635437136, - "csi": 0.0006051619853027667, - "f1": 0.001209591971526638, - "f2": 0.003017392062638281, - "frequency_bias": 1373.068012752391, - "predicted_positive_rate": 0.3196183055040198 - }, - "0.5000": { - "threshold": 0.5, - "tp": 1384, - "fp": 1397082, - "fn": 498, - "precision": 0.0009896558085788286, - "recall": 0.7353878852284803, - "far": 0.9990103441914212, - "csi": 0.0009893035131711752, - "f1": 0.0019766515180512276, - "f2": 0.004921784872481676, - "frequency_bias": 743.0743889479278, - "predicted_positive_rate": 0.1729704390847248 - }, - "0.7000": { - "threshold": 0.7, - "tp": 1057, - "fp": 755716, - "fn": 825, - "precision": 0.0013967200204024193, - "recall": 0.5616365568544102, - "far": 0.9986032799795976, - "csi": 0.001395199036956275, - "f1": 0.0027865103373733782, - "f2": 0.006914814974728543, - "frequency_bias": 402.11105207226353, - "predicted_positive_rate": 0.09360210265924551 - }, - "0.9000": { - "threshold": 0.9, - "tp": 696, - "fp": 261226, - "fn": 1186, - "precision": 0.0026572796481395223, - "recall": 0.3698193411264612, - "far": 0.9973427203518604, - "csi": 0.0026453015491737235, - "f1": 0.0052766447817318915, - "f2": 0.012915197624791243, - "frequency_bias": 139.1721572794899, - "predicted_positive_rate": 0.03239604205318491 - } - }, - "val_topk_area_metrics": { - "0.0100": { - "fraction": 0.01, - "cells": 80850, - "precision": 0.004947433713823557, - "recall": 0.21253985166549683, - "lift": 21.253986297827947 - }, - "0.0500": { - "fraction": 0.05, - "cells": 404250, - "precision": 0.0019517624750733376, - "recall": 0.41923484206199646, - "lift": 8.384697057368781 - }, - "0.1000": { - "fraction": 0.1, - "cells": 808500, - "precision": 0.0013469387777149677, - "recall": 0.5786397457122803, - "lift": 5.7863975510330645 - } - }, - "val_fss": { - "0.1000": { - "1": 0.0007600908508323956, - "2": 0.0007824004837565512, - "4": 0.0008174067935393659, - "8": 0.0008576593024997026 - }, - "0.2000": { - "1": 0.0010137899344645485, - "2": 0.001052954821482821, - "4": 0.001117004853520176, - "8": 0.0011786437603573674 - }, - "0.3000": { - "1": 0.0013153040680959904, - "2": 0.0013806229283441773, - "4": 0.0014748630018216469, - "8": 0.0015478778504813828 - }, - "0.5000": { - "1": 0.002183108279669388, - "2": 0.0022919188759134013, - "4": 0.0024628992277530015, - "8": 0.0025537066150409515 - }, - "0.7000": { - "1": 0.003187314180517764, - "2": 0.003451823023046807, - "4": 0.003854865378136174, - "8": 0.004092752948271272 - }, - "0.9000": { - "1": 0.006778339261959476, - "2": 0.008030481875325624, - "4": 0.010576859714348186, - "8": 0.013750931457852666 - } - }, - "minutes": 0.3167848944664001 - }, - { - "epoch": 5, - "train_loss": 0.4798422370943648, - "train_main_loss": 0.4526658858268445, - "train_aux_spatial_loss": 0.05435270264757918, - "val_loss": 0.22279856478174528, - "val_pr_auc": 0.002003247222421217, - "val_auroc": 0.8534472608933964, - "val_brier": 0.0562748983502388, - "val_brier_skill_score": -240.80891639613208, - "val_ece": 0.13882232630158547, - "val_positive_rate": 0.00023277674335986376, - "val_precision": 0.0015668106718637735, - "val_positive_recall": 0.45536663124335813, - "val_far": 0.9984331893281362, - "val_csi": 0.0015638800283213747, - "val_f1": 0.003122876252840014, - "val_f2": 0.007727696533266967, - "val_frequency_bias": 290.6328374070138, - "val_threshold_metrics": { - "0.1000": { - "threshold": 0.1, - "tp": 1660, - "fp": 3252099, - "fn": 222, - "precision": 0.0005101791497157596, - "recall": 0.8820403825717322, - "far": 0.9994898208502843, - "csi": 0.0005101443431906947, - "f1": 0.0010197684572715482, - "f2": 0.00254500753843498, - "frequency_bias": 1728.883634431456, - "predicted_positive_rate": 0.4024439084724799 - }, - "0.2000": { - "threshold": 0.2, - "tp": 1427, - "fp": 1717127, - "fn": 455, - "precision": 0.0008303492354619058, - "recall": 0.7582359192348566, - "far": 0.9991696507645381, - "csi": 0.0008301294524926862, - "f1": 0.0016588818183297725, - "f2": 0.004133639073925804, - "frequency_bias": 913.15302869288, - "predicted_positive_rate": 0.21256079158936303 - }, - "0.3000": { - "threshold": 0.3, - "tp": 1228, - "fp": 1043050, - "fn": 654, - "precision": 0.0011759320793888217, - "recall": 0.6524973432518597, - "far": 0.9988240679206112, - "csi": 0.0011751960893149028, - "f1": 0.002347633249216181, - "f2": 0.005837578412749118, - "frequency_bias": 554.8767268862912, - "predicted_positive_rate": 0.12916239950525665 - }, - "0.5000": { - "threshold": 0.5, - "tp": 857, - "fp": 546114, - "fn": 1025, - "precision": 0.0015668106718637735, - "recall": 0.45536663124335813, - "far": 0.9984331893281362, - "csi": 0.0015638800283213747, - "f1": 0.003122876252840014, - "f2": 0.007727696533266967, - "frequency_bias": 290.6328374070138, - "predicted_positive_rate": 0.0676525664811379 - }, - "0.7000": { - "threshold": 0.7, - "tp": 663, - "fp": 274830, - "fn": 1219, - "precision": 0.00240659472291492, - "recall": 0.3522848034006376, - "far": 0.9975934052770851, - "csi": 0.0023959929457341928, - "f1": 0.004780531771068049, - "f2": 0.011712911762731388, - "frequency_bias": 146.38310308182784, - "predicted_positive_rate": 0.03407458256029684 - }, - "0.9000": { - "threshold": 0.9, - "tp": 351, - "fp": 95622, - "fn": 1531, - "precision": 0.0036572786096089526, - "recall": 0.18650371944739638, - "far": 0.996342721390391, - "csi": 0.003599852313751231, - "f1": 0.0071738797199938685, - "f2": 0.016956357909585416, - "frequency_bias": 50.995217853347505, - "predicted_positive_rate": 0.011870500927643786 - } - }, - "val_topk_area_metrics": { - "0.0100": { - "fraction": 0.01, - "cells": 80850, - "precision": 0.0037229438312351704, - "recall": 0.15993623435497284, - "lift": 15.993624524077324 - }, - "0.0500": { - "fraction": 0.05, - "cells": 404250, - "precision": 0.001907235593535006, - "recall": 0.40967056155204773, - "lift": 8.193411274710096 - }, - "0.1000": { - "fraction": 0.1, - "cells": 808500, - "precision": 0.00128262210637331, - "recall": 0.5510095357894897, - "lift": 5.51009558712842 - } - }, - "val_fss": { - "0.1000": { - "1": 0.0010862218732200501, - "2": 0.0011306068777684475, - "4": 0.0011982156126056909, - "8": 0.0012650951550728928 - }, - "0.2000": { - "1": 0.001835039598222954, - "2": 0.0019328571658292493, - "4": 0.002076142578762763, - "8": 0.0021710459616259836 - }, - "0.3000": { - "1": 0.002590627468139206, - "2": 0.0027374592974450662, - "4": 0.0029640942360077416, - "8": 0.0031140683173659367 - }, - "0.5000": { - "1": 0.0036841942890660784, - "2": 0.003975892870806685, - "4": 0.004437908744343022, - "8": 0.004817028530572198 - }, - "0.7000": { - "1": 0.005823401546553808, - "2": 0.0063452492211902856, - "4": 0.007432855129457017, - "8": 0.008860312688528271 - }, - "0.9000": { - "1": 0.00951053387465195, - "2": 0.011239285109680752, - "4": 0.016969728658484562, - "8": 0.028380582988892455 - } - }, - "minutes": 0.3421301166216532 - }, - { - "epoch": 6, - "train_loss": 0.47200144252198706, - "train_main_loss": 0.4454874005446253, - "train_aux_spatial_loss": 0.053028083716555464, - "val_loss": 0.20492182870705922, - "val_pr_auc": 0.006023304112441664, - "val_auroc": 0.8747266650018508, - "val_brier": 0.049230922013521194, - "val_brier_skill_score": -210.54149104245388, - "val_ece": 0.12249408376988412, - "val_positive_rate": 0.00023277674335986376, - "val_precision": 0.0018782184614922284, - "val_positive_recall": 0.46493092454835283, - "val_far": 0.9981217815385077, - "val_csi": 0.001874167334227222, - "val_f1": 0.0037413228034693822, - "val_f2": 0.009241753715184991, - "val_frequency_bias": 247.53825717322, - "val_threshold_metrics": { - "0.1000": { - "threshold": 0.1, - "tp": 1672, - "fp": 2715974, - "fn": 210, - "precision": 0.0006152383349413426, - "recall": 0.8884165781083954, - "far": 0.9993847616650586, - "csi": 0.0006151907974521093, - "f1": 0.0012296251408332622, - "f2": 0.0030676940261429174, - "frequency_bias": 1444.020191285866, - "predicted_positive_rate": 0.3361343228200371 - }, - "0.2000": { - "threshold": 0.2, - "tp": 1451, - "fp": 1391802, - "fn": 431, - "precision": 0.0010414476049934937, - "recall": 0.7709883103081828, - "far": 0.9989585523950065, - "csi": 0.0010411255349132228, - "f1": 0.00208008543976031, - "f2": 0.005179253573542188, - "frequency_bias": 740.3044633368756, - "predicted_positive_rate": 0.1723256648113791 - }, - "0.3000": { - "threshold": 0.3, - "tp": 1254, - "fp": 892851, - "fn": 628, - "precision": 0.0014025198382740282, - "recall": 0.6663124335812965, - "far": 0.998597480161726, - "csi": 0.0014015354301227293, - "f1": 0.0027991477554919885, - "f2": 0.006954048931217025, - "frequency_bias": 475.0823591923486, - "predicted_positive_rate": 0.11058812615955473 - }, - "0.5000": { - "threshold": 0.5, - "tp": 875, - "fp": 464992, - "fn": 1007, - "precision": 0.0018782184614922284, - "recall": 0.46493092454835283, - "far": 0.9981217815385077, - "csi": 0.001874167334227222, - "f1": 0.0037413228034693822, - "f2": 0.009241753715184991, - "frequency_bias": 247.53825717322, - "predicted_positive_rate": 0.057621150278293135 - }, - "0.7000": { - "threshold": 0.7, - "tp": 665, - "fp": 253520, - "fn": 1217, - "precision": 0.0026162047327733737, - "recall": 0.35334750265674814, - "far": 0.9973837952672266, - "csi": 0.002603738420215973, - "f1": 0.005193953145075312, - "f2": 0.012704756737342051, - "frequency_bias": 135.06110520722635, - "predicted_positive_rate": 0.03143908472479901 - }, - "0.9000": { - "threshold": 0.9, - "tp": 437, - "fp": 108236, - "fn": 1445, - "precision": 0.004021238026004619, - "recall": 0.23219978746014877, - "far": 0.9959787619739954, - "csi": 0.00396847018652718, - "f1": 0.007905567364660124, - "f2": 0.018803624753659605, - "frequency_bias": 57.74335812964931, - "predicted_positive_rate": 0.013441311069882498 - } - }, - "val_topk_area_metrics": { - "0.0100": { - "fraction": 0.01, - "cells": 80850, - "precision": 0.0047000618651509285, - "recall": 0.20191286504268646, - "lift": 20.191286282774463 - }, - "0.0500": { - "fraction": 0.05, - "cells": 404250, - "precision": 0.0020086579024791718, - "recall": 0.43145591020584106, - "lift": 8.62911764073383 - }, - "0.1000": { - "fraction": 0.1, - "cells": 808500, - "precision": 0.0014755720039829612, - "recall": 0.6339001059532166, - "lift": 6.339000978726575 - } - }, - "val_fss": { - "0.1000": { - "1": 0.0013248742601514873, - "2": 0.00138582233525264, - "4": 0.0014793932080273686, - "8": 0.0015689502796606414 - }, - "0.2000": { - "1": 0.0023088385065342942, - "2": 0.002449153490201028, - "4": 0.002681505121324146, - "8": 0.0028831512044252827 - }, - "0.3000": { - "1": 0.0031503711680238577, - "2": 0.0033971130498923197, - "4": 0.003843801518220169, - "8": 0.00423863352271503 - }, - "0.5000": { - "1": 0.004490733019564042, - "2": 0.00513267474781276, - "4": 0.006267791721161653, - "8": 0.007324417037690489 - }, - "0.7000": { - "1": 0.006796398619910549, - "2": 0.008020939278983064, - "4": 0.010531110485300088, - "8": 0.013871225077630944 - }, - "0.9000": { - "1": 0.011415450466378263, - "2": 0.013839835459906191, - "4": 0.019889957919249968, - "8": 0.0312791122896422 - } - }, - "minutes": 0.3271060546239217 - }, - { - "epoch": 7, - "train_loss": 0.459911651689757, - "train_main_loss": 0.43420963327302686, - "train_aux_spatial_loss": 0.05140403612300002, - "val_loss": 0.30195711702108385, - "val_pr_auc": 0.007462668847578005, - "val_auroc": 0.8774544235682473, - "val_brier": 0.07583270967006683, - "val_brier_skill_score": -324.84732963135656, - "val_ece": 0.15615654503375537, - "val_positive_rate": 0.00023277674335986376, - "val_precision": 0.0015267438380724627, - "val_positive_recall": 0.6126461211477152, - "val_far": 0.9984732561619275, - "val_csi": 0.001525271486418734, - "val_f1": 0.0030458971527597993, - "val_f2": 0.00755837583417461, - "val_frequency_bias": 401.2763018065887, - "val_threshold_metrics": { - "0.1000": { - "threshold": 0.1, - "tp": 1722, - "fp": 3083597, - "fn": 160, - "precision": 0.0005581270526645705, - "recall": 0.9149840595111584, - "far": 0.9994418729473354, - "csi": 0.000558098110536484, - "f1": 0.0011155736215426207, - "f2": 0.0027838428477063365, - "frequency_bias": 1639.3831030818278, - "predicted_positive_rate": 0.38161026592455166 - }, - "0.2000": { - "threshold": 0.2, - "tp": 1535, - "fp": 1830430, - "fn": 347, - "precision": 0.0008378981039484925, - "recall": 0.8156216790648246, - "far": 0.9991621018960515, - "csi": 0.0008377394242901864, - "f1": 0.001674076408773469, - "f2": 0.004172345314714435, - "frequency_bias": 973.413921360255, - "predicted_positive_rate": 0.22658812615955473 - }, - "0.3000": { - "threshold": 0.3, - "tp": 1416, - "fp": 1303639, - "fn": 466, - "precision": 0.0010850117428001119, - "recall": 0.7523910733262487, - "far": 0.9989149882571999, - "csi": 0.0010846244526131713, - "f1": 0.002166898633981592, - "f2": 0.005393944611502663, - "frequency_bias": 693.4404888416578, - "predicted_positive_rate": 0.16141682127396412 - }, - "0.5000": { - "threshold": 0.5, - "tp": 1153, - "fp": 754049, - "fn": 729, - "precision": 0.0015267438380724627, - "recall": 0.6126461211477152, - "far": 0.9984732561619275, - "csi": 0.001525271486418734, - "f1": 0.0030458971527597993, - "f2": 0.00755837583417461, - "frequency_bias": 401.2763018065887, - "predicted_positive_rate": 0.0934077922077922 - }, - "0.7000": { - "threshold": 0.7, - "tp": 959, - "fp": 448284, - "fn": 923, - "precision": 0.002134702154513259, - "recall": 0.5095642933049946, - "far": 0.9978652978454867, - "csi": 0.0021303252577937917, - "f1": 0.004251593239124411, - "f2": 0.010497601642836345, - "frequency_bias": 238.70510095642933, - "predicted_positive_rate": 0.05556499690785405 - }, - "0.9000": { - "threshold": 0.9, - "tp": 660, - "fp": 227755, - "fn": 1222, - "precision": 0.002889477486154587, - "recall": 0.35069075451647186, - "far": 0.9971105225138455, - "csi": 0.002874101299006693, - "f1": 0.005731729028167975, - "f2": 0.013986428925630344, - "frequency_bias": 121.3682252922423, - "predicted_positive_rate": 0.028251700680272107 - } - }, - "val_topk_area_metrics": { - "0.0100": { - "fraction": 0.01, - "cells": 80850, - "precision": 0.004168212879449129, - "recall": 0.1790648251771927, - "lift": 17.906483350895734 - }, - "0.0500": { - "fraction": 0.05, - "cells": 404250, - "precision": 0.0022708720061928034, - "recall": 0.4877789616584778, - "lift": 9.755579416634951 - }, - "0.1000": { - "fraction": 0.1, - "cells": 808500, - "precision": 0.0014594928361475468, - "recall": 0.6269925832748413, - "lift": 6.269925487750414 - } - }, - "val_fss": { - "0.1000": { - "1": 0.0011922256706897727, - "2": 0.001241923098517117, - "4": 0.0013199747764957026, - "8": 0.0014018572289743458 - }, - "0.2000": { - "1": 0.0018412087795650622, - "2": 0.0019516962520630399, - "4": 0.0021122719154029745, - "8": 0.002258269200665519 - }, - "0.3000": { - "1": 0.0023970400348517495, - "2": 0.0025539329427168855, - "4": 0.0027957403234514233, - "8": 0.0029940557272635226 - }, - "0.5000": { - "1": 0.003458294529205541, - "2": 0.003808782182815551, - "4": 0.00435669994126564, - "8": 0.004792095791161022 - }, - "0.7000": { - "1": 0.005004234974898769, - "2": 0.005603576851424319, - "4": 0.006716963044160917, - "8": 0.007943951762853563 - }, - "0.9000": { - "1": 0.007190456588270111, - "2": 0.00818745081073502, - "4": 0.010293219734911818, - "8": 0.0139886851310419 - } - }, - "minutes": 0.32838242053985595 - }, - { - "epoch": 8, - "train_loss": 0.4499193408563182, - "train_main_loss": 0.4247753525127238, - "train_aux_spatial_loss": 0.05028797703851233, - "val_loss": 0.2679826237261295, - "val_pr_auc": 0.004651516864174585, - "val_auroc": 0.8589783508732233, - "val_brier": 0.07260610163211823, - "val_brier_skill_score": -310.98284269020706, - "val_ece": 0.16960450200447197, - "val_positive_rate": 0.00023277674335986376, - "val_precision": 0.001397625988590239, - "val_positive_recall": 0.5324123273113709, - "val_far": 0.9986023740114097, - "val_csi": 0.0013959125673924854, - "val_f1": 0.0027879334234820786, - "val_f2": 0.006915514771042627, - "val_frequency_bias": 380.94048884165784, - "val_threshold_metrics": { - "0.1000": { - "threshold": 0.1, - "tp": 1766, - "fp": 3901854, - "fn": 116, - "precision": 0.00045240059227076405, - "recall": 0.9383634431455898, - "far": 0.9995475994077292, - "csi": 0.0004523871491309863, - "f1": 0.0009043651750786454, - "f2": 0.0022576491608090514, - "frequency_bias": 2074.1870350690756, - "predicted_positive_rate": 0.4828225108225108 - }, - "0.2000": { - "threshold": 0.2, - "tp": 1578, - "fp": 2284591, - "fn": 304, - "precision": 0.0006902376858403731, - "recall": 0.8384697130712009, - "far": 0.9993097623141596, - "csi": 0.0006901459146904424, - "f1": 0.001379339883595252, - "f2": 0.003439861498707109, - "frequency_bias": 1214.7550478214666, - "predicted_positive_rate": 0.2827667285095857 - }, - "0.3000": { - "threshold": 0.3, - "tp": 1380, - "fp": 1443686, - "fn": 502, - "precision": 0.000954973682862928, - "recall": 0.7332624867162593, - "far": 0.9990450263171371, - "csi": 0.0009546420507371497, - "f1": 0.001907463156934458, - "f2": 0.004750122883613728, - "frequency_bias": 767.8352816153028, - "predicted_positive_rate": 0.17873419913419913 - }, - "0.5000": { - "threshold": 0.5, - "tp": 1002, - "fp": 715928, - "fn": 880, - "precision": 0.001397625988590239, - "recall": 0.5324123273113709, - "far": 0.9986023740114097, - "csi": 0.0013959125673924854, - "f1": 0.0027879334234820786, - "f2": 0.006915514771042627, - "frequency_bias": 380.94048884165784, - "predicted_positive_rate": 0.08867408781694495 - }, - "0.7000": { - "threshold": 0.7, - "tp": 688, - "fp": 355882, - "fn": 1194, - "precision": 0.001929494909835376, - "recall": 0.3655685441020191, - "far": 0.9980705050901646, - "csi": 0.00192305542201004, - "f1": 0.0038387287558724744, - "f2": 0.00944800575669188, - "frequency_bias": 189.4633368756642, - "predicted_positive_rate": 0.04410265924551639 - }, - "0.9000": { - "threshold": 0.9, - "tp": 466, - "fp": 111220, - "fn": 1416, - "precision": 0.004172411940619236, - "recall": 0.24760892667375134, - "far": 0.9958275880593808, - "csi": 0.004120174709554208, - "f1": 0.008206537052690898, - "f2": 0.019544684349153626, - "frequency_bias": 59.34431455897981, - "predicted_positive_rate": 0.013813976499690786 - } - }, - "val_topk_area_metrics": { - "0.0100": { - "fraction": 0.01, - "cells": 80850, - "precision": 0.004873222205787897, - "recall": 0.20935174822807312, - "lift": 20.935176493358213 - }, - "0.0500": { - "fraction": 0.05, - "cells": 404250, - "precision": 0.001823129248805344, - "recall": 0.3916046619415283, - "lift": 7.83209362967527 - }, - "0.1000": { - "fraction": 0.1, - "cells": 808500, - "precision": 0.0012888064375147223, - "recall": 0.5536662936210632, - "lift": 5.53666323753949 - } - }, - "val_fss": { - "0.1000": { - "1": 0.0009531461813420128, - "2": 0.0009859296090429792, - "4": 0.0010304564828290674, - "8": 0.0010758667476578054 - }, - "0.2000": { - "1": 0.001509795785677337, - "2": 0.00158154075286987, - "4": 0.0016902515481980185, - "8": 0.0017673814967695822 - }, - "0.3000": { - "1": 0.0021042057052488294, - "2": 0.0022340194523658985, - "4": 0.0024437855961459976, - "8": 0.0025600611757777836 - }, - "0.5000": { - "1": 0.0031793578612200335, - "2": 0.0034648747296572857, - "4": 0.0039216381286044655, - "8": 0.004107196995981055 - }, - "0.7000": { - "1": 0.004753539211719615, - "2": 0.005295789241588578, - "4": 0.006140676468973316, - "8": 0.006471503504579701 - }, - "0.9000": { - "1": 0.011358743818156714, - "2": 0.013515668144941873, - "4": 0.017924796538675958, - "8": 0.02419228618876912 - } - }, - "minutes": 0.31785523096720375 - }, - { - "epoch": 9, - "train_loss": 0.43544893010666147, - "train_main_loss": 0.41090481131353246, - "train_aux_spatial_loss": 0.049088238095766304, - "val_loss": 0.35741244157155355, - "val_pr_auc": 0.002585840733061817, - "val_auroc": 0.8817129713606411, - "val_brier": 0.09716344624757767, - "val_brier_skill_score": -416.5038665412512, - "val_ece": 0.18796858609779482, - "val_positive_rate": 0.00023277674335986376, - "val_precision": 0.0012806769018722003, - "val_positive_recall": 0.7109458023379384, - "val_far": 0.9987193230981278, - "val_csi": 0.0012800104084553393, - "val_f1": 0.0025567481526634706, - "val_f2": 0.006357575112516725, - "val_frequency_bias": 555.1328374070138, - "val_threshold_metrics": { - "0.1000": { - "threshold": 0.1, - "tp": 1758, - "fp": 3553508, - "fn": 124, - "precision": 0.0004944777690333157, - "recall": 0.9341126461211477, - "far": 0.9995055222309667, - "csi": 0.000494460523318117, - "f1": 0.0009884323058810035, - "f2": 0.0024671648150300014, - "frequency_bias": 1889.0892667375133, - "predicted_positive_rate": 0.4397360544217687 - }, - "0.2000": { - "threshold": 0.2, - "tp": 1622, - "fp": 2315636, - "fn": 260, - "precision": 0.0006999652175113863, - "recall": 0.8618490967056323, - "far": 0.9993000347824886, - "csi": 0.0006998866891217241, - "f1": 0.0013987943806755953, - "f2": 0.003488493134421835, - "frequency_bias": 1231.2741764080765, - "predicted_positive_rate": 0.28661199752628325 - }, - "0.3000": { - "threshold": 0.3, - "tp": 1531, - "fp": 1735553, - "fn": 351, - "precision": 0.0008813620987816363, - "recall": 0.8134962805526036, - "far": 0.9991186379012184, - "csi": 0.0008811840442951822, - "f1": 0.0017608164851986754, - "f2": 0.004387795108597212, - "frequency_bias": 922.9989373007439, - "predicted_positive_rate": 0.2148526901669759 - }, - "0.5000": { - "threshold": 0.5, - "tp": 1338, - "fp": 1043422, - "fn": 544, - "precision": 0.0012806769018722003, - "recall": 0.7109458023379384, - "far": 0.9987193230981278, - "csi": 0.0012800104084553393, - "f1": 0.0025567481526634706, - "f2": 0.006357575112516725, - "frequency_bias": 555.1328374070138, - "predicted_positive_rate": 0.12922201607915892 - }, - "0.7000": { - "threshold": 0.7, - "tp": 1104, - "fp": 583644, - "fn": 778, - "precision": 0.0018879927763754644, - "recall": 0.5866099893730075, - "far": 0.9981120072236246, - "csi": 0.0018854841629577508, - "f1": 0.0037638716056117147, - "f2": 0.009319979198887005, - "frequency_bias": 310.7056323060574, - "predicted_positive_rate": 0.07232504638218924 - }, - "0.9000": { - "threshold": 0.9, - "tp": 774, - "fp": 249875, - "fn": 1108, - "precision": 0.0030879835945884482, - "recall": 0.4112646121147715, - "far": 0.9969120164054116, - "csi": 0.003074393164837522, - "f1": 0.0061299404825546164, - "f2": 0.014989716357382724, - "frequency_bias": 133.18225292242295, - "predicted_positive_rate": 0.031001731601731603 - } - }, - "val_topk_area_metrics": { - "0.0100": { - "fraction": 0.01, - "cells": 80850, - "precision": 0.0047000618651509285, - "recall": 0.20191286504268646, - "lift": 20.191286282774463 - }, - "0.0500": { - "fraction": 0.05, - "cells": 404250, - "precision": 0.002372294431552291, - "recall": 0.5095642805099487, - "lift": 10.191286282774461 - }, - "0.1000": { - "fraction": 0.1, - "cells": 808500, - "precision": 0.0015374149661511183, - "recall": 0.6604675650596619, - "lift": 6.604675982489947 - } - }, - "val_fss": { - "0.1000": { - "1": 0.001055759333565831, - "2": 0.0010976495838397327, - "4": 0.001157430376061197, - "8": 0.0012108268924492238 - }, - "0.2000": { - "1": 0.0015193550700235425, - "2": 0.0015891089547401815, - "4": 0.00168767572063655, - "8": 0.0017540287421734524 - }, - "0.3000": { - "1": 0.0019130861375810104, - "2": 0.00200484623513697, - "4": 0.0021368045034607652, - "8": 0.002225743866744012 - }, - "0.5000": { - "1": 0.0028767211501119494, - "2": 0.0030536082429117872, - "4": 0.0033209392323471087, - "8": 0.0035238602609450886 - }, - "0.7000": { - "1": 0.004462833938854716, - "2": 0.0048919187751156334, - "4": 0.005581633696340349, - "8": 0.006139927162454928 - }, - "0.9000": { - "1": 0.007988241741801017, - "2": 0.009144048690186368, - "4": 0.011405277940876535, - "8": 0.014317102888635747 - } - }, - "minutes": 0.34604887167612713 - }, - { - "epoch": 10, - "train_loss": 0.4286219271382876, - "train_main_loss": 0.404440414390365, - "train_aux_spatial_loss": 0.048363025166727865, - "val_loss": 0.42824133137861886, - "val_pr_auc": 0.003648303842240367, - "val_auroc": 0.8661930422066306, - "val_brier": 0.11499285697937012, - "val_brier_skill_score": -493.1154751878621, - "val_ece": 0.2244935538477554, - "val_positive_rate": 0.00023277674335986376, - "val_precision": 0.0011714322983281467, - "val_positive_recall": 0.7348565356004251, - "val_far": 0.9988285677016718, - "val_csi": 0.001170937384906507, - "val_f1": 0.0023391357882701557, - "val_f2": 0.005820050600353159, - "val_frequency_bias": 627.3145589798087, - "val_threshold_metrics": { - "0.1000": { - "threshold": 0.1, - "tp": 1753, - "fp": 4504075, - "fn": 129, - "precision": 0.0003890516903885368, - "recall": 0.9314558979808714, - "far": 0.9996109483096115, - "csi": 0.0003890405523177429, - "f1": 0.0007777785172515536, - "f2": 0.0019420138805802158, - "frequency_bias": 2394.1700318809776, - "predicted_positive_rate": 0.5573071119356834 - }, - "0.2000": { - "threshold": 0.2, - "tp": 1645, - "fp": 2894097, - "fn": 237, - "precision": 0.0005680754708119715, - "recall": 0.8740701381509033, - "far": 0.999431924529188, - "csi": 0.0005680289808731348, - "f1": 0.0011354130142489157, - "f2": 0.002833012430810775, - "frequency_bias": 1538.6514346439958, - "predicted_positive_rate": 0.3581622758194187 - }, - "0.3000": { - "threshold": 0.3, - "tp": 1523, - "fp": 2045614, - "fn": 359, - "precision": 0.0007439658410746325, - "recall": 0.8092454835281615, - "far": 0.9992560341589254, - "csi": 0.0007438353969922286, - "f1": 0.00148656503429202, - "f2": 0.0037062002808243683, - "frequency_bias": 1087.7454835281615, - "predicted_positive_rate": 0.25320185528756955 - }, - "0.5000": { - "threshold": 0.5, - "tp": 1383, - "fp": 1179223, - "fn": 499, - "precision": 0.0011714322983281467, - "recall": 0.7348565356004251, - "far": 0.9988285677016718, - "csi": 0.001170937384906507, - "f1": 0.0023391357882701557, - "f2": 0.005820050600353159, - "frequency_bias": 627.3145589798087, - "predicted_positive_rate": 0.14602424242424242 - }, - "0.7000": { - "threshold": 0.7, - "tp": 1242, - "fp": 689422, - "fn": 640, - "precision": 0.0017982694913879977, - "recall": 0.6599362380446334, - "far": 0.998201730508612, - "csi": 0.0017966046775369448, - "f1": 0.003586765355658686, - "f2": 0.008894401539977542, - "frequency_bias": 366.98405951115836, - "predicted_positive_rate": 0.08542535559678417 - }, - "0.9000": { - "threshold": 0.9, - "tp": 930, - "fp": 302454, - "fn": 952, - "precision": 0.0030654220393956173, - "recall": 0.49415515409139216, - "far": 0.9969345779606044, - "csi": 0.00305583302665475, - "f1": 0.006093046719909849, - "f2": 0.014956000411692054, - "frequency_bias": 161.2029755579171, - "predicted_positive_rate": 0.03752430426716141 - } - }, - "val_topk_area_metrics": { - "0.0100": { - "fraction": 0.01, - "cells": 80850, - "precision": 0.005058750975877047, - "recall": 0.2173219919204712, - "lift": 21.73220100453255 - }, - "0.0500": { - "fraction": 0.05, - "cells": 404250, - "precision": 0.002555349376052618, - "recall": 0.5488841533660889, - "lift": 10.977683333691749 - }, - "0.1000": { - "fraction": 0.1, - "cells": 808500, - "precision": 0.001611626474186778, - "recall": 0.6923485398292542, - "lift": 6.923485786959681 - } - }, - "val_fss": { - "0.1000": { - "1": 0.0008190413195137136, - "2": 0.0008461932775331027, - "4": 0.0008874666072198334, - "8": 0.0009354570390477379 - }, - "0.2000": { - "1": 0.0012240361098765362, - "2": 0.0012842780888633332, - "4": 0.0013672079974895723, - "8": 0.0014385782612603704 - }, - "0.3000": { - "1": 0.0016296457718533164, - "2": 0.00172595038714074, - "4": 0.0018654638060817685, - "8": 0.0019670592517716523 - }, - "0.5000": { - "1": 0.002614059790545964, - "2": 0.002783774308284359, - "4": 0.0030265840125089927, - "8": 0.0031953274262443987 - }, - "0.7000": { - "1": 0.004117212678526938, - "2": 0.00442237575814286, - "4": 0.00490498934248651, - "8": 0.005264629986009961 - }, - "0.9000": { - "1": 0.007561816117923059, - "2": 0.008478388329967612, - "4": 0.010294830494736118, - "8": 0.01224892406096545 - } - }, - "minutes": 0.31807684898376465 - } -] diff --git a/models/wildfire_fm/metrics/reference_fireprone_seed_123_20260504/run_summary.json b/models/wildfire_fm/metrics/reference_fireprone_seed_123_20260504/run_summary.json deleted file mode 100644 index 531fa93f54789e5e0a619f468f4c794be8015e09..0000000000000000000000000000000000000000 --- a/models/wildfire_fm/metrics/reference_fireprone_seed_123_20260504/run_summary.json +++ /dev/null @@ -1,172 +0,0 @@ -{ - "run_name": "reference_fireprone_seed_123_20260504", - "device": "cuda", - "index_root": "", - "input_channels": 16, - "tile_size": 32, - "num_train_tiles": 27177, - "positive_train_tiles": 6724, - "negative_train_tiles": 20453, - "train_positive_rate": 0.0018302686439820436, - "metric_reference_positive_rate": 0.0002787771234976204, - "pos_weight": 50.0, - "use_aux_spatial_head": true, - "aux_spatial_radius": 2, - "aux_spatial_loss_weight": 0.5, - "aux_positive_rate": 0.004560296846011132, - "init_checkpoint": "", - "best_val_pr_auc": 0.007462668847578005, - "best_val_auroc": 0.8774544235682473, - "best_test_pr_auc": 0.001212019793975473, - "best_test_auroc": 0.8262438111464772, - "best_test_brier": 0.08898080885410309, - "best_test_ece": 0.17619957022100255, - "best_test_topk_area_metrics": { - "0.0100": { - "fraction": 0.01, - "cells": 80850, - "precision": 0.0011750154662877321, - "recall": 0.04844466969370842, - "lift": 4.844467048285472 - }, - "0.0500": { - "fraction": 0.05, - "cells": 404250, - "precision": 0.001756338868290186, - "recall": 0.36206015944480896, - "lift": 7.241203215763695 - }, - "0.1000": { - "fraction": 0.1, - "cells": 808500, - "precision": 0.0012789116008207202, - "recall": 0.5272819995880127, - "lift": 5.272820048420403 - } - }, - "best_test_threshold_metrics": { - "0.1000": { - "threshold": 0.1, - "tp": 1706, - "fp": 3320161, - "fn": 255, - "precision": 0.0005135666178085998, - "recall": 0.8699643039265681, - "far": 0.9994864333821915, - "csi": 0.0005135271973756533, - "f1": 0.0010265272450921048, - "f2": 0.002561783890553865, - "frequency_bias": 1693.9658337582866, - "predicted_positive_rate": 0.4108679035250464 - }, - "0.2000": { - "threshold": 0.2, - "tp": 1496, - "fp": 2134207, - "fn": 465, - "precision": 0.0007004719289152097, - "recall": 0.7628760836308006, - "far": 0.9992995280710848, - "csi": 0.0007003194505301081, - "f1": 0.0013996586928535072, - "f2": 0.0034895432663711134, - "frequency_bias": 1089.0887302396736, - "predicted_positive_rate": 0.2641562152133581 - }, - "0.3000": { - "threshold": 0.3, - "tp": 1345, - "fp": 1593641, - "fn": 616, - "precision": 0.0008432675898095657, - "recall": 0.6858745537990821, - "far": 0.9991567324101904, - "csi": 0.0008429420369239948, - "f1": 0.0016844641681909294, - "f2": 0.004195703848817405, - "frequency_bias": 813.353391126976, - "predicted_positive_rate": 0.19727717996289426 - }, - "0.5000": { - "threshold": 0.5, - "tp": 1088, - "fp": 912330, - "fn": 873, - "precision": 0.0011911304572495833, - "recall": 0.5548189699133096, - "far": 0.9988088695427504, - "csi": 0.001189993120352273, - "f1": 0.002377157439705302, - "f2": 0.005904943436286311, - "frequency_bias": 465.7919428862825, - "predicted_positive_rate": 0.11297687074829932 - }, - "0.7000": { - "threshold": 0.7, - "tp": 836, - "fp": 535422, - "fn": 1125, - "precision": 0.0015589511018949834, - "recall": 0.4263131055583886, - "far": 0.998441048898105, - "csi": 0.0015556874705749903, - "f1": 0.003106542132477672, - "f2": 0.007682383082583779, - "frequency_bias": 273.46149923508415, - "predicted_positive_rate": 0.06632752009894867 - }, - "0.9000": { - "threshold": 0.9, - "tp": 405, - "fp": 246171, - "fn": 1556, - "precision": 0.00164249562001168, - "recall": 0.2065272819989801, - "far": 0.9983575043799883, - "csi": 0.0016321957667693003, - "f1": 0.003259072089869919, - "f2": 0.007959279930823049, - "frequency_bias": 125.73992860785313, - "predicted_positive_rate": 0.03049795918367347 - } - }, - "best_test_fss": { - "0.1000": { - "1": 0.0010957041077256369, - "2": 0.0011377779586465575, - "4": 0.0011947677181146643, - "8": 0.0012470182839290356 - }, - "0.2000": { - "1": 0.0015170543049650709, - "2": 0.0015877971019127024, - "4": 0.0016814901428712226, - "8": 0.0017498239432793827 - }, - "0.3000": { - "1": 0.0018421386324615208, - "2": 0.0019416047187071905, - "4": 0.00207931527885874, - "8": 0.0021687368541809526 - }, - "0.5000": { - "1": 0.0026440385644092013, - "2": 0.002817713478884265, - "4": 0.0030958883338918497, - "8": 0.003316757768246803 - }, - "0.7000": { - "1": 0.0035326072867271074, - "2": 0.0038065323590337208, - "4": 0.004319262090524356, - "8": 0.004943475244678974 - }, - "0.9000": { - "1": 0.004119845404175648, - "2": 0.004748959668431807, - "4": 0.006481934958671998, - "8": 0.009729806335369129 - } - }, - "epochs": 10 -} diff --git a/models/wildfire_fm/metrics/reference_fireprone_seed_123_20260504/tile_summary.json b/models/wildfire_fm/metrics/reference_fireprone_seed_123_20260504/tile_summary.json deleted file mode 100644 index 6bc5fdb7a28709bfd7523073af81f9402c86dc03..0000000000000000000000000000000000000000 --- a/models/wildfire_fm/metrics/reference_fireprone_seed_123_20260504/tile_summary.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "index_root": "", - "tile_size": 32, - "num_train_tiles": 27177, - "positive_train_tiles": 6724, - "negative_train_tiles": 20453, - "input_channels": 16, - "train_positive_rate": 0.0018302686439820436, - "metric_reference_positive_rate": 0.0002787771234976204, - "pos_weight_source": "tiles", - "positive_rate_source": "tiles", - "pos_weight": 50.0, - "loss_type": "bce", - "train_target_mode": "dilate_max", - "train_target_radius": 2 -} diff --git a/models/wildfire_fm/metrics/reference_fireprone_seed_1_20260504/history.json b/models/wildfire_fm/metrics/reference_fireprone_seed_1_20260504/history.json deleted file mode 100644 index 487fbf8b6931bc6c4f359964467e4de8afb6b7cd..0000000000000000000000000000000000000000 --- a/models/wildfire_fm/metrics/reference_fireprone_seed_1_20260504/history.json +++ /dev/null @@ -1,1682 +0,0 @@ -[ - { - "epoch": 1, - "train_loss": 0.573664065749939, - "train_main_loss": 0.5326367103663546, - "train_aux_spatial_loss": 0.08205470970182109, - "val_loss": 0.5349265058835347, - "val_pr_auc": 0.0018654076712032604, - "val_auroc": 0.8500520368869484, - "val_brier": 0.16269078850746155, - "val_brier_skill_score": -698.0698238454416, - "val_ece": 0.2954815917108688, - "val_positive_rate": 0.00023277674335986376, - "val_precision": 0.0008273187647548224, - "val_positive_recall": 0.7545164718384697, - "val_far": 0.9991726812352452, - "val_csi": 0.0008270961353641844, - "val_f1": 0.001652825225372031, - "val_f2": 0.00411853013719926, - "val_frequency_bias": 912.0021253985122, - "val_threshold_metrics": { - "0.1000": { - "threshold": 0.1, - "tp": 1857, - "fp": 5423702, - "fn": 25, - "precision": 0.00034226887957535803, - "recall": 0.9867162592986185, - "far": 0.9996577311204247, - "csi": 0.0003422673024691904, - "f1": 0.0006843003912893757, - "f2": 0.0017089731859622345, - "frequency_bias": 2882.8687566418703, - "predicted_positive_rate": 0.6710648113790971 - }, - "0.2000": { - "threshold": 0.2, - "tp": 1800, - "fp": 4467036, - "fn": 82, - "precision": 0.000402789451212799, - "recall": 0.9564293304994687, - "far": 0.9995972105487873, - "csi": 0.00040278206044505626, - "f1": 0.0008052397847504584, - "f2": 0.002010560356575113, - "frequency_bias": 2374.5143464399575, - "predicted_positive_rate": 0.5527317254174398 - }, - "0.3000": { - "threshold": 0.3, - "tp": 1720, - "fp": 3430554, - "fn": 162, - "precision": 0.000501125492894798, - "recall": 0.9139213602550478, - "far": 0.9994988745071052, - "csi": 0.0005011018413744641, - "f1": 0.001001701728168435, - "f2": 0.002500143903631663, - "frequency_bias": 1823.7375132837408, - "predicted_positive_rate": 0.42452368583797156 - }, - "0.5000": { - "threshold": 0.5, - "tp": 1420, - "fp": 1714968, - "fn": 462, - "precision": 0.0008273187647548224, - "recall": 0.7545164718384697, - "far": 0.9991726812352452, - "csi": 0.0008270961353641844, - "f1": 0.001652825225372031, - "f2": 0.00411853013719926, - "frequency_bias": 912.0021253985122, - "predicted_positive_rate": 0.21229288806431665 - }, - "0.7000": { - "threshold": 0.7, - "tp": 1050, - "fp": 918424, - "fn": 832, - "precision": 0.0011419572494708932, - "recall": 0.5579171094580234, - "far": 0.9988580427505291, - "csi": 0.0011409248662944717, - "f1": 0.002279249280408442, - "f2": 0.005663418201902477, - "frequency_bias": 488.56216790648244, - "predicted_positive_rate": 0.11372591218305504 - }, - "0.9000": { - "threshold": 0.9, - "tp": 610, - "fp": 339523, - "fn": 1272, - "precision": 0.0017934161048766219, - "recall": 0.3241232731137088, - "far": 0.9982065838951234, - "csi": 0.0017867342306058787, - "f1": 0.0035670950104527586, - "f2": 0.008772913844233319, - "frequency_bias": 180.72954303931988, - "predicted_positive_rate": 0.042069635126777985 - } - }, - "val_topk_area_metrics": { - "0.0100": { - "fraction": 0.01, - "cells": 80850, - "precision": 0.002622139872983098, - "recall": 0.11264611780643463, - "lift": 11.264612757853444 - }, - "0.0500": { - "fraction": 0.05, - "cells": 404250, - "precision": 0.0016697588143870234, - "recall": 0.3586609959602356, - "lift": 7.173220100453255 - }, - "0.1000": { - "fraction": 0.1, - "cells": 808500, - "precision": 0.0012207792606204748, - "recall": 0.5244420766830444, - "lift": 5.244421083480826 - } - }, - "val_fss": { - "0.1000": { - "1": 0.0006955746753622982, - "2": 0.0007043072962146235, - "4": 0.0007184011183636319, - "8": 0.0007378939517806993 - }, - "0.2000": { - "1": 0.0008314270074428753, - "2": 0.0008458942856476304, - "4": 0.0008682692460694197, - "8": 0.0008963995358247656 - }, - "0.3000": { - "1": 0.0010519556378738582, - "2": 0.0010779076249621822, - "4": 0.001118951638737209, - "8": 0.0011639559236974462 - }, - "0.5000": { - "1": 0.001817472102352835, - "2": 0.0019079406476985028, - "4": 0.0020492128618442296, - "8": 0.002166019917211992 - }, - "0.7000": { - "1": 0.0025795801528006868, - "2": 0.0027239390264135466, - "4": 0.0029668123834800664, - "8": 0.003214106060204891 - }, - "0.9000": { - "1": 0.004266565643152309, - "2": 0.004592005457379367, - "4": 0.005288208913823755, - "8": 0.006419555152632372 - } - }, - "minutes": 0.39989219506581625 - }, - { - "epoch": 2, - "train_loss": 0.5310182435958991, - "train_main_loss": 0.5007337570216748, - "train_aux_spatial_loss": 0.06056897338257285, - "val_loss": 0.38492824087540306, - "val_pr_auc": 0.007534030208235136, - "val_auroc": 0.8668780863986516, - "val_brier": 0.11073322594165802, - "val_brier_skill_score": -474.81216775110823, - "val_ece": 0.24345293157139367, - "val_positive_rate": 0.00023277674335986376, - "val_precision": 0.0011833787160754057, - "val_positive_recall": 0.6849096705632306, - "val_far": 0.9988166212839246, - "val_csi": 0.0011827348242459722, - "val_f1": 0.002362675230218781, - "val_f2": 0.005876281704112576, - "val_frequency_bias": 578.7747077577046, - "val_threshold_metrics": { - "0.1000": { - "threshold": 0.1, - "tp": 1846, - "fp": 5321276, - "fn": 36, - "precision": 0.00034678897083328166, - "recall": 0.9808714133900106, - "far": 0.9996532110291667, - "csi": 0.0003467866255331891, - "f1": 0.000693332812519953, - "f2": 0.0017314961590049992, - "frequency_bias": 2828.4388947927737, - "predicted_positive_rate": 0.6583948051948052 - }, - "0.2000": { - "threshold": 0.2, - "tp": 1770, - "fp": 3730075, - "fn": 112, - "precision": 0.00047429622612943465, - "recall": 0.9404888416578109, - "far": 0.9995257037738705, - "csi": 0.0004742819919950846, - "f1": 0.0009481143104463716, - "f2": 0.002366706931884035, - "frequency_bias": 1982.914452709883, - "predicted_positive_rate": 0.46157637600494744 - }, - "0.3000": { - "threshold": 0.3, - "tp": 1617, - "fp": 2614515, - "fn": 265, - "precision": 0.0006180880781244983, - "recall": 0.859192348565356, - "far": 0.9993819119218755, - "csi": 0.0006180254754916781, - "f1": 0.0012352875118314876, - "f2": 0.003081573069681285, - "frequency_bias": 1390.0807651434643, - "predicted_positive_rate": 0.3235784786641929 - }, - "0.5000": { - "threshold": 0.5, - "tp": 1289, - "fp": 1087965, - "fn": 593, - "precision": 0.0011833787160754057, - "recall": 0.6849096705632306, - "far": 0.9988166212839246, - "csi": 0.0011827348242459722, - "f1": 0.002362675230218781, - "f2": 0.005876281704112576, - "frequency_bias": 578.7747077577046, - "predicted_positive_rate": 0.1347252937538652 - }, - "0.7000": { - "threshold": 0.7, - "tp": 895, - "fp": 459555, - "fn": 987, - "precision": 0.0019437506786838961, - "recall": 0.47555791710945805, - "far": 0.9980562493213161, - "csi": 0.0019395930538730097, - "f1": 0.003871676630646375, - "f2": 0.009562415327216238, - "frequency_bias": 244.65993623804462, - "predicted_positive_rate": 0.05695114409400124 - }, - "0.9000": { - "threshold": 0.9, - "tp": 579, - "fp": 168755, - "fn": 1303, - "precision": 0.0034192778768587524, - "recall": 0.30765143464399575, - "far": 0.9965807221231412, - "csi": 0.003393167953023084, - "f1": 0.006763386599383235, - "f2": 0.01636869423618414, - "frequency_bias": 89.97555791710946, - "predicted_positive_rate": 0.02094421768707483 - } - }, - "val_topk_area_metrics": { - "0.0100": { - "fraction": 0.01, - "cells": 80850, - "precision": 0.004910327959805727, - "recall": 0.21094579994678497, - "lift": 21.094581395593078 - }, - "0.0500": { - "fraction": 0.05, - "cells": 404250, - "precision": 0.0020927644800394773, - "recall": 0.4495217800140381, - "lift": 8.99043628600021 - }, - "0.1000": { - "fraction": 0.1, - "cells": 808500, - "precision": 0.001413729158230126, - "recall": 0.6073326468467712, - "lift": 6.07332647507898 - } - }, - "val_fss": { - "0.1000": { - "1": 0.0007153574635551108, - "2": 0.0007299636456388203, - "4": 0.0007542019435764491, - "8": 0.0007881568769358971 - }, - "0.2000": { - "1": 0.0009921033280078095, - "2": 0.0010166009346802563, - "4": 0.0010565960391205609, - "8": 0.001105867798709581 - }, - "0.3000": { - "1": 0.0013214325599043564, - "2": 0.0013689794030732338, - "4": 0.0014429016059425148, - "8": 0.0015061326630954053 - }, - "0.5000": { - "1": 0.0026297043943714904, - "2": 0.0027883475350057862, - "4": 0.0030552534698978784, - "8": 0.003262873105559061 - }, - "0.7000": { - "1": 0.004579501009785769, - "2": 0.005086050780598539, - "4": 0.00593845439040841, - "8": 0.00666004273072307 - }, - "0.9000": { - "1": 0.008891211711206992, - "2": 0.010287198394993147, - "4": 0.013515825530875492, - "8": 0.01799509369630581 - } - }, - "minutes": 0.3222003976504008 - }, - { - "epoch": 3, - "train_loss": 0.5111621306671731, - "train_main_loss": 0.48230293577254363, - "train_aux_spatial_loss": 0.05771839129187639, - "val_loss": 0.3889197667439779, - "val_pr_auc": 0.002294898825423087, - "val_auroc": 0.8612537704398479, - "val_brier": 0.1059054285287857, - "val_brier_skill_score": -454.0674930344856, - "val_ece": 0.2274857362708612, - "val_positive_rate": 0.00023277674335986376, - "val_precision": 0.0012264433693911362, - "val_positive_recall": 0.7088204038257173, - "val_far": 0.9987735566306088, - "val_csi": 0.001225825778362613, - "val_f1": 0.002448649938508416, - "val_f2": 0.006090067255525344, - "val_frequency_bias": 577.9479277364505, - "val_threshold_metrics": { - "0.1000": { - "threshold": 0.1, - "tp": 1765, - "fp": 4849350, - "fn": 117, - "precision": 0.00036383388148910093, - "recall": 0.9378320935175345, - "far": 0.9996361661185109, - "csi": 0.0003638251066945469, - "f1": 0.0007273855722556596, - "f2": 0.0018163507794254484, - "frequency_bias": 2577.6381509032944, - "predicted_positive_rate": 0.6000142238713667 - }, - "0.2000": { - "threshold": 0.2, - "tp": 1663, - "fp": 3010913, - "fn": 219, - "precision": 0.0005520192685595318, - "recall": 0.8836344314558979, - "far": 0.9994479807314405, - "csi": 0.0005519791422914603, - "f1": 0.0011033492588053972, - "f2": 0.0027532164455263794, - "frequency_bias": 1600.7311370882042, - "predicted_positive_rate": 0.372612987012987 - }, - "0.3000": { - "threshold": 0.3, - "tp": 1557, - "fp": 2091911, - "fn": 325, - "precision": 0.0007437419630966415, - "recall": 0.8273113708820404, - "far": 0.9992562580369033, - "csi": 0.0007436265189538794, - "f1": 0.0014861478989190349, - "f2": 0.0037053854457600108, - "frequency_bias": 1112.3634431455898, - "predicted_positive_rate": 0.25893234384662955 - }, - "0.5000": { - "threshold": 0.5, - "tp": 1334, - "fp": 1086364, - "fn": 548, - "precision": 0.0012264433693911362, - "recall": 0.7088204038257173, - "far": 0.9987735566306088, - "csi": 0.001225825778362613, - "f1": 0.002448649938508416, - "f2": 0.006090067255525344, - "frequency_bias": 577.9479277364505, - "predicted_positive_rate": 0.13453283858998144 - }, - "0.7000": { - "threshold": 0.7, - "tp": 1035, - "fp": 542540, - "fn": 847, - "precision": 0.001904061077128271, - "recall": 0.5499468650371945, - "far": 0.9980959389228717, - "csi": 0.001901098779990522, - "f1": 0.0037949829225768484, - "f2": 0.009390259171153124, - "frequency_bias": 288.8283740701381, - "predicted_positive_rate": 0.06723252937538651 - }, - "0.9000": { - "threshold": 0.9, - "tp": 668, - "fp": 215450, - "fn": 1214, - "precision": 0.003090904043161606, - "recall": 0.35494155154091395, - "far": 0.9969090959568384, - "csi": 0.0030736384885796846, - "f1": 0.006128440366972477, - "f2": 0.014934315838423222, - "frequency_bias": 114.83421891604677, - "predicted_positive_rate": 0.02673073593073593 - } - }, - "val_topk_area_metrics": { - "0.0100": { - "fraction": 0.01, - "cells": 80850, - "precision": 0.003809523768723011, - "recall": 0.16365568339824677, - "lift": 16.365568629137645 - }, - "0.0500": { - "fraction": 0.05, - "cells": 404250, - "precision": 0.0022337662521749735, - "recall": 0.4798087179660797, - "lift": 9.596174514400083 - }, - "0.1000": { - "fraction": 0.1, - "cells": 808500, - "precision": 0.001512677758000791, - "recall": 0.6498405933380127, - "lift": 6.4984058809614424 - } - }, - "val_fss": { - "0.1000": { - "1": 0.000766430190058176, - "2": 0.0007919140518234391, - "4": 0.0008313854990248393, - "8": 0.0008733206367607105 - }, - "0.2000": { - "1": 0.001183482713540407, - "2": 0.0012321062159653673, - "4": 0.0013054377666424921, - "8": 0.001372829646716478 - }, - "0.3000": { - "1": 0.0016257922850572148, - "2": 0.0016941779647912103, - "4": 0.0018051660509925682, - "8": 0.0019058108793008621 - }, - "0.5000": { - "1": 0.002719293315745386, - "2": 0.002881546868972018, - "4": 0.003177453926384799, - "8": 0.003420751645848541 - }, - "0.7000": { - "1": 0.004398705160378991, - "2": 0.0048159192047847466, - "4": 0.0055998536030907875, - "8": 0.006373926334045654 - }, - "0.9000": { - "1": 0.007842918508978447, - "2": 0.009046435090570681, - "4": 0.01234545434024481, - "8": 0.01698910587516045 - } - }, - "minutes": 0.3439892331759135 - }, - { - "epoch": 4, - "train_loss": 0.49738037993818707, - "train_main_loss": 0.46940162589285345, - "train_aux_spatial_loss": 0.05595750918425256, - "val_loss": 0.17950345166027545, - "val_pr_auc": 0.001609884375517846, - "val_auroc": 0.8526320117801028, - "val_brier": 0.042433541268110275, - "val_brier_skill_score": -181.33366800650572, - "val_ece": 0.10635442963314233, - "val_positive_rate": 0.00023277674335986376, - "val_precision": 0.00177325478436419, - "val_positive_recall": 0.3990435706695005, - "val_far": 0.9982267452156358, - "val_csi": 0.0017685319065763013, - "val_f1": 0.003530819446305451, - "val_f2": 0.008711427862185442, - "val_frequency_bias": 225.03453772582358, - "val_threshold_metrics": { - "0.1000": { - "threshold": 0.1, - "tp": 1553, - "fp": 2208817, - "fn": 329, - "precision": 0.0007025973027140253, - "recall": 0.8251859723698194, - "far": 0.999297402697286, - "csi": 0.0007024927409837341, - "f1": 0.001403999182733251, - "f2": 0.0035010627179428455, - "frequency_bias": 1174.4792773645058, - "predicted_positive_rate": 0.27339146567718 - }, - "0.2000": { - "threshold": 0.2, - "tp": 1192, - "fp": 1156206, - "fn": 690, - "precision": 0.0010298963709977035, - "recall": 0.6333687566418703, - "far": 0.9989701036290023, - "csi": 0.0010292827488066537, - "f1": 0.002056448830308467, - "f2": 0.005116204806142193, - "frequency_bias": 614.9829968119022, - "predicted_positive_rate": 0.14315374149659865 - }, - "0.3000": { - "threshold": 0.3, - "tp": 969, - "fp": 763555, - "fn": 913, - "precision": 0.0012674553055234369, - "recall": 0.5148777895855473, - "far": 0.9987325446944766, - "csi": 0.0012659435067810937, - "f1": 0.0025286858401421704, - "f2": 0.006275484034754136, - "frequency_bias": 406.2295430393199, - "predicted_positive_rate": 0.09456079158936302 - }, - "0.5000": { - "threshold": 0.5, - "tp": 751, - "fp": 422764, - "fn": 1131, - "precision": 0.00177325478436419, - "recall": 0.3990435706695005, - "far": 0.9982267452156358, - "csi": 0.0017685319065763013, - "f1": 0.003530819446305451, - "f2": 0.008711427862185442, - "frequency_bias": 225.03453772582358, - "predicted_positive_rate": 0.052382807668521957 - }, - "0.7000": { - "threshold": 0.7, - "tp": 524, - "fp": 220213, - "fn": 1358, - "precision": 0.002373865731617264, - "recall": 0.2784272051009564, - "far": 0.9976261342683828, - "csi": 0.0023593507282919472, - "f1": 0.004707594589859806, - "f2": 0.011477887542987317, - "frequency_bias": 117.288522848034, - "predicted_positive_rate": 0.02730204081632653 - }, - "0.9000": { - "threshold": 0.9, - "tp": 192, - "fp": 86741, - "fn": 1690, - "precision": 0.0022085974256036258, - "recall": 0.10201912858660998, - "far": 0.9977914025743964, - "csi": 0.0021664804847500085, - "f1": 0.004323593987502111, - "f2": 0.010162924381490773, - "frequency_bias": 46.19181721572795, - "predicted_positive_rate": 0.010752380952380952 - } - }, - "val_topk_area_metrics": { - "0.0100": { - "fraction": 0.01, - "cells": 80850, - "precision": 0.0022387136705219746, - "recall": 0.09617428481578827, - "lift": 9.61742843468263 - }, - "0.0500": { - "fraction": 0.05, - "cells": 404250, - "precision": 0.0018157081212848425, - "recall": 0.39001062512397766, - "lift": 7.800212749251452 - }, - "0.1000": { - "fraction": 0.1, - "cells": 808500, - "precision": 0.001217068638652563, - "recall": 0.5228480100631714, - "lift": 5.228480393211028 - } - }, - "val_fss": { - "0.1000": { - "1": 0.0015197337144259304, - "2": 0.0015784324521881388, - "4": 0.001671154714090517, - "8": 0.0017417425576053969 - }, - "0.2000": { - "1": 0.0023132093983118107, - "2": 0.0024571591639005463, - "4": 0.0026935164349046747, - "8": 0.0028878255110219575 - }, - "0.3000": { - "1": 0.0029228902666941226, - "2": 0.0031646210950836418, - "4": 0.0035768996077181203, - "8": 0.003942389423331805 - }, - "0.5000": { - "1": 0.004163957521057138, - "2": 0.004514281813615906, - "4": 0.005317944336750369, - "8": 0.006180973638990284 - }, - "0.7000": { - "1": 0.005914502692297763, - "2": 0.006898543924811484, - "4": 0.009541669667679553, - "8": 0.012957827370078667 - }, - "0.9000": { - "1": 0.006528501792766983, - "2": 0.00881690475047614, - "4": 0.015511980715039009, - "8": 0.0273878632698773 - } - }, - "minutes": 0.31681207021077473 - }, - { - "epoch": 5, - "train_loss": 0.4843243009870309, - "train_main_loss": 0.45709068199154484, - "train_aux_spatial_loss": 0.054467237078325896, - "val_loss": 0.24718355586131413, - "val_pr_auc": 0.0019814322834402955, - "val_auroc": 0.8503887921685119, - "val_brier": 0.061934370547533035, - "val_brier_skill_score": -265.1272337902309, - "val_ece": 0.14998965867960198, - "val_positive_rate": 0.00023277674335986376, - "val_precision": 0.0014891822055106815, - "val_positive_recall": 0.44739638682252925, - "val_far": 0.9985108177944894, - "val_csi": 0.0014864480775918835, - "val_f1": 0.00296848365835644, - "val_f2": 0.007348077194954437, - "val_frequency_bias": 300.43092454835283, - "val_threshold_metrics": { - "0.1000": { - "threshold": 0.1, - "tp": 1703, - "fp": 3546784, - "fn": 179, - "precision": 0.0004799228516266228, - "recall": 0.9048884165781084, - "far": 0.9995200771483734, - "csi": 0.00047989864360297645, - "f1": 0.0009593369027275756, - "f2": 0.0023945343312668814, - "frequency_bias": 1885.4872476089267, - "predicted_positive_rate": 0.43889758812615953 - }, - "0.2000": { - "threshold": 0.2, - "tp": 1455, - "fp": 1959072, - "fn": 427, - "precision": 0.0007421473920022525, - "recall": 0.7731137088204039, - "far": 0.9992578526079977, - "csi": 0.0007419857885498589, - "f1": 0.0014828713076631833, - "f2": 0.003696543033604244, - "frequency_bias": 1041.7252922422954, - "predicted_positive_rate": 0.24248942486085343 - }, - "0.3000": { - "threshold": 0.3, - "tp": 1192, - "fp": 1178364, - "fn": 690, - "precision": 0.001010549732272143, - "recall": 0.6333687566418703, - "far": 0.9989894502677279, - "csi": 0.0010099589407631968, - "f1": 0.0020178799056742715, - "f2": 0.005020706201077599, - "frequency_bias": 626.7566418703507, - "predicted_positive_rate": 0.1458943722943723 - }, - "0.5000": { - "threshold": 0.5, - "tp": 842, - "fp": 564569, - "fn": 1040, - "precision": 0.0014891822055106815, - "recall": 0.44739638682252925, - "far": 0.9985108177944894, - "csi": 0.0014864480775918835, - "f1": 0.00296848365835644, - "f2": 0.007348077194954437, - "frequency_bias": 300.43092454835283, - "predicted_positive_rate": 0.06993333333333333 - }, - "0.7000": { - "threshold": 0.7, - "tp": 668, - "fp": 291990, - "fn": 1214, - "precision": 0.0022825277286115534, - "recall": 0.35494155154091395, - "far": 0.9977174722713884, - "csi": 0.0022730984918604017, - "f1": 0.00453588646703334, - "f2": 0.011126434943668257, - "frequency_bias": 155.50371944739638, - "predicted_positive_rate": 0.03619764996907854 - }, - "0.9000": { - "threshold": 0.9, - "tp": 508, - "fp": 128530, - "fn": 1374, - "precision": 0.003936824811295897, - "recall": 0.26992561105207225, - "far": 0.9960631751887041, - "csi": 0.00389534705395209, - "f1": 0.007760464405743966, - "f2": 0.01859906565323726, - "frequency_bias": 68.56429330499469, - "predicted_positive_rate": 0.01596017316017316 - } - }, - "val_topk_area_metrics": { - "0.0100": { - "fraction": 0.01, - "cells": 80850, - "precision": 0.004155843984335661, - "recall": 0.17853347957134247, - "lift": 17.85334704984204 - }, - "0.0500": { - "fraction": 0.05, - "cells": 404250, - "precision": 0.0018206555396318436, - "recall": 0.3910733163356781, - "lift": 7.821466669533997 - }, - "0.1000": { - "fraction": 0.1, - "cells": 808500, - "precision": 0.0012257266789674759, - "recall": 0.5265674591064453, - "lift": 5.265675003763371 - } - }, - "val_fss": { - "0.1000": { - "1": 0.0010178924030178615, - "2": 0.0010515169589780937, - "4": 0.0011000285717616132, - "8": 0.001146573354002478 - }, - "0.2000": { - "1": 0.0016254090694596268, - "2": 0.001703218739850576, - "4": 0.0018200106661242499, - "8": 0.0019167352524950276 - }, - "0.3000": { - "1": 0.0022847465603914596, - "2": 0.0024507638687161215, - "4": 0.0027246032507362017, - "8": 0.002940764611886193 - }, - "0.5000": { - "1": 0.0035209305166398197, - "2": 0.003927423032519695, - "4": 0.004626604558975789, - "8": 0.005203306081351444 - }, - "0.7000": { - "1": 0.005774099909606312, - "2": 0.006526096761710121, - "4": 0.008174907733284553, - "8": 0.010327235764531983 - }, - "0.9000": { - "1": 0.009730396821955645, - "2": 0.011013779690799308, - "4": 0.015337678551787581, - "8": 0.022811547211147087 - } - }, - "minutes": 0.32158944606781004 - }, - { - "epoch": 6, - "train_loss": 0.4694202790356114, - "train_main_loss": 0.44288265654665, - "train_aux_spatial_loss": 0.05307524496860166, - "val_loss": 0.31109391575058304, - "val_pr_auc": 0.002276307511486534, - "val_auroc": 0.8565989334771861, - "val_brier": 0.0804823711514473, - "val_brier_skill_score": -344.82656793089006, - "val_ece": 0.1884108935656475, - "val_positive_rate": 0.00023277674335986376, - "val_precision": 0.0012985133535061177, - "val_positive_recall": 0.524442082890542, - "val_far": 0.9987014866464938, - "val_csi": 0.0012969861825636174, - "val_f1": 0.002590612376670315, - "val_f2": 0.0064288952461348465, - "val_frequency_bias": 403.87885228480343, - "val_threshold_metrics": { - "0.1000": { - "threshold": 0.1, - "tp": 1792, - "fp": 4481944, - "fn": 90, - "precision": 0.0003996667065143889, - "recall": 0.9521785334750266, - "far": 0.9996003332934856, - "csi": 0.0003996586843468056, - "f1": 0.0007989980421872749, - "f2": 0.001994984040127679, - "frequency_bias": 2382.431455897981, - "predicted_positive_rate": 0.5545746444032158 - }, - "0.2000": { - "threshold": 0.2, - "tp": 1565, - "fp": 2389441, - "fn": 317, - "precision": 0.0006545362077719588, - "recall": 0.8315621679064825, - "far": 0.999345463792228, - "csi": 0.0006544494407489076, - "f1": 0.001308042833596892, - "f2": 0.0032624094551088295, - "frequency_bias": 1270.460148777896, - "predicted_positive_rate": 0.29573358070500927 - }, - "0.3000": { - "threshold": 0.3, - "tp": 1398, - "fp": 1490260, - "fn": 484, - "precision": 0.0009372121491655594, - "recall": 0.742826780021254, - "far": 0.9990627878508345, - "csi": 0.0009369081494924746, - "f1": 0.001872062348514268, - "f2": 0.004662530199721716, - "frequency_bias": 792.5919234856535, - "predicted_positive_rate": 0.1844969696969697 - }, - "0.5000": { - "threshold": 0.5, - "tp": 987, - "fp": 759113, - "fn": 895, - "precision": 0.0012985133535061177, - "recall": 0.524442082890542, - "far": 0.9987014866464938, - "csi": 0.0012969861825636174, - "f1": 0.002590612376670315, - "f2": 0.0064288952461348465, - "frequency_bias": 403.87885228480343, - "predicted_positive_rate": 0.09401360544217686 - }, - "0.7000": { - "threshold": 0.7, - "tp": 817, - "fp": 424418, - "fn": 1065, - "precision": 0.0019212905805025457, - "recall": 0.4341126461211477, - "far": 0.9980787094194975, - "csi": 0.0019164907342247243, - "f1": 0.0038256496463498295, - "f2": 0.009439346709399835, - "frequency_bias": 225.94845908607863, - "predicted_positive_rate": 0.05259554730983303 - }, - "0.9000": { - "threshold": 0.9, - "tp": 579, - "fp": 166503, - "fn": 1303, - "precision": 0.0034653643121341617, - "recall": 0.30765143464399575, - "far": 0.9965346356878658, - "csi": 0.003438548564302046, - "f1": 0.0068535309296654905, - "f2": 0.01657980642574881, - "frequency_bias": 88.77895855472902, - "predicted_positive_rate": 0.020665677179962896 - } - }, - "val_topk_area_metrics": { - "0.0100": { - "fraction": 0.01, - "cells": 80850, - "precision": 0.004650587681680918, - "recall": 0.1997874528169632, - "lift": 19.97874707994901 - }, - "0.0500": { - "fraction": 0.05, - "cells": 404250, - "precision": 0.001996289473026991, - "recall": 0.4287991523742676, - "lift": 8.575983340143242 - }, - "0.1000": { - "fraction": 0.1, - "cells": 808500, - "precision": 0.001272727269679308, - "recall": 0.546758770942688, - "lift": 5.467587746563329 - } - }, - "val_fss": { - "0.1000": { - "1": 0.0008456018359040973, - "2": 0.0008747226916036022, - "4": 0.0009169214186822705, - "8": 0.0009552239909184079 - }, - "0.2000": { - "1": 0.0014371762062292959, - "2": 0.0015125023076059119, - "4": 0.0016182172539731532, - "8": 0.001698721433581074 - }, - "0.3000": { - "1": 0.002066243274234769, - "2": 0.0021820389578350863, - "4": 0.0023651199981745696, - "8": 0.00250168028465092 - }, - "0.5000": { - "1": 0.0030274957578868555, - "2": 0.0033090501311099363, - "4": 0.003784045326573948, - "8": 0.004098759927500861 - }, - "0.7000": { - "1": 0.0045338085209784, - "2": 0.004961997974645471, - "4": 0.005855624169939677, - "8": 0.006579670951755601 - }, - "0.9000": { - "1": 0.008876876067272832, - "2": 0.010220203844888975, - "4": 0.013385368093454919, - "8": 0.017526068461561795 - } - }, - "minutes": 0.35659926732381186 - }, - { - "epoch": 7, - "train_loss": 0.4594879526371206, - "train_main_loss": 0.43360181109779, - "train_aux_spatial_loss": 0.0517722830069709, - "val_loss": 0.24388390369713306, - "val_pr_auc": 0.00280721100408273, - "val_auroc": 0.8806628065269809, - "val_brier": 0.061469998210668564, - "val_brier_skill_score": -263.1318615217164, - "val_ece": 0.14419293259630367, - "val_positive_rate": 0.00023277674335986376, - "val_precision": 0.0018425305628724307, - "val_positive_recall": 0.5924548352816154, - "val_far": 0.9981574694371276, - "val_csi": 0.001840198180266804, - "val_f1": 0.0036736361419901554, - "val_f2": 0.009099455828058642, - "val_frequency_bias": 321.54410201912856, - "val_threshold_metrics": { - "0.1000": { - "threshold": 0.1, - "tp": 1701, - "fp": 3069167, - "fn": 181, - "precision": 0.000553915049425765, - "recall": 0.9038257173219979, - "far": 0.9994460849505742, - "csi": 0.0005538824030486, - "f1": 0.0011071515743226751, - "f2": 0.0027628024464688757, - "frequency_bias": 1631.7045696068012, - "predicted_positive_rate": 0.3798228818800247 - }, - "0.2000": { - "threshold": 0.2, - "tp": 1553, - "fp": 1735305, - "fn": 329, - "precision": 0.0008941433323852612, - "recall": 0.8251859723698194, - "far": 0.9991058566676148, - "csi": 0.0008939739935884853, - "f1": 0.0017863510358075386, - "f2": 0.004451423022198068, - "frequency_bias": 922.8788522848034, - "predicted_positive_rate": 0.21482473716759432 - }, - "0.3000": { - "threshold": 0.3, - "tp": 1386, - "fp": 1140133, - "fn": 496, - "precision": 0.0012141716432227585, - "recall": 0.7364505844845909, - "far": 0.9987858283567772, - "csi": 0.001213644304146618, - "f1": 0.002424346314197731, - "f2": 0.006031084890348262, - "frequency_bias": 606.5456960680127, - "predicted_positive_rate": 0.14118973407544835 - }, - "0.5000": { - "threshold": 0.5, - "tp": 1115, - "fp": 604031, - "fn": 767, - "precision": 0.0018425305628724307, - "recall": 0.5924548352816154, - "far": 0.9981574694371276, - "csi": 0.001840198180266804, - "f1": 0.0036736361419901554, - "f2": 0.009099455828058642, - "frequency_bias": 321.54410201912856, - "predicted_positive_rate": 0.07484799010513296 - }, - "0.7000": { - "threshold": 0.7, - "tp": 843, - "fp": 315079, - "fn": 1039, - "precision": 0.0026683801697887453, - "recall": 0.4479277364505845, - "far": 0.9973316198302112, - "csi": 0.002659633204085045, - "f1": 0.005305156637424324, - "f2": 0.013031380429741845, - "frequency_bias": 167.86503719447396, - "predicted_positive_rate": 0.03907507730364873 - }, - "0.9000": { - "threshold": 0.9, - "tp": 585, - "fp": 138799, - "fn": 1297, - "precision": 0.004197038397520519, - "recall": 0.3108395324123273, - "far": 0.9958029616024795, - "csi": 0.004158344054989657, - "f1": 0.008282247674599692, - "f2": 0.019909878022217383, - "frequency_bias": 74.06163655685441, - "predicted_positive_rate": 0.01723982683982684 - } - }, - "val_topk_area_metrics": { - "0.0100": { - "fraction": 0.01, - "cells": 80850, - "precision": 0.005553494207561016, - "recall": 0.2385759800672531, - "lift": 23.857599034176413 - }, - "0.0500": { - "fraction": 0.05, - "cells": 404250, - "precision": 0.0023129251785576344, - "recall": 0.4968118965625763, - "lift": 9.936238239152363 - }, - "0.1000": { - "fraction": 0.1, - "cells": 808500, - "precision": 0.001546073006466031, - "recall": 0.6641870141029358, - "lift": 6.641870593042289 - } - }, - "val_fss": { - "0.1000": { - "1": 0.001182645589865583, - "2": 0.001235610970657941, - "4": 0.0013156511427493411, - "8": 0.001399485420612856 - }, - "0.2000": { - "1": 0.001955440214953197, - "2": 0.002070129096730211, - "4": 0.0022438521231362696, - "8": 0.002400362116154686 - }, - "0.3000": { - "1": 0.0027211631165605654, - "2": 0.002910697745954649, - "4": 0.0032097807960153046, - "8": 0.00344582755765388 - }, - "0.5000": { - "1": 0.004202872161291116, - "2": 0.004619998632292344, - "4": 0.0053646052916888065, - "8": 0.00604022995299458 - }, - "0.7000": { - "1": 0.0065952032055480725, - "2": 0.00754062200693062, - "4": 0.009568511759124099, - "8": 0.011927180679107141 - }, - "0.9000": { - "1": 0.011117215021448135, - "2": 0.013052893418606493, - "4": 0.018349644962076805, - "8": 0.025929086293738135 - } - }, - "minutes": 0.34816946188608805 - }, - { - "epoch": 8, - "train_loss": 0.44826403022158967, - "train_main_loss": 0.42296392545852457, - "train_aux_spatial_loss": 0.050600208696345975, - "val_loss": 0.3248936382432779, - "val_pr_auc": 0.0044442059631609795, - "val_auroc": 0.8849275341349525, - "val_brier": 0.08548334985971451, - "val_brier_skill_score": -366.3153893738006, - "val_ece": 0.18183675751987904, - "val_positive_rate": 0.00023277674335986376, - "val_precision": 0.0015666706745220632, - "val_positive_recall": 0.6923485653560042, - "val_far": 0.9984333293254779, - "val_csi": 0.0015655807727937387, - "val_f1": 0.0031262671218908276, - "val_f2": 0.0077630870276015574, - "val_frequency_bias": 441.92348565356, - "val_threshold_metrics": { - "0.1000": { - "threshold": 0.1, - "tp": 1772, - "fp": 3855746, - "fn": 110, - "precision": 0.0004593627301285438, - "recall": 0.9415515409139213, - "far": 0.9995406372698714, - "csi": 0.0004593496314315429, - "f1": 0.0009182774524537493, - "f2": 0.0022923401170387107, - "frequency_bias": 2049.6907545164718, - "predicted_positive_rate": 0.47712034632034633 - }, - "0.2000": { - "threshold": 0.2, - "tp": 1618, - "fp": 2317858, - "fn": 264, - "precision": 0.0006975713480113612, - "recall": 0.8597236981934112, - "far": 0.9993024286519886, - "csi": 0.0006974919603058964, - "f1": 0.0013940116087221358, - "f2": 0.0034765733105744556, - "frequency_bias": 1232.4527098831031, - "predicted_positive_rate": 0.28688633271490416 - }, - "0.3000": { - "threshold": 0.3, - "tp": 1481, - "fp": 1547894, - "fn": 401, - "precision": 0.0009558693021379589, - "recall": 0.7869287991498406, - "far": 0.999044130697862, - "csi": 0.0009556219737562073, - "f1": 0.0019094192645061393, - "f2": 0.0047562372222290025, - "frequency_bias": 823.259829968119, - "predicted_positive_rate": 0.19163574520717377 - }, - "0.5000": { - "threshold": 0.5, - "tp": 1303, - "fp": 830397, - "fn": 579, - "precision": 0.0015666706745220632, - "recall": 0.6923485653560042, - "far": 0.9984333293254779, - "csi": 0.0015655807727937387, - "f1": 0.0031262671218908276, - "f2": 0.0077630870276015574, - "frequency_bias": 441.92348565356, - "predicted_positive_rate": 0.10286951144094002 - }, - "0.7000": { - "threshold": 0.7, - "tp": 1100, - "fp": 493742, - "fn": 782, - "precision": 0.002222931764078231, - "recall": 0.5844845908607864, - "far": 0.9977770682359217, - "csi": 0.002219424402369538, - "f1": 0.004429018932042743, - "f2": 0.010948105977665863, - "frequency_bias": 262.93411264612115, - "predicted_positive_rate": 0.06120494743351886 - }, - "0.9000": { - "threshold": 0.9, - "tp": 839, - "fp": 211101, - "fn": 1043, - "precision": 0.0039586675474190805, - "recall": 0.44580233793836344, - "far": 0.996041332452581, - "csi": 0.003939281538902166, - "f1": 0.007847648979057346, - "f2": 0.019114403922211892, - "frequency_bias": 112.61424017003188, - "predicted_positive_rate": 0.026213976499690785 - } - }, - "val_topk_area_metrics": { - "0.0100": { - "fraction": 0.01, - "cells": 80850, - "precision": 0.007482993416488171, - "recall": 0.3214665353298187, - "lift": 32.14665395038951 - }, - "0.0500": { - "fraction": 0.05, - "cells": 404250, - "precision": 0.002540507121011615, - "recall": 0.5456960797309875, - "lift": 10.913921572844114 - }, - "0.1000": { - "fraction": 0.1, - "cells": 808500, - "precision": 0.001606679055839777, - "recall": 0.6902231574058533, - "lift": 6.902231866677136 - } - }, - "val_fss": { - "0.1000": { - "1": 0.0009645709705659034, - "2": 0.0009937018941826858, - "4": 0.001037849954013348, - "8": 0.0010919201251842203 - }, - "0.2000": { - "1": 0.0015015023453176513, - "2": 0.0015689402217066162, - "4": 0.0016632836621220948, - "8": 0.001740115974165457 - }, - "0.3000": { - "1": 0.0020927497428742114, - "2": 0.00219703009118255, - "4": 0.0023412446037914814, - "8": 0.0024308848199744437 - }, - "0.5000": { - "1": 0.003430910518313901, - "2": 0.0035887345058116527, - "4": 0.003812179025492024, - "8": 0.0039358805986901 - }, - "0.7000": { - "1": 0.004948203201738943, - "2": 0.00521956377670052, - "4": 0.005637850205958861, - "8": 0.0059804616256462495 - }, - "0.9000": { - "1": 0.00948030325480842, - "2": 0.01033684622192943, - "4": 0.01188914050876233, - "8": 0.014040969292779426 - } - }, - "minutes": 0.31795929272969564 - }, - { - "epoch": 9, - "train_loss": 0.4411211615542718, - "train_main_loss": 0.4163666974302027, - "train_aux_spatial_loss": 0.04950892778797695, - "val_loss": 0.2851397772630056, - "val_pr_auc": 0.0032182950594722933, - "val_auroc": 0.884300165942819, - "val_brier": 0.07394615560770035, - "val_brier_skill_score": -316.7409517094562, - "val_ece": 0.16132334364664333, - "val_positive_rate": 0.00023277674335986376, - "val_precision": 0.0016357551889862657, - "val_positive_recall": 0.6609989373007439, - "val_far": 0.9983642448110137, - "val_csi": 0.0016343840776306161, - "val_f1": 0.0032634344499578296, - "val_f2": 0.008098610346170022, - "val_frequency_bias": 404.0940488841658, - "val_threshold_metrics": { - "0.1000": { - "threshold": 0.1, - "tp": 1733, - "fp": 3292718, - "fn": 149, - "precision": 0.0005260360527444481, - "recall": 0.9208289054197663, - "far": 0.9994739639472555, - "csi": 0.0005260122624901354, - "f1": 0.0010514714381101666, - "f2": 0.002624183860648417, - "frequency_bias": 1750.5053134962805, - "predicted_positive_rate": 0.4074769325912183 - }, - "0.2000": { - "threshold": 0.2, - "tp": 1584, - "fp": 1951842, - "fn": 298, - "precision": 0.0008108830331939884, - "recall": 0.8416578108395324, - "far": 0.999189116966806, - "csi": 0.0008107593498365174, - "f1": 0.0016202051032369327, - "f2": 0.004038850477879644, - "frequency_bias": 1037.952178533475, - "predicted_positive_rate": 0.24161113172541743 - }, - "0.3000": { - "threshold": 0.3, - "tp": 1453, - "fp": 1379073, - "fn": 429, - "precision": 0.0010524973814328741, - "recall": 0.7720510095642933, - "far": 0.9989475026185671, - "csi": 0.0010521704182974826, - "f1": 0.0021021290386051008, - "f2": 0.005233946229757632, - "frequency_bias": 733.5419766206163, - "predicted_positive_rate": 0.17075151515151515 - }, - "0.5000": { - "threshold": 0.5, - "tp": 1244, - "fp": 759261, - "fn": 638, - "precision": 0.0016357551889862657, - "recall": 0.6609989373007439, - "far": 0.9983642448110137, - "csi": 0.0016343840776306161, - "f1": 0.0032634344499578296, - "f2": 0.008098610346170022, - "frequency_bias": 404.0940488841658, - "predicted_positive_rate": 0.09406369820655534 - }, - "0.7000": { - "threshold": 0.7, - "tp": 996, - "fp": 385640, - "fn": 886, - "precision": 0.0025760663776782295, - "recall": 0.5292242295430393, - "far": 0.9974239336223217, - "csi": 0.0025701766609379597, - "f1": 0.005127175574876839, - "f2": 0.012634334946874904, - "frequency_bias": 205.43889479277365, - "predicted_positive_rate": 0.04782139764996908 - }, - "0.9000": { - "threshold": 0.9, - "tp": 768, - "fp": 182254, - "fn": 1114, - "precision": 0.0041962168482477516, - "recall": 0.40807651434643993, - "far": 0.9958037831517522, - "csi": 0.004170830255897815, - "f1": 0.008307013369099642, - "f2": 0.020152191025977435, - "frequency_bias": 97.24867162592986, - "predicted_positive_rate": 0.02263722943722944 - } - }, - "val_topk_area_metrics": { - "0.0100": { - "fraction": 0.01, - "cells": 80850, - "precision": 0.005862708669155836, - "recall": 0.25185972452163696, - "lift": 25.18597255264594 - }, - "0.0500": { - "fraction": 0.05, - "cells": 404250, - "precision": 0.002503401366993785, - "recall": 0.5377258062362671, - "lift": 10.754516670609245 - }, - "0.1000": { - "fraction": 0.1, - "cells": 808500, - "precision": 0.0015658626798540354, - "recall": 0.6726886034011841, - "lift": 6.726886274172471 - } - }, - "val_fss": { - "0.1000": { - "1": 0.0011201829491246373, - "2": 0.0011650008395039935, - "4": 0.0012291472998500197, - "8": 0.001288563228972528 - }, - "0.2000": { - "1": 0.0017641369780564409, - "2": 0.0018498610763283407, - "4": 0.0019697883619944623, - "8": 0.0020608795760591603 - }, - "0.3000": { - "1": 0.0023264923565211593, - "2": 0.002448559375719639, - "4": 0.0026245463430637495, - "8": 0.002768340451214768 - }, - "0.5000": { - "1": 0.0037134975103053636, - "2": 0.003964199893520592, - "4": 0.004366414205446922, - "8": 0.004762426177904011 - }, - "0.7000": { - "1": 0.006053201961429555, - "2": 0.006659116708271573, - "4": 0.007824432033392337, - "8": 0.0091423801874152 - }, - "0.9000": { - "1": 0.01022660873826664, - "2": 0.011336714828667716, - "4": 0.01395971459290235, - "8": 0.017751013937979288 - } - }, - "minutes": 0.3203668077786764 - }, - { - "epoch": 10, - "train_loss": 0.4325153522077466, - "train_main_loss": 0.40816109493837044, - "train_aux_spatial_loss": 0.048708514710370465, - "val_loss": 0.19746975464125474, - "val_pr_auc": 0.0028221717878134725, - "val_auroc": 0.8620318173394532, - "val_brier": 0.044820670038461685, - "val_brier_skill_score": -191.59097700534562, - "val_ece": 0.13526803746421806, - "val_positive_rate": 0.00023277674335986376, - "val_precision": 0.002322708563657786, - "val_positive_recall": 0.47555791710945805, - "val_far": 0.9976772914363422, - "val_csi": 0.0023167742219392046, - "val_f1": 0.004622838371108036, - "val_f2": 0.011391000218910844, - "val_frequency_bias": 204.74282678002126, - "val_threshold_metrics": { - "0.1000": { - "threshold": 0.1, - "tp": 1694, - "fp": 3439838, - "fn": 188, - "precision": 0.000492222649680433, - "recall": 0.900106269925611, - "far": 0.9995077773503196, - "csi": 0.0004921957625838244, - "f1": 0.0009839072501883306, - "f2": 0.0024557415643682626, - "frequency_bias": 1828.6567481402762, - "predicted_positive_rate": 0.4256687693259122 - }, - "0.2000": { - "threshold": 0.2, - "tp": 1437, - "fp": 1559537, - "fn": 445, - "precision": 0.0009205790743471704, - "recall": 0.7635494155154091, - "far": 0.9990794209256528, - "csi": 0.0009203167119139706, - "f1": 0.0018389410156789876, - "f2": 0.004580803849787887, - "frequency_bias": 829.422954303932, - "predicted_positive_rate": 0.1930703772418058 - }, - "0.3000": { - "threshold": 0.3, - "tp": 1178, - "fp": 891195, - "fn": 704, - "precision": 0.0013200757979006537, - "recall": 0.6259298618490967, - "far": 0.9986799242020994, - "csi": 0.0013190352007721618, - "f1": 0.0026345952776333378, - "f2": 0.006545164412529823, - "frequency_bias": 474.16206163655687, - "predicted_positive_rate": 0.110373902288188 - }, - "0.5000": { - "threshold": 0.5, - "tp": 895, - "fp": 384431, - "fn": 987, - "precision": 0.002322708563657786, - "recall": 0.47555791710945805, - "far": 0.9976772914363422, - "csi": 0.0023167742219392046, - "f1": 0.004622838371108036, - "f2": 0.011391000218910844, - "frequency_bias": 204.74282678002126, - "predicted_positive_rate": 0.04765936920222635 - }, - "0.7000": { - "threshold": 0.7, - "tp": 659, - "fp": 170866, - "fn": 1223, - "precision": 0.0038420055385512317, - "recall": 0.35015940488841657, - "far": 0.9961579944614488, - "csi": 0.003814805381248987, - "f1": 0.007600615892092015, - "f2": 0.01840237248189084, - "frequency_bias": 91.13974495217853, - "predicted_positive_rate": 0.0212152133580705 - }, - "0.9000": { - "threshold": 0.9, - "tp": 327, - "fp": 52950, - "fn": 1555, - "precision": 0.006137732980460612, - "recall": 0.17375132837407015, - "far": 0.9938622670195394, - "csi": 0.0059636708491391886, - "f1": 0.011856632643811527, - "f2": 0.026889236082559, - "frequency_bias": 28.308714133900107, - "predicted_positive_rate": 0.00658961038961039 - } - }, - "val_topk_area_metrics": { - "0.0100": { - "fraction": 0.01, - "cells": 80850, - "precision": 0.005701917223632336, - "recall": 0.24495217204093933, - "lift": 24.49521864311588 - }, - "0.0500": { - "fraction": 0.05, - "cells": 404250, - "precision": 0.002246134914457798, - "recall": 0.4824654757976532, - "lift": 9.649309815222225 - }, - "0.1000": { - "fraction": 0.1, - "cells": 808500, - "precision": 0.001405071117915213, - "recall": 0.6036131978034973, - "lift": 6.036131864526638 - } - }, - "val_fss": { - "0.1000": { - "1": 0.0010545868717067863, - "2": 0.0011022352726061113, - "4": 0.001168632193011776, - "8": 0.0012236074960421117 - }, - "0.2000": { - "1": 0.0020329538883044895, - "2": 0.0021331716637861264, - "4": 0.0023056844278652733, - "8": 0.0024083609396829964 - }, - "0.3000": { - "1": 0.0030090804133009463, - "2": 0.00323794828427848, - "4": 0.0036126412524609686, - "8": 0.003827556736519133 - }, - "0.5000": { - "1": 0.005642367607543997, - "2": 0.006246436645517495, - "4": 0.007234446182773802, - "8": 0.007960937835548854 - }, - "0.7000": { - "1": 0.010066936508528523, - "2": 0.011589698733172615, - "4": 0.014878004135214007, - "8": 0.018806429552858606 - }, - "0.9000": { - "1": 0.016158702083284626, - "2": 0.01968974409486912, - "4": 0.028978356172675412, - "8": 0.04860085752482901 - } - }, - "minutes": 0.34885872999827067 - } -] diff --git a/models/wildfire_fm/metrics/reference_fireprone_seed_1_20260504/run_summary.json b/models/wildfire_fm/metrics/reference_fireprone_seed_1_20260504/run_summary.json deleted file mode 100644 index 8c6a4867769ab5e7859e0895d105bf948ada1327..0000000000000000000000000000000000000000 --- a/models/wildfire_fm/metrics/reference_fireprone_seed_1_20260504/run_summary.json +++ /dev/null @@ -1,172 +0,0 @@ -{ - "run_name": "reference_fireprone_seed_1_20260504", - "device": "cuda", - "index_root": "", - "input_channels": 16, - "tile_size": 32, - "num_train_tiles": 27177, - "positive_train_tiles": 6724, - "negative_train_tiles": 20453, - "train_positive_rate": 0.0018382099293520256, - "metric_reference_positive_rate": 0.0002787771234976204, - "pos_weight": 50.0, - "use_aux_spatial_head": true, - "aux_spatial_radius": 2, - "aux_spatial_loss_weight": 0.5, - "aux_positive_rate": 0.004560296846011132, - "init_checkpoint": "", - "best_val_pr_auc": 0.007534030208235136, - "best_val_auroc": 0.8668780863986516, - "best_test_pr_auc": 0.0013275812482877569, - "best_test_auroc": 0.8211195324224633, - "best_test_brier": 0.14432816207408905, - "best_test_ece": 0.2856647948942866, - "best_test_topk_area_metrics": { - "0.0100": { - "fraction": 0.01, - "cells": 80850, - "precision": 0.001855287584476173, - "recall": 0.0764915868639946, - "lift": 7.649158522554341 - }, - "0.0500": { - "fraction": 0.05, - "cells": 404250, - "precision": 0.001625231932848692, - "recall": 0.3350331485271454, - "lift": 6.7006629022351945 - }, - "0.1000": { - "fraction": 0.1, - "cells": 808500, - "precision": 0.0012022263836115599, - "recall": 0.4956654906272888, - "lift": 4.956654841647355 - } - }, - "best_test_threshold_metrics": { - "0.1000": { - "threshold": 0.1, - "tp": 1902, - "fp": 5715039, - "fn": 59, - "precision": 0.00033269540476279185, - "recall": 0.969913309535951, - "far": 0.9996673045952372, - "csi": 0.000332691971313626, - "f1": 0.0006651626483545268, - "f2": 0.0016611977567716515, - "frequency_bias": 2915.3192248852624, - "predicted_positive_rate": 0.7071046382189239 - }, - "0.2000": { - "threshold": 0.2, - "tp": 1820, - "fp": 4195336, - "fn": 141, - "precision": 0.0004336269607324579, - "recall": 0.9280979092299847, - "far": 0.9995663730392675, - "csi": 0.0004336123938811097, - "f1": 0.000866848911330644, - "f2": 0.0021640903686087993, - "frequency_bias": 2140.3141254462007, - "predicted_positive_rate": 0.5191287569573284 - }, - "0.3000": { - "threshold": 0.3, - "tp": 1681, - "fp": 3187599, - "fn": 280, - "precision": 0.0005270782120102343, - "recall": 0.85721570627231, - "far": 0.9994729217879897, - "csi": 0.0005270319417098284, - "f1": 0.001053508650709865, - "f2": 0.002628925246565351, - "frequency_bias": 1626.353901070882, - "predicted_positive_rate": 0.3944687693259122 - }, - "0.5000": { - "threshold": 0.5, - "tp": 1319, - "fp": 1642412, - "fn": 642, - "precision": 0.0008024427354597559, - "recall": 0.6726160122386537, - "far": 0.9991975572645403, - "csi": 0.0008021294438670545, - "f1": 0.0016029730958162284, - "f2": 0.0039931580461075025, - "frequency_bias": 838.2106068332483, - "predicted_positive_rate": 0.20330624613481757 - }, - "0.7000": { - "threshold": 0.7, - "tp": 863, - "fp": 666140, - "fn": 1098, - "precision": 0.0012938472540603265, - "recall": 0.44008159102498723, - "far": 0.9987061527459397, - "csi": 0.0012917208625641931, - "f1": 0.0025801089445769876, - "f2": 0.006394041908758577, - "frequency_bias": 340.13411524732277, - "predicted_positive_rate": 0.08249882498453927 - }, - "0.9000": { - "threshold": 0.9, - "tp": 444, - "fp": 213874, - "fn": 1517, - "precision": 0.002071687865694902, - "recall": 0.22641509433962265, - "far": 0.9979283121343051, - "csi": 0.002057126971992494, - "f1": 0.004105807776067025, - "f2": 0.009992708023874471, - "frequency_bias": 109.29015808261092, - "predicted_positive_rate": 0.02650810142238714 - } - }, - "best_test_fss": { - "0.1000": { - "1": 0.0006848046210931047, - "2": 0.0006972018571715921, - "4": 0.0007162254754048414, - "8": 0.0007405531411359689 - }, - "0.2000": { - "1": 0.0009004883576219092, - "2": 0.0009194891185105858, - "4": 0.0009493580716710692, - "8": 0.0009785076278155946 - }, - "0.3000": { - "1": 0.0011134993250828673, - "2": 0.0011478568434259984, - "4": 0.0011942054782714262, - "8": 0.0012221388013780032 - }, - "0.5000": { - "1": 0.0017304583364127213, - "2": 0.0017995263941176232, - "4": 0.0018910170119570413, - "8": 0.001927562442000319 - }, - "0.7000": { - "1": 0.0029083943170400417, - "2": 0.003109232173895693, - "4": 0.003414739147105883, - "8": 0.0036022619251578902 - }, - "0.9000": { - "1": 0.005409636856494537, - "2": 0.0063686345028376445, - "4": 0.008451726086051603, - "8": 0.010875712868847476 - } - }, - "epochs": 10 -} diff --git a/models/wildfire_fm/metrics/reference_fireprone_seed_1_20260504/tile_summary.json b/models/wildfire_fm/metrics/reference_fireprone_seed_1_20260504/tile_summary.json deleted file mode 100644 index a780a7c2a1b050634c3f3137ba115235cc4fbf78..0000000000000000000000000000000000000000 --- a/models/wildfire_fm/metrics/reference_fireprone_seed_1_20260504/tile_summary.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "index_root": "", - "tile_size": 32, - "num_train_tiles": 27177, - "positive_train_tiles": 6724, - "negative_train_tiles": 20453, - "input_channels": 16, - "train_positive_rate": 0.0018382099293520256, - "metric_reference_positive_rate": 0.0002787771234976204, - "pos_weight_source": "tiles", - "positive_rate_source": "tiles", - "pos_weight": 50.0, - "loss_type": "bce", - "train_target_mode": "dilate_max", - "train_target_radius": 2 -} diff --git a/models/wildfire_fm/metrics/reference_fireprone_seed_42_20260504/history.json b/models/wildfire_fm/metrics/reference_fireprone_seed_42_20260504/history.json deleted file mode 100644 index cd7156e95d7219330e7369e0995c44f8ad41ceb0..0000000000000000000000000000000000000000 --- a/models/wildfire_fm/metrics/reference_fireprone_seed_42_20260504/history.json +++ /dev/null @@ -1,1682 +0,0 @@ -[ - { - "epoch": 1, - "train_loss": 0.5732859523984933, - "train_main_loss": 0.5301023899695203, - "train_aux_spatial_loss": 0.08636712405139627, - "val_loss": 0.24509115666151046, - "val_pr_auc": 0.0017965274230398033, - "val_auroc": 0.8433017073874776, - "val_brier": 0.06070003658533096, - "val_brier_skill_score": -259.82339554936163, - "val_ece": 0.1579977289119908, - "val_positive_rate": 0.00023277674335986376, - "val_precision": 0.001549417642141233, - "val_positive_recall": 0.4500531349628055, - "val_far": 0.9984505823578588, - "val_csi": 0.0015464896328593442, - "val_f1": 0.003088203391190052, - "val_f2": 0.007641852449994136, - "val_frequency_bias": 290.4659936238045, - "val_threshold_metrics": { - "0.1000": { - "threshold": 0.1, - "tp": 1770, - "fp": 4182500, - "fn": 112, - "precision": 0.0004230128552889751, - "recall": 0.9404888416578109, - "far": 0.999576987144711, - "csi": 0.00042300153284284276, - "f1": 0.0008456453564036852, - "f2": 0.0021112658577536417, - "frequency_bias": 2223.3103081827844, - "predicted_positive_rate": 0.5175349412492269 - }, - "0.2000": { - "threshold": 0.2, - "tp": 1456, - "fp": 2049649, - "fn": 426, - "precision": 0.0007098612699008583, - "recall": 0.7736450584484591, - "far": 0.9992901387300991, - "csi": 0.0007097138673507736, - "f1": 0.0014184210616043841, - "f2": 0.0035363272618286015, - "frequency_bias": 1089.8538788522849, - "predicted_positive_rate": 0.2536926406926407 - }, - "0.3000": { - "threshold": 0.3, - "tp": 1161, - "fp": 1142810, - "fn": 721, - "precision": 0.0010148858668620096, - "recall": 0.6168969181721573, - "far": 0.998985114133138, - "csi": 0.0010142466270402868, - "f1": 0.0020264379462287046, - "f2": 0.005041254920759809, - "frequency_bias": 607.8485653560042, - "predicted_positive_rate": 0.1414930117501546 - }, - "0.5000": { - "threshold": 0.5, - "tp": 847, - "fp": 545810, - "fn": 1035, - "precision": 0.001549417642141233, - "recall": 0.4500531349628055, - "far": 0.9984505823578588, - "csi": 0.0015464896328593442, - "f1": 0.003088203391190052, - "f2": 0.007641852449994136, - "frequency_bias": 290.4659936238045, - "predicted_positive_rate": 0.06761372912801485 - }, - "0.7000": { - "threshold": 0.7, - "tp": 596, - "fp": 258780, - "fn": 1286, - "precision": 0.0022978224662266364, - "recall": 0.31668437832093516, - "far": 0.9977021775337733, - "csi": 0.00228648594731875, - "f1": 0.004562539711702608, - "f2": 0.011165063093846479, - "frequency_bias": 137.8193411264612, - "predicted_positive_rate": 0.03208113790970934 - }, - "0.9000": { - "threshold": 0.9, - "tp": 326, - "fp": 103328, - "fn": 1556, - "precision": 0.0031450788199201187, - "recall": 0.17321997874601489, - "far": 0.9968549211800799, - "csi": 0.0030985647752114816, - "f1": 0.006177986658580957, - "f2": 0.014660646507528196, - "frequency_bias": 55.07651434643996, - "predicted_positive_rate": 0.012820531849103277 - } - }, - "val_topk_area_metrics": { - "0.0100": { - "fraction": 0.01, - "cells": 80850, - "precision": 0.0032529374584555626, - "recall": 0.13974495232105255, - "lift": 13.974495095614635 - }, - "0.0500": { - "fraction": 0.05, - "cells": 404250, - "precision": 0.0018429190386086702, - "recall": 0.39585548639297485, - "lift": 7.917109810921228 - }, - "0.1000": { - "fraction": 0.1, - "cells": 808500, - "precision": 0.0012467532651498914, - "recall": 0.5356004238128662, - "lift": 5.356004415022078 - } - }, - "val_fss": { - "0.1000": { - "1": 0.0008853317818074968, - "2": 0.0009089386381523346, - "4": 0.0009457052841735925, - "8": 0.0009852276473808486 - }, - "0.2000": { - "1": 0.0015577706818201698, - "2": 0.0016341648495771244, - "4": 0.0017485784120635506, - "8": 0.001831827612458703 - }, - "0.3000": { - "1": 0.0023003002423931385, - "2": 0.002464986024784954, - "4": 0.0027264302361427184, - "8": 0.002898080903875666 - }, - "0.5000": { - "1": 0.003590797802279555, - "2": 0.0038924124691716022, - "4": 0.004390693252639188, - "8": 0.004810087814569819 - }, - "0.7000": { - "1": 0.005772289535979636, - "2": 0.006391883882916782, - "4": 0.0077055917957817854, - "8": 0.009441454533165472 - }, - "0.9000": { - "1": 0.008049290674849052, - "2": 0.009074456823212063, - "4": 0.011774809503574457, - "8": 0.016814201894514325 - } - }, - "minutes": 0.3870505134264628 - }, - { - "epoch": 2, - "train_loss": 0.5283296908698704, - "train_main_loss": 0.49774899936544365, - "train_aux_spatial_loss": 0.06116138393438454, - "val_loss": 0.4062879979610443, - "val_pr_auc": 0.002054551518322294, - "val_auroc": 0.8653528186778066, - "val_brier": 0.1165778785943985, - "val_brier_skill_score": -499.9261913407209, - "val_ece": 0.24187411134523915, - "val_positive_rate": 0.00023277674335986376, - "val_precision": 0.0010914377868432427, - "val_positive_recall": 0.7231668437832094, - "val_far": 0.9989085622131567, - "val_csi": 0.0010909819639278557, - "val_f1": 0.002179586038798553, - "val_f2": 0.005424441633247164, - "val_frequency_bias": 662.5818278427205, - "val_threshold_metrics": { - "0.1000": { - "threshold": 0.1, - "tp": 1812, - "fp": 4827814, - "fn": 70, - "precision": 0.00037518433104343895, - "recall": 0.9628055260361318, - "far": 0.9996248156689566, - "csi": 0.0003751788932471112, - "f1": 0.0007500763736704978, - "f2": 0.0018730021826884155, - "frequency_bias": 2566.219978746015, - "predicted_positive_rate": 0.597356338899196 - }, - "0.2000": { - "threshold": 0.2, - "tp": 1715, - "fp": 3400877, - "fn": 167, - "precision": 0.0005040275178452192, - "recall": 0.9112646121147715, - "far": 0.9994959724821548, - "csi": 0.000504002781272491, - "f1": 0.001007497780861302, - "f2": 0.0025145742671812132, - "frequency_bias": 1807.9659936238045, - "predicted_positive_rate": 0.42085244279529993 - }, - "0.3000": { - "threshold": 0.3, - "tp": 1594, - "fp": 2554643, - "fn": 288, - "precision": 0.0006235728533778362, - "recall": 0.8469713071200851, - "far": 0.9993764271466221, - "csi": 0.0006235026060766079, - "f1": 0.0012462281856317082, - "f2": 0.0031087092615742864, - "frequency_bias": 1358.2555791710945, - "predicted_positive_rate": 0.31617031539888685 - }, - "0.5000": { - "threshold": 0.5, - "tp": 1361, - "fp": 1245618, - "fn": 521, - "precision": 0.0010914377868432427, - "recall": 0.7231668437832094, - "far": 0.9989085622131567, - "csi": 0.0010909819639278557, - "f1": 0.002179586038798553, - "f2": 0.005424441633247164, - "frequency_bias": 662.5818278427205, - "predicted_positive_rate": 0.15423364254792826 - }, - "0.7000": { - "threshold": 0.7, - "tp": 1016, - "fp": 533253, - "fn": 866, - "precision": 0.0019016637686259169, - "recall": 0.5398512221041445, - "far": 0.9980983362313741, - "csi": 0.001898586338026853, - "f1": 0.0037899770773532084, - "f2": 0.009376205479173935, - "frequency_bias": 283.8836344314559, - "predicted_positive_rate": 0.06608150896722326 - }, - "0.9000": { - "threshold": 0.9, - "tp": 609, - "fp": 209521, - "fn": 1273, - "precision": 0.002898205872555085, - "recall": 0.3235919234856536, - "far": 0.9971017941274449, - "csi": 0.0028807538209013116, - "f1": 0.005744957832575515, - "f2": 0.01398983726764006, - "frequency_bias": 111.65249734325187, - "predicted_positive_rate": 0.025990105132962276 - } - }, - "val_topk_area_metrics": { - "0.0100": { - "fraction": 0.01, - "cells": 80850, - "precision": 0.0032900432124733925, - "recall": 0.1413390040397644, - "lift": 14.133899997849502 - }, - "0.0500": { - "fraction": 0.05, - "cells": 404250, - "precision": 0.002221397589892149, - "recall": 0.4771519601345062, - "lift": 9.543039213577943 - }, - "0.1000": { - "fraction": 0.1, - "cells": 808500, - "precision": 0.0015114409616217017, - "recall": 0.6493092179298401, - "lift": 6.493092650948695 - } - }, - "val_fss": { - "0.1000": { - "1": 0.0007845684873319936, - "2": 0.0008058636007729492, - "4": 0.0008395871424885515, - "8": 0.0008863828630808479 - }, - "0.2000": { - "1": 0.001061890053744352, - "2": 0.0010969197383139395, - "4": 0.0011554186534993294, - "8": 0.0012230009768544603 - }, - "0.3000": { - "1": 0.0013362506016462161, - "2": 0.0013943592723939524, - "4": 0.0014886648608402053, - "8": 0.0015775059478972953 - }, - "0.5000": { - "1": 0.002468789757546941, - "2": 0.002622560260418405, - "4": 0.0028690079941975366, - "8": 0.0030831279713063875 - }, - "0.7000": { - "1": 0.0044548071137847645, - "2": 0.004899878032674221, - "4": 0.005742732261700079, - "8": 0.006554892590571404 - }, - "0.9000": { - "1": 0.0075190310894935886, - "2": 0.008848901158122424, - "4": 0.01236010905206364, - "8": 0.017285273473687246 - } - }, - "minutes": 0.3233311374982198 - }, - { - "epoch": 3, - "train_loss": 0.5136756693969899, - "train_main_loss": 0.4847016787342131, - "train_aux_spatial_loss": 0.057947980659368674, - "val_loss": 0.3151714046796163, - "val_pr_auc": 0.0024664154896628107, - "val_auroc": 0.8703901645976795, - "val_brier": 0.08613400906324387, - "val_brier_skill_score": -369.111222001865, - "val_ece": 0.1926309692158547, - "val_positive_rate": 0.00023277674335986376, - "val_precision": 0.0014452201914565972, - "val_positive_recall": 0.6567481402763018, - "val_far": 0.9985547798085435, - "val_csi": 0.001444129368754228, - "val_f1": 0.0028840937330463245, - "val_f2": 0.007163049790150459, - "val_frequency_bias": 454.42773645058446, - "val_threshold_metrics": { - "0.1000": { - "threshold": 0.1, - "tp": 1758, - "fp": 4044915, - "fn": 124, - "precision": 0.00043443095105534843, - "recall": 0.9341126461211477, - "far": 0.9995655690489447, - "csi": 0.00043441763943187663, - "f1": 0.0008684580053871072, - "f2": 0.0021681214128258562, - "frequency_bias": 2150.1981934112646, - "predicted_positive_rate": 0.5005161410018553 - }, - "0.2000": { - "threshold": 0.2, - "tp": 1623, - "fp": 2661790, - "fn": 259, - "precision": 0.0006093685057480759, - "recall": 0.8623804463336876, - "far": 0.9993906314942519, - "csi": 0.0006093092542925706, - "f1": 0.001217876445196498, - "f2": 0.0030382550569256306, - "frequency_bias": 1415.2035069075453, - "predicted_positive_rate": 0.3294264687693259 - }, - "0.3000": { - "threshold": 0.3, - "tp": 1509, - "fp": 1763618, - "fn": 373, - "precision": 0.0008548959933194609, - "recall": 0.8018065887353879, - "far": 0.9991451040066806, - "csi": 0.0008547153780798641, - "f1": 0.0017079709271429857, - "f2": 0.00425632737334676, - "frequency_bias": 937.8995749202976, - "predicted_positive_rate": 0.2183212121212121 - }, - "0.5000": { - "threshold": 0.5, - "tp": 1236, - "fp": 853997, - "fn": 646, - "precision": 0.0014452201914565972, - "recall": 0.6567481402763018, - "far": 0.9985547798085435, - "csi": 0.001444129368754228, - "f1": 0.0028840937330463245, - "f2": 0.007163049790150459, - "frequency_bias": 454.42773645058446, - "predicted_positive_rate": 0.10578021026592455 - }, - "0.7000": { - "threshold": 0.7, - "tp": 941, - "fp": 447133, - "fn": 941, - "precision": 0.0021000995371300278, - "recall": 0.5, - "far": 0.99789990046287, - "csi": 0.00209569836197009, - "f1": 0.004182631190605304, - "f2": 0.010326995930658776, - "frequency_bias": 238.08395324123273, - "predicted_positive_rate": 0.05542040816326531 - }, - "0.9000": { - "threshold": 0.9, - "tp": 549, - "fp": 144590, - "fn": 1333, - "precision": 0.003782580836301752, - "recall": 0.2917109458023379, - "far": 0.9962174191636982, - "csi": 0.003748156644273308, - "f1": 0.007468320852123166, - "f2": 0.017980310086659197, - "frequency_bias": 77.11955366631243, - "predicted_positive_rate": 0.017951638837353123 - } - }, - "val_topk_area_metrics": { - "0.0100": { - "fraction": 0.01, - "cells": 80850, - "precision": 0.004155843984335661, - "recall": 0.17853347957134247, - "lift": 17.85334704984204 - }, - "0.0500": { - "fraction": 0.05, - "cells": 404250, - "precision": 0.0021917130798101425, - "recall": 0.4707757830619812, - "lift": 9.415515691882671 - }, - "0.1000": { - "fraction": 0.1, - "cells": 808500, - "precision": 0.0015003092121332884, - "recall": 0.6445270776748657, - "lift": 6.445271080255079 - } - }, - "val_fss": { - "0.1000": { - "1": 0.0009126233899767744, - "2": 0.0009396067987926804, - "4": 0.0009843825552887653, - "8": 0.0010422517525249253 - }, - "0.2000": { - "1": 0.001297803743524062, - "2": 0.0013508992011256327, - "4": 0.001432314924131206, - "8": 0.001506973317854765 - }, - "0.3000": { - "1": 0.0018602197823269773, - "2": 0.0019451288490225371, - "4": 0.00207146808966463, - "8": 0.002172537528579155 - }, - "0.5000": { - "1": 0.003223589167235419, - "2": 0.0034269782605044208, - "4": 0.0037462044010848627, - "8": 0.0039810770352668445 - }, - "0.7000": { - "1": 0.004801884901590947, - "2": 0.005215736541223648, - "4": 0.00592818083960267, - "8": 0.006606712295029626 - }, - "0.9000": { - "1": 0.0097866399903066, - "2": 0.011010246340202201, - "4": 0.014217749893489784, - "8": 0.02042521801444397 - } - }, - "minutes": 0.3174507300059001 - }, - { - "epoch": 4, - "train_loss": 0.49877583144292464, - "train_main_loss": 0.470812586229202, - "train_aux_spatial_loss": 0.05592649097478617, - "val_loss": 0.4736149037877719, - "val_pr_auc": 0.0029667986240187873, - "val_auroc": 0.8830141091475292, - "val_brier": 0.131561741232872, - "val_brier_skill_score": -564.3106983635114, - "val_ece": 0.2541514431854187, - "val_positive_rate": 0.00023277674335986376, - "val_precision": 0.0010748343359518568, - "val_positive_recall": 0.781615302869288, - "val_far": 0.9989251656640481, - "val_csi": 0.001074511648699702, - "val_f1": 0.0021467166253789772, - "val_f2": 0.005344772333045809, - "val_frequency_bias": 727.1960680127523, - "val_threshold_metrics": { - "0.1000": { - "threshold": 0.1, - "tp": 1852, - "fp": 4888523, - "fn": 30, - "precision": 0.00037870306469340285, - "recall": 0.9840595111583422, - "far": 0.9996212969353065, - "csi": 0.00037870074155412484, - "f1": 0.0007571147631859896, - "f2": 0.0018906050201484186, - "frequency_bias": 2598.498937300744, - "predicted_positive_rate": 0.6048701298701299 - }, - "0.2000": { - "threshold": 0.2, - "tp": 1764, - "fp": 3534590, - "fn": 118, - "precision": 0.0004988188399690755, - "recall": 0.9373007438894793, - "far": 0.999501181160031, - "csi": 0.0004988021960869477, - "f1": 0.0009971070329961031, - "f2": 0.0024887961845230736, - "frequency_bias": 1879.0403825717322, - "predicted_positive_rate": 0.4373969078540507 - }, - "0.3000": { - "threshold": 0.3, - "tp": 1675, - "fp": 2659269, - "fn": 207, - "precision": 0.0006294758551852275, - "recall": 0.8900106269925611, - "far": 0.9993705241448148, - "csi": 0.0006294268908453522, - "f1": 0.001258061923685588, - "f2": 0.0031385002353406747, - "frequency_bias": 1413.8916046758768, - "predicted_positive_rate": 0.32912108843537413 - }, - "0.5000": { - "threshold": 0.5, - "tp": 1471, - "fp": 1367112, - "fn": 411, - "precision": 0.0010748343359518568, - "recall": 0.781615302869288, - "far": 0.9989251656640481, - "csi": 0.001074511648699702, - "f1": 0.0021467166253789772, - "f2": 0.005344772333045809, - "frequency_bias": 727.1960680127523, - "predicted_positive_rate": 0.1692743351886209 - }, - "0.7000": { - "threshold": 0.7, - "tp": 1158, - "fp": 711745, - "fn": 724, - "precision": 0.001624344405900943, - "recall": 0.6153028692879915, - "far": 0.998375655594099, - "csi": 0.001622696450666805, - "f1": 0.003240135145533272, - "f2": 0.00803685571553695, - "frequency_bias": 378.8007438894793, - "predicted_positive_rate": 0.08817600494743352 - }, - "0.9000": { - "threshold": 0.9, - "tp": 796, - "fp": 316530, - "fn": 1086, - "precision": 0.0025084613299887182, - "recall": 0.42295430393198724, - "far": 0.9974915386700113, - "csi": 0.002499905782445385, - "f1": 0.004987343675597103, - "f2": 0.012251657667752283, - "frequency_bias": 168.61105207226356, - "predicted_positive_rate": 0.039248732220160794 - } - }, - "val_topk_area_metrics": { - "0.0100": { - "fraction": 0.01, - "cells": 80850, - "precision": 0.004007420968264341, - "recall": 0.17215727269649506, - "lift": 17.21572744090257 - }, - "0.0500": { - "fraction": 0.05, - "cells": 404250, - "precision": 0.002221397589892149, - "recall": 0.4771519601345062, - "lift": 9.543039213577943 - }, - "0.1000": { - "fraction": 0.1, - "cells": 808500, - "precision": 0.0015114409616217017, - "recall": 0.6493092179298401, - "lift": 6.493092650948695 - } - }, - "val_fss": { - "0.1000": { - "1": 0.0007882371861520632, - "2": 0.0008068278383324134, - "4": 0.0008376526392290806, - "8": 0.0008803104743912993 - }, - "0.2000": { - "1": 0.0010456654051311798, - "2": 0.0010781494695234484, - "4": 0.00113452177870621, - "8": 0.0012067720438921903 - }, - "0.3000": { - "1": 0.001341867235808647, - "2": 0.0013990919727997486, - "4": 0.0014963756841404052, - "8": 0.0016008149955504258 - }, - "0.5000": { - "1": 0.002401290586052096, - "2": 0.002581335230269266, - "4": 0.0028727047230912017, - "8": 0.0031094243892790807 - }, - "0.7000": { - "1": 0.0037611063108259346, - "2": 0.004170442864082857, - "4": 0.004911188688805468, - "8": 0.005619346494556421 - }, - "0.9000": { - "1": 0.00620797412047025, - "2": 0.0072512678310659995, - "4": 0.009527837752599244, - "8": 0.012433301157436127 - } - }, - "minutes": 0.34307103554407753 - }, - { - "epoch": 5, - "train_loss": 0.482236114504778, - "train_main_loss": 0.45522032067156104, - "train_aux_spatial_loss": 0.05403158970131391, - "val_loss": 0.6312598725159962, - "val_pr_auc": 0.001904165446995025, - "val_auroc": 0.8616241410980919, - "val_brier": 0.17233318090438843, - "val_brier_skill_score": -739.5024434559806, - "val_ece": 0.2959978739301973, - "val_positive_rate": 0.00023277674335986376, - "val_precision": 0.0008378176445403336, - "val_positive_recall": 0.7954303931987248, - "val_far": 0.9991621823554596, - "val_csi": 0.0008376371581886446, - "val_f1": 0.0016738722188087553, - "val_f2": 0.004171512996896305, - "val_frequency_bias": 949.4075451647184, - "val_threshold_metrics": { - "0.1000": { - "threshold": 0.1, - "tp": 1849, - "fp": 5330029, - "fn": 33, - "precision": 0.00034678212817322525, - "recall": 0.9824654622741764, - "far": 0.9996532178718268, - "csi": 0.00034677998188641936, - "f1": 0.000693319534437245, - "f2": 0.001731466009514916, - "frequency_bias": 2833.0913921360257, - "predicted_positive_rate": 0.6594777983920841 - }, - "0.2000": { - "threshold": 0.2, - "tp": 1760, - "fp": 3845604, - "fn": 122, - "precision": 0.0004574560660233864, - "recall": 0.9351753453772582, - "far": 0.9995425439339766, - "csi": 0.000457441560541091, - "f1": 0.0009144648068738656, - "f2": 0.0022828136300575997, - "frequency_bias": 2044.2954303931988, - "predicted_positive_rate": 0.4758644403215832 - }, - "0.3000": { - "threshold": 0.3, - "tp": 1689, - "fp": 2918064, - "fn": 193, - "precision": 0.0005784735900605291, - "recall": 0.8974495217853348, - "far": 0.9994215264099394, - "csi": 0.0005784353546264212, - "f1": 0.0011562019211845422, - "f2": 0.002884929735136463, - "frequency_bias": 1551.4096705632305, - "predicted_positive_rate": 0.36113209647495365 - }, - "0.5000": { - "threshold": 0.5, - "tp": 1497, - "fp": 1785288, - "fn": 385, - "precision": 0.0008378176445403336, - "recall": 0.7954303931987248, - "far": 0.9991621823554596, - "csi": 0.0008376371581886446, - "f1": 0.0016738722188087553, - "f2": 0.004171512996896305, - "frequency_bias": 949.4075451647184, - "predicted_positive_rate": 0.221 - }, - "0.7000": { - "threshold": 0.7, - "tp": 1281, - "fp": 1103534, - "fn": 601, - "precision": 0.0011594701375343384, - "recall": 0.6806588735387885, - "far": 0.9988405298624656, - "csi": 0.0011588397490175644, - "f1": 0.002314996787738649, - "f2": 0.005758115976816504, - "frequency_bias": 587.0430393198725, - "predicted_positive_rate": 0.13664996907854052 - }, - "0.9000": { - "threshold": 0.9, - "tp": 929, - "fp": 573131, - "fn": 953, - "precision": 0.0016182977389123088, - "recall": 0.49362380446333687, - "far": 0.9983817022610877, - "csi": 0.0016156156469505907, - "f1": 0.003226019286664282, - "f2": 0.007986753509357139, - "frequency_bias": 305.02656748140276, - "predicted_positive_rate": 0.07100309214594928 - } - }, - "val_topk_area_metrics": { - "0.0100": { - "fraction": 0.01, - "cells": 80850, - "precision": 0.002671614056453109, - "recall": 0.1147715225815773, - "lift": 11.477151960678897 - }, - "0.0500": { - "fraction": 0.05, - "cells": 404250, - "precision": 0.0017588125774636865, - "recall": 0.37778958678245544, - "lift": 7.555791665770626 - }, - "0.1000": { - "fraction": 0.1, - "cells": 808500, - "precision": 0.0013654916547238827, - "recall": 0.586609959602356, - "lift": 5.866100002150498 - } - }, - "val_fss": { - "0.1000": { - "1": 0.000720903487296698, - "2": 0.0007379622256847851, - "4": 0.0007659140531370845, - "8": 0.0008043442027690917 - }, - "0.2000": { - "1": 0.0009597972490167361, - "2": 0.0009899780347129017, - "4": 0.0010392513227689415, - "8": 0.0010982538039768786 - }, - "0.3000": { - "1": 0.0012276114505155311, - "2": 0.0012735628537789978, - "4": 0.0013438876742029038, - "8": 0.0014172341902246588 - }, - "0.5000": { - "1": 0.0018178385262409558, - "2": 0.0018914580444638895, - "4": 0.002005861068742676, - "8": 0.0021171275849246607 - }, - "0.7000": { - "1": 0.0025095121564123746, - "2": 0.0026071244397790627, - "4": 0.002787665044875265, - "8": 0.003009586547953802 - }, - "0.9000": { - "1": 0.00349269641993466, - "2": 0.0036221493430278473, - "4": 0.004001940413619787, - "8": 0.004630463384076355 - } - }, - "minutes": 0.32958046595255536 - }, - { - "epoch": 6, - "train_loss": 0.4705298146818342, - "train_main_loss": 0.4441832022530821, - "train_aux_spatial_loss": 0.052693225361118125, - "val_loss": 0.3288325930635134, - "val_pr_auc": 0.0028623518804880635, - "val_auroc": 0.8795604343470619, - "val_brier": 0.08432164788246155, - "val_brier_skill_score": -361.32364519424686, - "val_ece": 0.1792877230218383, - "val_positive_rate": 0.00023277674335986376, - "val_precision": 0.0015739775487745805, - "val_positive_recall": 0.6923485653560042, - "val_far": 0.9984260224512255, - "val_csi": 0.001572877460412497, - "val_f1": 0.0031408148040124335, - "val_f2": 0.007798967399957145, - "val_frequency_bias": 439.8719447396387, - "val_threshold_metrics": { - "0.1000": { - "threshold": 0.1, - "tp": 1758, - "fp": 3773348, - "fn": 124, - "precision": 0.0004656822881264791, - "recall": 0.9341126461211477, - "far": 0.9995343177118735, - "csi": 0.0004656669924746307, - "f1": 0.0009309004953153147, - "f2": 0.002323777558177714, - "frequency_bias": 2005.9011689691818, - "predicted_positive_rate": 0.46692714904143473 - }, - "0.2000": { - "threshold": 0.2, - "tp": 1577, - "fp": 2267086, - "fn": 305, - "precision": 0.0006951230746920102, - "recall": 0.8379383634431455, - "far": 0.999304876925308, - "csi": 0.000695029634618029, - "f1": 0.001389093807874321, - "f2": 0.0034641205417295824, - "frequency_bias": 1205.4532412327312, - "predicted_positive_rate": 0.28060148423005565 - }, - "0.3000": { - "threshold": 0.3, - "tp": 1469, - "fp": 1517955, - "fn": 413, - "precision": 0.0009668137399435576, - "recall": 0.7805526036131775, - "far": 0.9990331862600564, - "csi": 0.0009665510182999888, - "f1": 0.0019312353990584405, - "f2": 0.004810236340107613, - "frequency_bias": 807.3453772582359, - "predicted_positive_rate": 0.18793123067408782 - }, - "0.5000": { - "threshold": 0.5, - "tp": 1303, - "fp": 826536, - "fn": 579, - "precision": 0.0015739775487745805, - "recall": 0.6923485653560042, - "far": 0.9984260224512255, - "csi": 0.001572877460412497, - "f1": 0.0031408148040124335, - "f2": 0.007798967399957145, - "frequency_bias": 439.8719447396387, - "predicted_positive_rate": 0.10239196042053185 - }, - "0.7000": { - "threshold": 0.7, - "tp": 1048, - "fp": 466249, - "fn": 834, - "precision": 0.0022426850589667814, - "recall": 0.5568544102019128, - "far": 0.9977573149410333, - "csi": 0.002238689597569911, - "f1": 0.004467378122209221, - "f2": 0.011035644711209392, - "frequency_bias": 248.298087141339, - "predicted_positive_rate": 0.057798021026592455 - }, - "0.9000": { - "threshold": 0.9, - "tp": 742, - "fp": 226824, - "fn": 1140, - "precision": 0.0032605925313974847, - "recall": 0.3942614240170032, - "far": 0.9967394074686026, - "csi": 0.0032443398948868852, - "f1": 0.006467696384365957, - "f2": 0.015780921673883636, - "frequency_bias": 120.91710945802338, - "predicted_positive_rate": 0.028146691403834263 - } - }, - "val_topk_area_metrics": { - "0.0100": { - "fraction": 0.01, - "cells": 80850, - "precision": 0.004848484881222248, - "recall": 0.20828905701637268, - "lift": 20.82890589171393 - }, - "0.0500": { - "fraction": 0.05, - "cells": 404250, - "precision": 0.002466295612975955, - "recall": 0.5297555923461914, - "lift": 10.595111768374379 - }, - "0.1000": { - "fraction": 0.1, - "cells": 808500, - "precision": 0.0016029684338718653, - "recall": 0.6886290907859802, - "lift": 6.8862911764073385 - } - }, - "val_fss": { - "0.1000": { - "1": 0.0009832075371584148, - "2": 0.0010169042550060015, - "4": 0.0010721484966499384, - "8": 0.0011368377314868727 - }, - "0.2000": { - "1": 0.0015207106192908215, - "2": 0.0016047733082307225, - "4": 0.0017274780230661646, - "8": 0.0018439819139227653 - }, - "0.3000": { - "1": 0.002164479861285651, - "2": 0.002303876071255262, - "4": 0.002502814516543217, - "8": 0.002677417485086986 - }, - "0.5000": { - "1": 0.0035466153498271424, - "2": 0.0038054359562748052, - "4": 0.004256125038073377, - "8": 0.004679745806242752 - }, - "0.7000": { - "1": 0.005259027399887928, - "2": 0.0058170650135721536, - "4": 0.006905022198597899, - "8": 0.008165048178681578 - }, - "0.9000": { - "1": 0.008040151750769331, - "2": 0.009007045143097692, - "4": 0.011282893901345625, - "8": 0.014929724649083487 - } - }, - "minutes": 0.33652125199635824 - }, - { - "epoch": 7, - "train_loss": 0.462470636422584, - "train_main_loss": 0.4368088399746645, - "train_aux_spatial_loss": 0.05132359440849709, - "val_loss": 0.2877284549176693, - "val_pr_auc": 0.0013908105247300113, - "val_auroc": 0.8529700408228243, - "val_brier": 0.07498431950807571, - "val_brier_skill_score": -321.20186226017756, - "val_ece": 0.16345792957978955, - "val_positive_rate": 0.00023277674335986376, - "val_precision": 0.00137407744359688, - "val_positive_recall": 0.5425079702444209, - "val_far": 0.9986259225564031, - "val_csi": 0.001372487078323173, - "val_f1": 0.002741211878763797, - "val_f2": 0.006801479405040423, - "val_frequency_bias": 394.8161530286929, - "val_threshold_metrics": { - "0.1000": { - "threshold": 0.1, - "tp": 1712, - "fp": 3476150, - "fn": 170, - "precision": 0.0004922564495083474, - "recall": 0.9096705632306057, - "far": 0.9995077435504917, - "csi": 0.000492232388891189, - "f1": 0.0009839804307443306, - "f2": 0.002455966190297212, - "frequency_bias": 1847.9606801275238, - "predicted_positive_rate": 0.4301622758194187 - }, - "0.2000": { - "threshold": 0.2, - "tp": 1530, - "fp": 2143152, - "fn": 352, - "precision": 0.0007133924749683169, - "recall": 0.8129649309245484, - "far": 0.9992866075250317, - "csi": 0.0007132754072895814, - "f1": 0.0014255340162231363, - "f2": 0.003554485854075578, - "frequency_bias": 1139.575982996812, - "predicted_positive_rate": 0.2652667903525046 - }, - "0.3000": { - "threshold": 0.3, - "tp": 1354, - "fp": 1448620, - "fn": 528, - "precision": 0.000933809847624854, - "recall": 0.7194473963868225, - "far": 0.9990661901523752, - "csi": 0.0009334699297208829, - "f1": 0.0018651987524933604, - "f2": 0.00464493359185785, - "frequency_bias": 770.4431455897981, - "predicted_positive_rate": 0.1793412492269635 - }, - "0.5000": { - "threshold": 0.5, - "tp": 1021, - "fp": 742023, - "fn": 861, - "precision": 0.00137407744359688, - "recall": 0.5425079702444209, - "far": 0.9986259225564031, - "csi": 0.001372487078323173, - "f1": 0.002741211878763797, - "f2": 0.006801479405040423, - "frequency_bias": 394.8161530286929, - "predicted_positive_rate": 0.09190401978973407 - }, - "0.7000": { - "threshold": 0.7, - "tp": 669, - "fp": 387288, - "fn": 1213, - "precision": 0.0017244179122944037, - "recall": 0.3554729011689692, - "far": 0.9982755820877056, - "csi": 0.001719043091707994, - "f1": 0.0034321861075982648, - "f2": 0.008457969328798816, - "frequency_bias": 206.14080765143464, - "predicted_positive_rate": 0.0479847866419295 - }, - "0.9000": { - "threshold": 0.9, - "tp": 330, - "fp": 170442, - "fn": 1552, - "precision": 0.00193240109619844, - "recall": 0.17534537725823593, - "far": 0.9980675989038016, - "csi": 0.0019149973306097816, - "f1": 0.0038226742502345734, - "f2": 0.009254066180594505, - "frequency_bias": 90.73963868225292, - "predicted_positive_rate": 0.02112207792207792 - } - }, - "val_topk_area_metrics": { - "0.0100": { - "fraction": 0.01, - "cells": 80850, - "precision": 0.002251082332804799, - "recall": 0.09670563042163849, - "lift": 9.67056373550477 - }, - "0.0500": { - "fraction": 0.05, - "cells": 404250, - "precision": 0.0017093382775783539, - "recall": 0.3671625852584839, - "lift": 7.343251962829395 - }, - "0.1000": { - "fraction": 0.1, - "cells": 808500, - "precision": 0.0013395176501944661, - "recall": 0.575451672077179, - "lift": 5.754516670609246 - } - }, - "val_fss": { - "0.1000": { - "1": 0.001043168774059744, - "2": 0.0010819388013025844, - "4": 0.0011386362023144692, - "8": 0.0011957179318650146 - }, - "0.2000": { - "1": 0.0015483993663276685, - "2": 0.001623910048448085, - "4": 0.001730854780802371, - "8": 0.0018125949671790043 - }, - "0.3000": { - "1": 0.0020767730914419413, - "2": 0.002216874591054907, - "4": 0.0024268280557646627, - "8": 0.002582330742552652 - }, - "0.5000": { - "1": 0.003179344493123515, - "2": 0.0034580954569070066, - "4": 0.004022383332455304, - "8": 0.004531362170723141 - }, - "0.7000": { - "1": 0.004287884524174168, - "2": 0.004893582597532031, - "4": 0.006272511260630553, - "8": 0.00800338321521643 - }, - "0.9000": { - "1": 0.00525683498068652, - "2": 0.006461524593853074, - "4": 0.009610762688246233, - "8": 0.014847628576755034 - } - }, - "minutes": 0.3354960521062215 - }, - { - "epoch": 8, - "train_loss": 0.45163499733658824, - "train_main_loss": 0.42649205012479513, - "train_aux_spatial_loss": 0.05028589440088308, - "val_loss": 0.389868001639843, - "val_pr_auc": 0.004338335499478623, - "val_auroc": 0.8726679507819024, - "val_brier": 0.11727461218833923, - "val_brier_skill_score": -502.9200020859495, - "val_ece": 0.24583306557241666, - "val_positive_rate": 0.00023277674335986376, - "val_precision": 0.0011231134016327171, - "val_positive_recall": 0.7454835281615303, - "val_far": 0.9988768865983673, - "val_csi": 0.0011226829160948559, - "val_f1": 0.0022428478252529, - "val_f2": 0.005581929031919244, - "val_frequency_bias": 663.7651434643996, - "val_threshold_metrics": { - "0.1000": { - "threshold": 0.1, - "tp": 1825, - "fp": 5047814, - "fn": 57, - "precision": 0.0003614119742025123, - "recall": 0.9697130712008502, - "far": 0.9996385880257975, - "csi": 0.0003614078946534603, - "f1": 0.0007225546523512424, - "f2": 0.0018043699169910743, - "frequency_bias": 2683.123804463337, - "predicted_positive_rate": 0.6245688311688312 - }, - "0.2000": { - "threshold": 0.2, - "tp": 1727, - "fp": 3624057, - "fn": 155, - "precision": 0.0004763107785791983, - "recall": 0.9176408076514346, - "far": 0.9995236892214208, - "csi": 0.00047629041746151824, - "f1": 0.0009521273457920327, - "f2": 0.002376619459050035, - "frequency_bias": 1926.5589798087142, - "predicted_positive_rate": 0.4484581323438466 - }, - "0.3000": { - "threshold": 0.3, - "tp": 1618, - "fp": 2596717, - "fn": 264, - "precision": 0.0006227064639471046, - "recall": 0.8597236981934112, - "far": 0.9993772935360529, - "csi": 0.0006226432012018784, - "f1": 0.0012445115157696453, - "f2": 0.0031045377289596575, - "frequency_bias": 1380.624335812965, - "predicted_positive_rate": 0.32137724180581323 - }, - "0.5000": { - "threshold": 0.5, - "tp": 1403, - "fp": 1247803, - "fn": 479, - "precision": 0.0011231134016327171, - "recall": 0.7454835281615303, - "far": 0.9988768865983673, - "csi": 0.0011226829160948559, - "f1": 0.0022428478252529, - "f2": 0.005581929031919244, - "frequency_bias": 663.7651434643996, - "predicted_positive_rate": 0.1545090909090909 - }, - "0.7000": { - "threshold": 0.7, - "tp": 1064, - "fp": 572445, - "fn": 818, - "precision": 0.0018552455148916581, - "recall": 0.565356004250797, - "far": 0.9981447544851083, - "csi": 0.001852603133754812, - "f1": 0.0036983546840322493, - "f2": 0.009156043418921686, - "frequency_bias": 304.7337938363443, - "predicted_positive_rate": 0.07093494124922696 - }, - "0.9000": { - "threshold": 0.9, - "tp": 616, - "fp": 143363, - "fn": 1266, - "precision": 0.004278401711360684, - "recall": 0.3273113708820404, - "far": 0.9957215982886393, - "csi": 0.004241109848876037, - "f1": 0.008446397597712891, - "f2": 0.02032909370524134, - "frequency_bias": 76.50318809776833, - "predicted_positive_rate": 0.017808163265306123 - } - }, - "val_topk_area_metrics": { - "0.0100": { - "fraction": 0.01, - "cells": 80850, - "precision": 0.005640074145048857, - "recall": 0.24229542911052704, - "lift": 24.229543139236732 - }, - "0.0500": { - "fraction": 0.05, - "cells": 404250, - "precision": 0.002325293840840459, - "recall": 0.4994686543941498, - "lift": 9.989373539974505 - }, - "0.1000": { - "fraction": 0.1, - "cells": 808500, - "precision": 0.0015052566304802895, - "recall": 0.6466525197029114, - "lift": 6.466525000537624 - } - }, - "val_fss": { - "0.1000": { - "1": 0.0007513765476421241, - "2": 0.0007711290741327925, - "4": 0.0008023612192573637, - "8": 0.0008460677286081797 - }, - "0.2000": { - "1": 0.0010087632973743155, - "2": 0.001044669279142485, - "4": 0.0010981556816972438, - "8": 0.0011592603845778093 - }, - "0.3000": { - "1": 0.00134257679793226, - "2": 0.0014032317950684048, - "4": 0.0014828012007669367, - "8": 0.0015567722214533397 - }, - "0.5000": { - "1": 0.002475717082535134, - "2": 0.0025902566873002852, - "4": 0.0027648772728405824, - "8": 0.0028595649359570707 - }, - "0.7000": { - "1": 0.004229141272194981, - "2": 0.004534755794878054, - "4": 0.004987145697110007, - "8": 0.005228016705880245 - }, - "0.9000": { - "1": 0.011542436340720164, - "2": 0.013278581972967496, - "4": 0.017111917801594156, - "8": 0.022356010708446727 - } - }, - "minutes": 0.34035869836807253 - }, - { - "epoch": 9, - "train_loss": 0.4406228200723544, - "train_main_loss": 0.4159060165082645, - "train_aux_spatial_loss": 0.04943360574618867, - "val_loss": 0.31516960461934407, - "val_pr_auc": 0.005532091282032133, - "val_auroc": 0.8821471865606735, - "val_brier": 0.07678419351577759, - "val_brier_skill_score": -328.9357826440623, - "val_ece": 0.18066359303804805, - "val_positive_rate": 0.00023277674335986376, - "val_precision": 0.0017832156643714674, - "val_positive_recall": 0.6514346439957492, - "val_far": 0.9982167843356285, - "val_csi": 0.0017815158287536074, - "val_f1": 0.003556695348445904, - "val_f2": 0.008819509387813826, - "val_frequency_bias": 365.31455897980874, - "val_threshold_metrics": { - "0.1000": { - "threshold": 0.1, - "tp": 1795, - "fp": 4233910, - "fn": 87, - "precision": 0.00042377833206042444, - "recall": 0.9537725823591924, - "far": 0.9995762216679396, - "csi": 0.00042376962797040083, - "f1": 0.0008471802466828409, - "f2": 0.002115132494491818, - "frequency_bias": 2250.640276301807, - "predicted_positive_rate": 0.5238967223252937 - }, - "0.2000": { - "threshold": 0.2, - "tp": 1595, - "fp": 2216109, - "fn": 287, - "precision": 0.0007192123024533482, - "recall": 0.8475026567481403, - "far": 0.9992807876975467, - "csi": 0.0007191192389869932, - "f1": 0.0014372049562395872, - "f2": 0.0035838959712964765, - "frequency_bias": 1178.376195536663, - "predicted_positive_rate": 0.2742985776128633 - }, - "0.3000": { - "threshold": 0.3, - "tp": 1456, - "fp": 1317007, - "fn": 426, - "precision": 0.0011043161620766, - "recall": 0.7736450584484591, - "far": 0.9988956838379234, - "csi": 0.0011039594689166413, - "f1": 0.0022054841726972876, - "f2": 0.005490233342458584, - "frequency_bias": 700.5648246546227, - "predicted_positive_rate": 0.1630752009894867 - }, - "0.5000": { - "threshold": 0.5, - "tp": 1226, - "fp": 686296, - "fn": 656, - "precision": 0.0017832156643714674, - "recall": 0.6514346439957492, - "far": 0.9982167843356285, - "csi": 0.0017815158287536074, - "f1": 0.003556695348445904, - "f2": 0.008819509387813826, - "frequency_bias": 365.31455897980874, - "predicted_positive_rate": 0.08503673469387756 - }, - "0.7000": { - "threshold": 0.7, - "tp": 959, - "fp": 404322, - "fn": 923, - "precision": 0.0023662594594861344, - "recall": 0.5095642933049946, - "far": 0.9976337405405139, - "csi": 0.0023608827091806087, - "f1": 0.004710644140061843, - "f2": 0.011615541327829578, - "frequency_bias": 215.34590860786398, - "predicted_positive_rate": 0.05012752009894867 - }, - "0.9000": { - "threshold": 0.9, - "tp": 736, - "fp": 206564, - "fn": 1146, - "precision": 0.0035504100337674866, - "recall": 0.39107332624867164, - "far": 0.9964495899662326, - "csi": 0.0035308904944206174, - "f1": 0.007036934344255241, - "f2": 0.01712998305621241, - "frequency_bias": 110.14877789585547, - "predicted_positive_rate": 0.025640074211502783 - } - }, - "val_topk_area_metrics": { - "0.0100": { - "fraction": 0.01, - "cells": 80850, - "precision": 0.006629561074078083, - "recall": 0.2848033905029297, - "lift": 28.480341198987567 - }, - "0.0500": { - "fraction": 0.05, - "cells": 404250, - "precision": 0.0023698206059634686, - "recall": 0.5090329647064209, - "lift": 10.180658822517413 - }, - "0.1000": { - "fraction": 0.1, - "cells": 808500, - "precision": 0.0015807050513103604, - "recall": 0.6790648102760315, - "lift": 6.790648535135884 - } - }, - "val_fss": { - "0.1000": { - "1": 0.0008930744755271114, - "2": 0.000921422318230225, - "4": 0.0009668763668795322, - "8": 0.0010260119580676186 - }, - "0.2000": { - "1": 0.0015794562705196968, - "2": 0.0016669067534983428, - "4": 0.001799706921313815, - "8": 0.0019474635373948335 - }, - "0.3000": { - "1": 0.00246002599564632, - "2": 0.0026177117656231985, - "4": 0.002871564863597964, - "8": 0.003131757057791784 - }, - "0.5000": { - "1": 0.004021237206644379, - "2": 0.004332716953577953, - "4": 0.004884649782999739, - "8": 0.0054515150420901826 - }, - "0.7000": { - "1": 0.005615133103695769, - "2": 0.006360224351620203, - "4": 0.007663492450681542, - "8": 0.00913398414028943 - }, - "0.9000": { - "1": 0.00885985202765427, - "2": 0.01012085108726446, - "4": 0.012505376077093011, - "8": 0.015618568706976754 - } - }, - "minutes": 0.33895504077275596 - }, - { - "epoch": 10, - "train_loss": 0.4308571537754615, - "train_main_loss": 0.4065246517781404, - "train_aux_spatial_loss": 0.04866500381171554, - "val_loss": 0.2545071696241697, - "val_pr_auc": 0.00492243368142961, - "val_auroc": 0.8938939230518667, - "val_brier": 0.06364420056343079, - "val_brier_skill_score": -272.47424205655614, - "val_ece": 0.15454458198181653, - "val_positive_rate": 0.00023277674335986376, - "val_precision": 0.002100446914367563, - "val_positive_recall": 0.6615302869287991, - "val_far": 0.9978995530856324, - "val_csi": 0.0020981920157473947, - "val_f1": 0.004187597647545546, - "val_f2": 0.010370523390736333, - "val_frequency_bias": 314.9473963868225, - "val_threshold_metrics": { - "0.1000": { - "threshold": 0.1, - "tp": 1759, - "fp": 3554529, - "fn": 123, - "precision": 0.000494616858927061, - "recall": 0.934643995749203, - "far": 0.999505383141073, - "csi": 0.0004945997523908232, - "f1": 0.0009887104888186905, - "f2": 0.0024678602935729564, - "frequency_bias": 1889.6323060573857, - "predicted_positive_rate": 0.43986246134817564 - }, - "0.2000": { - "threshold": 0.2, - "tp": 1593, - "fp": 1928992, - "fn": 289, - "precision": 0.0008251384942905907, - "recall": 0.8464399574920297, - "far": 0.9991748615057094, - "csi": 0.0008250149932103285, - "f1": 0.0016486698091092889, - "f2": 0.004109667496167664, - "frequency_bias": 1025.8156216790649, - "predicted_positive_rate": 0.23878602350030922 - }, - "0.3000": { - "threshold": 0.3, - "tp": 1485, - "fp": 1212388, - "fn": 397, - "precision": 0.0012233569739173702, - "recall": 0.7890541976620616, - "far": 0.9987766430260826, - "csi": 0.0012229570029729797, - "f1": 0.0024429264119826774, - "f2": 0.006079084592201907, - "frequency_bias": 644.990967056323, - "predicted_positive_rate": 0.15013889919604206 - }, - "0.5000": { - "threshold": 0.5, - "tp": 1245, - "fp": 591486, - "fn": 637, - "precision": 0.002100446914367563, - "recall": 0.6615302869287991, - "far": 0.9978995530856324, - "csi": 0.0020981920157473947, - "f1": 0.004187597647545546, - "f2": 0.010370523390736333, - "frequency_bias": 314.9473963868225, - "predicted_positive_rate": 0.07331243042671615 - }, - "0.7000": { - "threshold": 0.7, - "tp": 948, - "fp": 305265, - "fn": 934, - "precision": 0.0030958842374425644, - "recall": 0.5037194473963869, - "far": 0.9969041157625574, - "csi": 0.0030864699964512107, - "f1": 0.0061539460231422125, - "f2": 0.015108003098096838, - "frequency_bias": 162.70616365568543, - "predicted_positive_rate": 0.03787421150278293 - }, - "0.9000": { - "threshold": 0.9, - "tp": 697, - "fp": 143067, - "fn": 1185, - "precision": 0.004848223477365683, - "recall": 0.3703506907545165, - "far": 0.9951517765226343, - "csi": 0.004808587848139691, - "f1": 0.009571151971217886, - "f2": 0.023034925838775352, - "frequency_bias": 76.38894792773645, - "predicted_positive_rate": 0.017781570810142238 - } - }, - "val_topk_area_metrics": { - "0.0100": { - "fraction": 0.01, - "cells": 80850, - "precision": 0.006716141011565924, - "recall": 0.2885228395462036, - "lift": 28.852285304047886 - }, - "0.0500": { - "fraction": 0.05, - "cells": 404250, - "precision": 0.0026839827187359333, - "recall": 0.5765143632888794, - "lift": 11.530287261501037 - }, - "0.1000": { - "fraction": 0.1, - "cells": 808500, - "precision": 0.001667285105213523, - "recall": 0.7162593007087708, - "lift": 7.162593140311982 - } - }, - "val_fss": { - "0.1000": { - "1": 0.0010543004451490923, - "2": 0.0010971495172527046, - "4": 0.0011617131253165436, - "8": 0.001235067155033076 - }, - "0.2000": { - "1": 0.0018153448265396666, - "2": 0.00192737425428402, - "4": 0.0020974439448524507, - "8": 0.0022549619868982296 - }, - "0.3000": { - "1": 0.0027408731997776714, - "2": 0.002930739084691436, - "4": 0.0032168364951372652, - "8": 0.0034672212918110956 - }, - "0.5000": { - "1": 0.0048490908581931835, - "2": 0.0052456592900168575, - "4": 0.005907669840747531, - "8": 0.006570117783041063 - }, - "0.7000": { - "1": 0.007577933898357814, - "2": 0.008558571828157113, - "4": 0.010329712940135849, - "8": 0.012440375731332498 - }, - "0.9000": { - "1": 0.012198325070619909, - "2": 0.013721391504927793, - "4": 0.017129504146446983, - "8": 0.022312787696272984 - } - }, - "minutes": 0.3265586694081624 - } -] diff --git a/models/wildfire_fm/metrics/reference_fireprone_seed_42_20260504/run_summary.json b/models/wildfire_fm/metrics/reference_fireprone_seed_42_20260504/run_summary.json deleted file mode 100644 index 6e9588c714c891a0f6c0147756632a7ec7df4c48..0000000000000000000000000000000000000000 --- a/models/wildfire_fm/metrics/reference_fireprone_seed_42_20260504/run_summary.json +++ /dev/null @@ -1,172 +0,0 @@ -{ - "run_name": "reference_fireprone_seed_42_20260504", - "device": "cuda", - "index_root": "", - "input_channels": 16, - "tile_size": 32, - "num_train_tiles": 27177, - "positive_train_tiles": 6724, - "negative_train_tiles": 20453, - "train_positive_rate": 0.0018277173713066195, - "metric_reference_positive_rate": 0.0002787771234976204, - "pos_weight": 50.0, - "use_aux_spatial_head": true, - "aux_spatial_radius": 2, - "aux_spatial_loss_weight": 0.5, - "aux_positive_rate": 0.004560296846011132, - "init_checkpoint": "", - "best_val_pr_auc": 0.005532091282032133, - "best_val_auroc": 0.8821471865606735, - "best_test_pr_auc": 0.0016100872246761122, - "best_test_auroc": 0.8121210769028695, - "best_test_brier": 0.09570477902889252, - "best_test_ece": 0.2090970246477704, - "best_test_topk_area_metrics": { - "0.0100": { - "fraction": 0.01, - "cells": 80850, - "precision": 0.0027087198104709387, - "recall": 0.11167771369218826, - "lift": 11.16777118374645 - }, - "0.0500": { - "fraction": 0.05, - "cells": 404250, - "precision": 0.0018082869937643409, - "recall": 0.37276899814605713, - "lift": 7.455379955815317 - }, - "0.1000": { - "fraction": 0.1, - "cells": 808500, - "precision": 0.0012504637707024813, - "recall": 0.5155532956123352, - "lift": 5.155532591738293 - } - }, - "best_test_threshold_metrics": { - "0.1000": { - "threshold": 0.1, - "tp": 1806, - "fp": 4593586, - "fn": 155, - "precision": 0.00039300238151609263, - "recall": 0.9209586945436002, - "far": 0.9996069976184839, - "csi": 0.0003929891262128317, - "f1": 0.000785669492858173, - "f2": 0.0019616634906400626, - "frequency_bias": 2343.392146863845, - "predicted_positive_rate": 0.5683849103277675 - }, - "0.2000": { - "threshold": 0.2, - "tp": 1550, - "fp": 2681763, - "fn": 411, - "precision": 0.0005776441287319072, - "recall": 0.7904130545639979, - "far": 0.999422355871268, - "csi": 0.0005775556651876273, - "f1": 0.0011544445743711814, - "f2": 0.002879802256055667, - "frequency_bias": 1368.3391126976032, - "predicted_positive_rate": 0.3318878169449598 - }, - "0.3000": { - "threshold": 0.3, - "tp": 1334, - "fp": 1753061, - "fn": 627, - "precision": 0.0007603760840631671, - "recall": 0.6802651708312085, - "far": 0.9992396239159368, - "csi": 0.0007601044317393172, - "f1": 0.0015190542236312, - "f2": 0.003784957658978152, - "frequency_bias": 894.6430392656807, - "predicted_positive_rate": 0.21699381570810142 - }, - "0.5000": { - "threshold": 0.5, - "tp": 1069, - "fp": 930711, - "fn": 892, - "precision": 0.0011472665221404193, - "recall": 0.5451300356960734, - "far": 0.9988527334778596, - "csi": 0.0011461692856652714, - "f1": 0.0022897141712744755, - "f2": 0.0056884455910023585, - "frequency_bias": 475.15553289138194, - "predicted_positive_rate": 0.11524799010513297 - }, - "0.7000": { - "threshold": 0.7, - "tp": 843, - "fp": 514028, - "fn": 1118, - "precision": 0.0016373033245220647, - "recall": 0.4298827129015808, - "far": 0.998362696675478, - "csi": 0.0016337557583591899, - "f1": 0.0032621819082409758, - "f2": 0.008063667581760615, - "frequency_bias": 262.5553289138195, - "predicted_positive_rate": 0.06368225108225108 - }, - "0.9000": { - "threshold": 0.9, - "tp": 494, - "fp": 230401, - "fn": 1467, - "precision": 0.0021395006388185106, - "recall": 0.2519122896481387, - "far": 0.9978604993611815, - "csi": 0.002125993062548954, - "f1": 0.004242965609647164, - "f2": 0.010346026413782414, - "frequency_bias": 117.74349821519633, - "predicted_positive_rate": 0.028558441558441557 - } - }, - "best_test_fss": { - "0.1000": { - "1": 0.0008268794294157278, - "2": 0.0008514076234114043, - "4": 0.0008867121111854059, - "8": 0.0009292544410690606 - }, - "0.2000": { - "1": 0.001245705784772344, - "2": 0.0012999708169816326, - "4": 0.0013747560191506025, - "8": 0.0014380244101879214 - }, - "0.3000": { - "1": 0.0016567546208199335, - "2": 0.0017403972985233773, - "4": 0.0018648856518644896, - "8": 0.001957178150997607 - }, - "0.5000": { - "1": 0.002557529963904237, - "2": 0.0027217819781574226, - "4": 0.0029546256500705415, - "8": 0.003125849885388443 - }, - "0.7000": { - "1": 0.0037552341559934366, - "2": 0.004108313787275497, - "4": 0.004633106027329581, - "8": 0.005223108606388305 - }, - "0.9000": { - "1": 0.005267983383194896, - "2": 0.006025153257028237, - "4": 0.007611176315223789, - "8": 0.009809039775823591 - } - }, - "epochs": 10 -} diff --git a/models/wildfire_fm/metrics/reference_fireprone_seed_42_20260504/tile_summary.json b/models/wildfire_fm/metrics/reference_fireprone_seed_42_20260504/tile_summary.json deleted file mode 100644 index 06b28297f37a2574afa4aa9a9b9e9d027adeb73e..0000000000000000000000000000000000000000 --- a/models/wildfire_fm/metrics/reference_fireprone_seed_42_20260504/tile_summary.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "index_root": "", - "tile_size": 32, - "num_train_tiles": 27177, - "positive_train_tiles": 6724, - "negative_train_tiles": 20453, - "input_channels": 16, - "train_positive_rate": 0.0018277173713066195, - "metric_reference_positive_rate": 0.0002787771234976204, - "pos_weight_source": "tiles", - "positive_rate_source": "tiles", - "pos_weight": 50.0, - "loss_type": "bce", - "train_target_mode": "dilate_max", - "train_target_radius": 2 -} diff --git a/models/wildfire_fm/metrics/reference_fireprone_seed_7_20260504/history.json b/models/wildfire_fm/metrics/reference_fireprone_seed_7_20260504/history.json deleted file mode 100644 index 06bea4d3b4fe37ea265d90d780631e02919bb805..0000000000000000000000000000000000000000 --- a/models/wildfire_fm/metrics/reference_fireprone_seed_7_20260504/history.json +++ /dev/null @@ -1,1682 +0,0 @@ -[ - { - "epoch": 1, - "train_loss": 0.5795233543991986, - "train_main_loss": 0.5336806391359897, - "train_aux_spatial_loss": 0.0916854308093406, - "val_loss": 0.5250215088327725, - "val_pr_auc": 0.00302956816406225, - "val_auroc": 0.8476268012299131, - "val_brier": 0.15357030928134918, - "val_brier_skill_score": -658.879825047864, - "val_ece": 0.28153358592477284, - "val_positive_rate": 0.00023277674335986376, - "val_precision": 0.0008681095747196535, - "val_positive_recall": 0.7412327311370882, - "val_far": 0.9991318904252804, - "val_csi": 0.0008678465647273562, - "val_f1": 0.001734188120251961, - "val_f2": 0.004320308609399505, - "val_frequency_bias": 853.84697130712, - "val_threshold_metrics": { - "0.1000": { - "threshold": 0.1, - "tp": 1838, - "fp": 5383727, - "fn": 44, - "precision": 0.0003412826695063563, - "recall": 0.9766206163655685, - "far": 0.9996587173304936, - "csi": 0.00034127988125391207, - "f1": 0.0006823268980650761, - "f2": 0.0017040314342808473, - "frequency_bias": 2861.6179596174284, - "predicted_positive_rate": 0.6661181199752628 - }, - "0.2000": { - "threshold": 0.2, - "tp": 1736, - "fp": 3897785, - "fn": 146, - "precision": 0.0004451828827181595, - "recall": 0.922422954303932, - "far": 0.9995548171172819, - "csi": 0.0004451662154742956, - "f1": 0.0008899362613910945, - "f2": 0.0022216255798174016, - "frequency_bias": 2072.009032943677, - "predicted_positive_rate": 0.4823155225726654 - }, - "0.3000": { - "threshold": 0.3, - "tp": 1615, - "fp": 2934454, - "fn": 267, - "precision": 0.0005500551928445823, - "recall": 0.8581296493092455, - "far": 0.9994499448071554, - "csi": 0.0005500051765193084, - "f1": 0.0010994056742266976, - "f2": 0.002743242366397302, - "frequency_bias": 1560.0791710945803, - "predicted_positive_rate": 0.36315015460729744 - }, - "0.5000": { - "threshold": 0.5, - "tp": 1395, - "fp": 1605545, - "fn": 487, - "precision": 0.0008681095747196535, - "recall": 0.7412327311370882, - "far": 0.9991318904252804, - "csi": 0.0008678465647273562, - "f1": 0.001734188120251961, - "f2": 0.004320308609399505, - "frequency_bias": 853.84697130712, - "predicted_positive_rate": 0.1987557204700062 - }, - "0.7000": { - "threshold": 0.7, - "tp": 1088, - "fp": 910358, - "fn": 794, - "precision": 0.001193707581140298, - "recall": 0.5781083953241233, - "far": 0.9988062924188597, - "csi": 0.0011926685959835132, - "f1": 0.002382495664208258, - "f2": 0.005919645169504252, - "frequency_bias": 484.29649309245485, - "predicted_positive_rate": 0.11273296227581941 - }, - "0.9000": { - "threshold": 0.9, - "tp": 701, - "fp": 394270, - "fn": 1181, - "precision": 0.0017748138470925713, - "recall": 0.37247608926673753, - "far": 0.9982251861529075, - "csi": 0.0017695228094266847, - "f1": 0.0035327942588313558, - "f2": 0.008708096169183027, - "frequency_bias": 209.86769394261424, - "predicted_positive_rate": 0.048852319109461965 - } - }, - "val_topk_area_metrics": { - "0.0100": { - "fraction": 0.01, - "cells": 80850, - "precision": 0.0043290043249726295, - "recall": 0.18597236275672913, - "lift": 18.59723726042579 - }, - "0.0500": { - "fraction": 0.05, - "cells": 404250, - "precision": 0.0017637599958106875, - "recall": 0.3788522779941559, - "lift": 7.577045586053171 - }, - "0.1000": { - "fraction": 0.1, - "cells": 808500, - "precision": 0.0012479900615289807, - "recall": 0.5361317992210388, - "lift": 5.361317645034825 - } - }, - "val_fss": { - "0.1000": { - "1": 0.0007067254219508268, - "2": 0.0007234213847714388, - "4": 0.0007495019182500728, - "8": 0.0007830543185699534 - }, - "0.2000": { - "1": 0.0009375066389885056, - "2": 0.0009701896382775521, - "4": 0.0010247521613826471, - "8": 0.0010816868830242354 - }, - "0.3000": { - "1": 0.0011841353931208998, - "2": 0.0012332094695163276, - "4": 0.0013160793352786637, - "8": 0.0013954395228068517 - }, - "0.5000": { - "1": 0.001895017558836809, - "2": 0.001973506551422588, - "4": 0.0021128830563802437, - "8": 0.0022158294695364678 - }, - "0.7000": { - "1": 0.0026256074379803485, - "2": 0.0027993105699366527, - "4": 0.0030546381753130403, - "8": 0.003188899049464955 - }, - "0.9000": { - "1": 0.004244499691496761, - "2": 0.004616982081301013, - "4": 0.005185018364351035, - "8": 0.005703623485958986 - } - }, - "minutes": 0.38692938486735023 - }, - { - "epoch": 2, - "train_loss": 0.5295870734150671, - "train_main_loss": 0.4984965211945434, - "train_aux_spatial_loss": 0.06218110369714074, - "val_loss": 0.1981954239308834, - "val_pr_auc": 0.002046157994282752, - "val_auroc": 0.8487165040319367, - "val_brier": 0.04589230194687843, - "val_brier_skill_score": -196.19569701633563, - "val_ece": 0.12395327587383405, - "val_positive_rate": 0.00023277674335986376, - "val_precision": 0.001856401444928432, - "val_positive_recall": 0.3926673751328374, - "val_far": 0.9981435985550716, - "val_csi": 0.0018510864800551068, - "val_f1": 0.0036953325799321942, - "val_f2": 0.00910973595325559, - "val_frequency_bias": 211.52072263549417, - "val_threshold_metrics": { - "0.1000": { - "threshold": 0.1, - "tp": 1653, - "fp": 3106762, - "fn": 229, - "precision": 0.0005317822748892925, - "recall": 0.8783209351753454, - "far": 0.9994682177251107, - "csi": 0.0005317431008504029, - "f1": 0.001062921000791886, - "f2": 0.0026524875455038815, - "frequency_bias": 1651.6551540913922, - "predicted_positive_rate": 0.3844669140383426 - }, - "0.2000": { - "threshold": 0.2, - "tp": 1331, - "fp": 1490961, - "fn": 551, - "precision": 0.0008919165954116219, - "recall": 0.7072263549415515, - "far": 0.9991080834045883, - "csi": 0.0008915873939858378, - "f1": 0.0017815863480424637, - "f2": 0.0044371991305623346, - "frequency_bias": 792.9287991498406, - "predicted_positive_rate": 0.18457538651824365 - }, - "0.3000": { - "threshold": 0.3, - "tp": 1067, - "fp": 834474, - "fn": 815, - "precision": 0.0012770169267576337, - "recall": 0.5669500531349628, - "far": 0.9987229830732424, - "csi": 0.0012757725179229898, - "f1": 0.0025482939924028835, - "f2": 0.006328070418909959, - "frequency_bias": 443.9643995749203, - "predicted_positive_rate": 0.10334458874458874 - }, - "0.5000": { - "threshold": 0.5, - "tp": 739, - "fp": 397343, - "fn": 1143, - "precision": 0.001856401444928432, - "recall": 0.3926673751328374, - "far": 0.9981435985550716, - "csi": 0.0018510864800551068, - "f1": 0.0036953325799321942, - "f2": 0.00910973595325559, - "frequency_bias": 211.52072263549417, - "predicted_positive_rate": 0.049237105751391466 - }, - "0.7000": { - "threshold": 0.7, - "tp": 617, - "fp": 206285, - "fn": 1265, - "precision": 0.0029820881383456903, - "recall": 0.32784272051009566, - "far": 0.9970179118616543, - "csi": 0.0029639664307983494, - "f1": 0.0059104145911564095, - "f2": 0.014386979433847877, - "frequency_bias": 109.93730074388948, - "predicted_positive_rate": 0.025590847247990104 - }, - "0.9000": { - "threshold": 0.9, - "tp": 348, - "fp": 86362, - "fn": 1534, - "precision": 0.004013377926421404, - "recall": 0.1849096705632306, - "far": 0.9959866220735786, - "csi": 0.003943610897058157, - "f1": 0.00785623984106917, - "f2": 0.018463889301555635, - "frequency_bias": 46.07332624867163, - "predicted_positive_rate": 0.010724799010513297 - } - }, - "val_topk_area_metrics": { - "0.0100": { - "fraction": 0.01, - "cells": 80850, - "precision": 0.004044526722282171, - "recall": 0.1737513244152069, - "lift": 17.375132343137437 - }, - "0.0500": { - "fraction": 0.05, - "cells": 404250, - "precision": 0.0018429190386086702, - "recall": 0.39585548639297485, - "lift": 7.917109810921228 - }, - "0.1000": { - "fraction": 0.1, - "cells": 808500, - "precision": 0.0013036486925557256, - "recall": 0.5600425004959106, - "lift": 5.600424998387127 - } - }, - "val_fss": { - "0.1000": { - "1": 0.0011313298295642094, - "2": 0.0011683798708436166, - "4": 0.0012267995062213988, - "8": 0.0012904870520601408 - }, - "0.2000": { - "1": 0.0019838003913644764, - "2": 0.0020951564940783207, - "4": 0.002271241452076689, - "8": 0.0024139105838063557 - }, - "0.3000": { - "1": 0.0029177942126825673, - "2": 0.0031946482222295902, - "4": 0.0036810675049008257, - "8": 0.004083577845361508 - }, - "0.5000": { - "1": 0.0046250396513106695, - "2": 0.005241562890837548, - "4": 0.006354549298866807, - "8": 0.00746207518491393 - }, - "0.7000": { - "1": 0.007309355074624491, - "2": 0.008090759551170335, - "4": 0.009921752700405628, - "8": 0.012786519539836116 - }, - "0.9000": { - "1": 0.01030998864826782, - "2": 0.011657290946334498, - "4": 0.016018656643936313, - "8": 0.025224938209435277 - } - }, - "minutes": 0.3169411182403564 - }, - { - "epoch": 3, - "train_loss": 0.5118337455830854, - "train_main_loss": 0.48268066439472623, - "train_aux_spatial_loss": 0.05830616311089252, - "val_loss": 0.31896051540970805, - "val_pr_auc": 0.001019430946186465, - "val_auroc": 0.8272157621144766, - "val_brier": 0.08598973602056503, - "val_brier_skill_score": -368.49129181739414, - "val_ece": 0.18084020724136254, - "val_positive_rate": 0.00023277674335986376, - "val_precision": 0.0010149243054251504, - "val_positive_recall": 0.4824654622741764, - "val_far": 0.9989850756945748, - "val_csi": 0.0010138205626927432, - "val_f1": 0.0020255875430827746, - "val_f2": 0.005032277515695386, - "val_frequency_bias": 475.3708820403826, - "val_threshold_metrics": { - "0.1000": { - "threshold": 0.1, - "tp": 1684, - "fp": 3663042, - "fn": 198, - "precision": 0.0004595159365256775, - "recall": 0.8947927736450585, - "far": 0.9995404840634743, - "csi": 0.0004594911108661462, - "f1": 0.0009185601515078786, - "f2": 0.0022928697197960707, - "frequency_bias": 1947.2507970244421, - "predicted_positive_rate": 0.4532747062461348 - }, - "0.2000": { - "threshold": 0.2, - "tp": 1507, - "fp": 2277895, - "fn": 375, - "precision": 0.0006611383161022057, - "recall": 0.8007438894792773, - "far": 0.9993388616838977, - "csi": 0.0006610295656110225, - "f1": 0.0013211857883542776, - "f2": 0.0032948100728924803, - "frequency_bias": 1211.1594048884165, - "predicted_positive_rate": 0.2819297464440322 - }, - "0.3000": { - "threshold": 0.3, - "tp": 1334, - "fp": 1581254, - "fn": 548, - "precision": 0.000842923110752767, - "recall": 0.7088204038257173, - "far": 0.9991570768892473, - "csi": 0.0008426313342631334, - "f1": 0.0016838438089708232, - "f2": 0.00419466252776527, - "frequency_bias": 840.9075451647184, - "predicted_positive_rate": 0.19574372294372294 - }, - "0.5000": { - "threshold": 0.5, - "tp": 908, - "fp": 893740, - "fn": 974, - "precision": 0.0010149243054251504, - "recall": 0.4824654622741764, - "far": 0.9989850756945748, - "csi": 0.0010138205626927432, - "f1": 0.0020255875430827746, - "f2": 0.005032277515695386, - "frequency_bias": 475.3708820403826, - "predicted_positive_rate": 0.11065528756957328 - }, - "0.7000": { - "threshold": 0.7, - "tp": 621, - "fp": 497577, - "fn": 1261, - "precision": 0.0012464923584598895, - "recall": 0.3299681190223167, - "far": 0.9987535076415401, - "csi": 0.0012433452996141825, - "f1": 0.002483602623580227, - "f2": 0.006139688289706283, - "frequency_bias": 264.7173219978746, - "predicted_positive_rate": 0.06162003710575139 - }, - "0.9000": { - "threshold": 0.9, - "tp": 286, - "fp": 184589, - "fn": 1596, - "precision": 0.0015469912102772144, - "recall": 0.15196599362380447, - "far": 0.9984530087897228, - "csi": 0.001533750556386784, - "f1": 0.0030628035361459013, - "f2": 0.007432316543920833, - "frequency_bias": 98.23326248671626, - "predicted_positive_rate": 0.022866419294990725 - } - }, - "val_topk_area_metrics": { - "0.0100": { - "fraction": 0.01, - "cells": 80850, - "precision": 0.0017439703224226832, - "recall": 0.07492029666900635, - "lift": 7.4920299049229895 - }, - "0.0500": { - "fraction": 0.05, - "cells": 404250, - "precision": 0.001338280737400055, - "recall": 0.2874601483345032, - "lift": 5.749202940480721 - }, - "0.1000": { - "fraction": 0.1, - "cells": 808500, - "precision": 0.0010674088262021542, - "recall": 0.4585547149181366, - "lift": 4.585547554258811 - } - }, - "val_fss": { - "0.1000": { - "1": 0.0009740815185863561, - "2": 0.001005812161545827, - "4": 0.001055261976948163, - "8": 0.0010992879644106113 - }, - "0.2000": { - "1": 0.0014229145916550001, - "2": 0.0014700339354011271, - "4": 0.0015364414825030925, - "8": 0.0015827285477550967 - }, - "0.3000": { - "1": 0.0018098067083015534, - "2": 0.0018650612200171723, - "4": 0.0019665999384206323, - "8": 0.002042833409939604 - }, - "0.5000": { - "1": 0.0023055038912407344, - "2": 0.0024273937090060205, - "4": 0.0026763229300285074, - "8": 0.0029077273131987713 - }, - "0.7000": { - "1": 0.002861966206934352, - "2": 0.0030903014392590356, - "4": 0.0036201637468585846, - "8": 0.004202406612231591 - }, - "0.9000": { - "1": 0.0038396208986245517, - "2": 0.004425201788265554, - "4": 0.005877068610142366, - "8": 0.008076184699332467 - } - }, - "minutes": 0.33083393971125286 - }, - { - "epoch": 4, - "train_loss": 0.5014545876805161, - "train_main_loss": 0.4733086560772328, - "train_aux_spatial_loss": 0.056291862974635516, - "val_loss": 0.714298881093661, - "val_pr_auc": 0.0030682622199010843, - "val_auroc": 0.8761108875200969, - "val_brier": 0.2028871327638626, - "val_brier_skill_score": -870.7904281054932, - "val_ece": 0.3354083814166639, - "val_positive_rate": 0.00023277674335986376, - "val_precision": 0.0007264174734926152, - "val_positive_recall": 0.844845908607864, - "val_far": 0.9992735825265073, - "val_csi": 0.0007263205787176193, - "val_f1": 0.0014515868400415207, - "val_f2": 0.003619638382190104, - "val_frequency_bias": 1163.0308182784272, - "val_threshold_metrics": { - "0.1000": { - "threshold": 0.1, - "tp": 1871, - "fp": 5911512, - "fn": 11, - "precision": 0.0003164009501836766, - "recall": 0.9941551540913921, - "far": 0.9996835990498163, - "csi": 0.00031640036161973984, - "f1": 0.0006326005681909433, - "f2": 0.0015799933489964635, - "frequency_bias": 3142.0738575982996, - "predicted_positive_rate": 0.7314017316017316 - }, - "0.2000": { - "threshold": 0.2, - "tp": 1790, - "fp": 4373008, - "fn": 92, - "precision": 0.00040916174872531255, - "recall": 0.9511158342189161, - "far": 0.9995908382512747, - "csi": 0.0004091531444219169, - "f1": 0.0008179716131862507, - "f2": 0.0020422944345080667, - "frequency_bias": 2324.547290116897, - "predicted_positive_rate": 0.5411005565862709 - }, - "0.3000": { - "threshold": 0.3, - "tp": 1725, - "fp": 3415319, - "fn": 157, - "precision": 0.00050482229669855, - "recall": 0.9165781083953242, - "far": 0.9994951777033014, - "csi": 0.0005047991031256283, - "f1": 0.0010090888191203904, - "f2": 0.002518562903627081, - "frequency_bias": 1815.6450584484592, - "predicted_positive_rate": 0.42263995052566483 - }, - "0.5000": { - "threshold": 0.5, - "tp": 1590, - "fp": 2187234, - "fn": 292, - "precision": 0.0007264174734926152, - "recall": 0.844845908607864, - "far": 0.9992735825265073, - "csi": 0.0007263205787176193, - "f1": 0.0014515868400415207, - "f2": 0.003619638382190104, - "frequency_bias": 1163.0308182784272, - "predicted_positive_rate": 0.2707265306122449 - }, - "0.7000": { - "threshold": 0.7, - "tp": 1441, - "fp": 1361655, - "fn": 441, - "precision": 0.0010571522475306214, - "recall": 0.7656748140276302, - "far": 0.9989428477524693, - "csi": 0.0010568103395800774, - "f1": 0.002111389341073629, - "f2": 0.005256729781471797, - "frequency_bias": 724.2805526036132, - "predicted_positive_rate": 0.168595670995671 - }, - "0.9000": { - "threshold": 0.9, - "tp": 1115, - "fp": 634236, - "fn": 767, - "precision": 0.001754935460871235, - "recall": 0.5924548352816154, - "far": 0.9982450645391288, - "csi": 0.0017528194454488003, - "f1": 0.003499504890675781, - "f2": 0.008671927376691415, - "frequency_bias": 337.59351753453774, - "predicted_positive_rate": 0.07858392084106369 - } - }, - "val_topk_area_metrics": { - "0.0100": { - "fraction": 0.01, - "cells": 80850, - "precision": 0.005293753929436207, - "recall": 0.22741764783859253, - "lift": 22.74176471853234 - }, - "0.0500": { - "fraction": 0.05, - "cells": 404250, - "precision": 0.002278293250128627, - "recall": 0.48937299847602844, - "lift": 9.787460797174546 - }, - "0.1000": { - "fraction": 0.1, - "cells": 808500, - "precision": 0.0015337043441832066, - "recall": 0.6588735580444336, - "lift": 6.588735292220149 - } - }, - "val_fss": { - "0.1000": { - "1": 0.0006518448132221133, - "2": 0.0006635725065556874, - "4": 0.0006826997042340688, - "8": 0.0007084853296767113 - }, - "0.2000": { - "1": 0.0008589763023596486, - "2": 0.0008854152202262267, - "4": 0.0009275047159825744, - "8": 0.000974054012801262 - }, - "0.3000": { - "1": 0.0010670370878855806, - "2": 0.0011091460766166072, - "4": 0.0011764363698075764, - "8": 0.0012380013792546718 - }, - "0.5000": { - "1": 0.001577796273116161, - "2": 0.001657621748606397, - "4": 0.001773829668805127, - "8": 0.0018645216828079292 - }, - "0.7000": { - "1": 0.002322673564303268, - "2": 0.0024379540533610022, - "4": 0.002617086020650139, - "8": 0.0027739584877713996 - }, - "0.9000": { - "1": 0.004049673487098726, - "2": 0.004396817960682808, - "4": 0.004958663276014907, - "8": 0.0055254589787200326 - } - }, - "minutes": 0.32502801418304444 - }, - { - "epoch": 5, - "train_loss": 0.4840681991702775, - "train_main_loss": 0.4566405527103148, - "train_aux_spatial_loss": 0.05485529401597806, - "val_loss": 0.32642485077182454, - "val_pr_auc": 0.00269643573270509, - "val_auroc": 0.8832613542277497, - "val_brier": 0.0865185409784317, - "val_brier_skill_score": -370.7635261100412, - "val_ece": 0.17086548094873133, - "val_positive_rate": 0.00023277674335986376, - "val_precision": 0.0014307592400306989, - "val_positive_recall": 0.6567481402763018, - "val_far": 0.9985692407599693, - "val_csi": 0.001429690129701581, - "val_f1": 0.0028552980679380753, - "val_f2": 0.007091995111343175, - "val_frequency_bias": 459.02072263549417, - "val_threshold_metrics": { - "0.1000": { - "threshold": 0.1, - "tp": 1748, - "fp": 3381937, - "fn": 134, - "precision": 0.0005165965508018625, - "recall": 0.9287991498405951, - "far": 0.9994834034491982, - "csi": 0.0005165760934612637, - "f1": 0.0010326187607570608, - "f2": 0.002577248907691732, - "frequency_bias": 1797.9197662061636, - "predicted_positive_rate": 0.4185139146567718 - }, - "0.2000": { - "threshold": 0.2, - "tp": 1614, - "fp": 2195701, - "fn": 268, - "precision": 0.0007345328275645504, - "recall": 0.8575982996811902, - "far": 0.9992654671724355, - "csi": 0.0007344432496975085, - "f1": 0.0014678084773669664, - "f2": 0.003660124553086093, - "frequency_bias": 1167.5425079702445, - "predicted_positive_rate": 0.27177674706246135 - }, - "0.3000": { - "threshold": 0.3, - "tp": 1505, - "fp": 1542529, - "fn": 377, - "precision": 0.0009747194686127378, - "recall": 0.7996811902231669, - "far": 0.9990252805313873, - "csi": 0.0009744815337368097, - "f1": 0.0019470656879157727, - "f2": 0.00484995121045759, - "frequency_bias": 820.4218916046759, - "predicted_positive_rate": 0.19097513914656772 - }, - "0.5000": { - "threshold": 0.5, - "tp": 1236, - "fp": 862641, - "fn": 646, - "precision": 0.0014307592400306989, - "recall": 0.6567481402763018, - "far": 0.9985692407599693, - "csi": 0.001429690129701581, - "f1": 0.0028552980679380753, - "f2": 0.007091995111343175, - "frequency_bias": 459.02072263549417, - "predicted_positive_rate": 0.10684935064935065 - }, - "0.7000": { - "threshold": 0.7, - "tp": 974, - "fp": 505344, - "fn": 908, - "precision": 0.0019236922250443398, - "recall": 0.5175345377258236, - "far": 0.9980763077749557, - "csi": 0.0019202485676996053, - "f1": 0.0038331365604092884, - "f2": 0.009477547747768785, - "frequency_bias": 269.03188097768333, - "predicted_positive_rate": 0.06262436611008039 - }, - "0.9000": { - "threshold": 0.9, - "tp": 767, - "fp": 250939, - "fn": 1115, - "precision": 0.003047205867162483, - "recall": 0.4075451647183847, - "far": 0.9969527941328375, - "csi": 0.0030337669734713493, - "f1": 0.006049182137956055, - "f2": 0.014793584174915327, - "frequency_bias": 133.74388947927736, - "predicted_positive_rate": 0.031132467532467532 - } - }, - "val_topk_area_metrics": { - "0.0100": { - "fraction": 0.01, - "cells": 80850, - "precision": 0.004836116451770067, - "recall": 0.20775771141052246, - "lift": 20.775771591123345 - }, - "0.0500": { - "fraction": 0.05, - "cells": 404250, - "precision": 0.002166975988075137, - "recall": 0.46546226739883423, - "lift": 9.309246090469944 - }, - "0.1000": { - "fraction": 0.1, - "cells": 808500, - "precision": 0.0014780457131564617, - "recall": 0.634962797164917, - "lift": 6.349627938867848 - } - }, - "val_fss": { - "0.1000": { - "1": 0.0010873373288891042, - "2": 0.001124730156876863, - "4": 0.0011842569881339582, - "8": 0.0012454325913199282 - }, - "0.2000": { - "1": 0.0015967200495251577, - "2": 0.001672856211459739, - "4": 0.00177710787777563, - "8": 0.0018681221664564873 - }, - "0.3000": { - "1": 0.0021362061921686992, - "2": 0.002243340057739074, - "4": 0.0024038793817035, - "8": 0.0025597638125007816 - }, - "0.5000": { - "1": 0.0032197830545794837, - "2": 0.0034434210744950233, - "4": 0.003829941863601527, - "8": 0.004228975808156621 - }, - "0.7000": { - "1": 0.004541360883536072, - "2": 0.004996111235622602, - "4": 0.005838071075877749, - "8": 0.0067937173415755225 - }, - "0.9000": { - "1": 0.0074909006005016154, - "2": 0.00836342260268863, - "4": 0.010405964121525169, - "8": 0.013064023833393557 - } - }, - "minutes": 0.3387693285942078 - }, - { - "epoch": 6, - "train_loss": 0.4704399423556344, - "train_main_loss": 0.4438283516374887, - "train_aux_spatial_loss": 0.05322318294850407, - "val_loss": 0.5198802714546521, - "val_pr_auc": 0.0020561227899118244, - "val_auroc": 0.8650460273834328, - "val_brier": 0.1523989737033844, - "val_brier_skill_score": -653.8466860258955, - "val_ece": 0.3041266358289659, - "val_positive_rate": 0.00023277674335986376, - "val_precision": 0.0009245181088592227, - "val_positive_recall": 0.793836344314559, - "val_far": 0.9990754818911408, - "val_csi": 0.0009242961831022078, - "val_f1": 0.0018468852971736105, - "val_f2": 0.004601156140572404, - "val_frequency_bias": 858.6487778958555, - "val_threshold_metrics": { - "0.1000": { - "threshold": 0.1, - "tp": 1849, - "fp": 6336772, - "fn": 33, - "precision": 0.00029170382643164814, - "recall": 0.9824654622741764, - "far": 0.9997082961735684, - "csi": 0.00029170230777701386, - "f1": 0.0005832344847088631, - "f2": 0.0014567889912449265, - "frequency_bias": 3368.0239107332627, - "predicted_positive_rate": 0.7839976499690785 - }, - "0.2000": { - "threshold": 0.2, - "tp": 1762, - "fp": 4527029, - "fn": 120, - "precision": 0.00038906630930859916, - "recall": 0.9362380446333688, - "far": 0.9996109336906914, - "csi": 0.0003890560004380744, - "f1": 0.0007778093894659802, - "f2": 0.0019421032780102105, - "frequency_bias": 2406.3714133900107, - "predicted_positive_rate": 0.5601473098330241 - }, - "0.3000": { - "threshold": 0.3, - "tp": 1682, - "fp": 3194137, - "fn": 200, - "precision": 0.0005263126603853348, - "recall": 0.8937300743889479, - "far": 0.9994736873396146, - "csi": 0.0005262797248702214, - "f1": 0.0010520058004172372, - "f2": 0.0026253790176337436, - "frequency_bias": 1698.0972369819342, - "predicted_positive_rate": 0.3952775510204082 - }, - "0.5000": { - "threshold": 0.5, - "tp": 1494, - "fp": 1614483, - "fn": 388, - "precision": 0.0009245181088592227, - "recall": 0.793836344314559, - "far": 0.9990754818911408, - "csi": 0.0009242961831022078, - "f1": 0.0018468852971736105, - "f2": 0.004601156140572404, - "frequency_bias": 858.6487778958555, - "predicted_positive_rate": 0.1998734693877551 - }, - "0.7000": { - "threshold": 0.7, - "tp": 1227, - "fp": 770825, - "fn": 655, - "precision": 0.0015892711889872702, - "recall": 0.6519659936238045, - "far": 0.9984107288110128, - "csi": 0.0015879240125946834, - "f1": 0.0031708130150633, - "f2": 0.00786962210420996, - "frequency_bias": 410.2295430393199, - "predicted_positive_rate": 0.09549189857761287 - }, - "0.9000": { - "threshold": 0.9, - "tp": 733, - "fp": 284579, - "fn": 1149, - "precision": 0.0025691173171825932, - "recall": 0.3894792773645058, - "far": 0.9974308826828174, - "csi": 0.0025588125434177775, - "f1": 0.005104563465810568, - "f2": 0.012515366753175796, - "frequency_bias": 151.60042507970243, - "predicted_positive_rate": 0.03528905380333952 - } - }, - "val_topk_area_metrics": { - "0.0100": { - "fraction": 0.01, - "cells": 80850, - "precision": 0.0032776747830212116, - "recall": 0.14080765843391418, - "lift": 14.080765697258915 - }, - "0.0500": { - "fraction": 0.05, - "cells": 404250, - "precision": 0.002169449580833316, - "recall": 0.46599361300468445, - "lift": 9.31987255049544 - }, - "0.1000": { - "fraction": 0.1, - "cells": 808500, - "precision": 0.00154483609367162, - "recall": 0.663655698299408, - "lift": 6.636556862913764 - } - }, - "val_fss": { - "0.1000": { - "1": 0.0006032450292200453, - "2": 0.0006159225446106253, - "4": 0.0006359492894152385, - "8": 0.0006629478573787484 - }, - "0.2000": { - "1": 0.0008224392220558263, - "2": 0.0008506113892607914, - "4": 0.0008909354914231615, - "8": 0.0009322944740023376 - }, - "0.3000": { - "1": 0.001132169119496873, - "2": 0.0011820837106975013, - "4": 0.0012528655821539836, - "8": 0.0013113866777990646 - }, - "0.5000": { - "1": 0.0020500064266273377, - "2": 0.0021609626362443235, - "4": 0.002330766029421505, - "8": 0.002448917281466567 - }, - "0.7000": { - "1": 0.0035578224345581555, - "2": 0.0038334909485748803, - "4": 0.004359088724627869, - "8": 0.004831371146690966 - }, - "0.9000": { - "1": 0.00637013151022392, - "2": 0.007172450990244439, - "4": 0.009163883225476899, - "8": 0.011610480962539693 - } - }, - "minutes": 0.32045116027196247 - }, - { - "epoch": 7, - "train_loss": 0.45992261002683726, - "train_main_loss": 0.4340675990611428, - "train_aux_spatial_loss": 0.05171002073115929, - "val_loss": 0.3403168035050233, - "val_pr_auc": 0.0019248177157325489, - "val_auroc": 0.8516727976804799, - "val_brier": 0.09707184135913849, - "val_brier_skill_score": -416.11024737072506, - "val_ece": 0.20351086646269692, - "val_positive_rate": 0.00023277674335986376, - "val_precision": 0.0011260303225245256, - "val_positive_recall": 0.6259298618490967, - "val_far": 0.9988739696774754, - "val_csi": 0.0011252730793221996, - "val_f1": 0.002248016526165634, - "val_f2": 0.005589927122155567, - "val_frequency_bias": 555.8730074388948, - "val_threshold_metrics": { - "0.1000": { - "threshold": 0.1, - "tp": 1753, - "fp": 4223621, - "fn": 129, - "precision": 0.00041487451761666543, - "recall": 0.9314558979808714, - "far": 0.9995851254823833, - "csi": 0.00041486185194993354, - "f1": 0.0008293796259322832, - "f2": 0.0020706834223896517, - "frequency_bias": 2245.1509032943677, - "predicted_positive_rate": 0.5226189239332096 - }, - "0.2000": { - "threshold": 0.2, - "tp": 1653, - "fp": 2706333, - "fn": 229, - "precision": 0.0006104167451382688, - "recall": 0.8783209351753454, - "far": 0.9993895832548617, - "csi": 0.0006103651297995174, - "f1": 0.00121998562291595, - "f2": 0.003043622680641676, - "frequency_bias": 1438.8873538788523, - "predicted_positive_rate": 0.3349395176252319 - }, - "0.3000": { - "threshold": 0.3, - "tp": 1499, - "fp": 1884843, - "fn": 383, - "precision": 0.0007946597170608511, - "recall": 0.7964930924548352, - "far": 0.9992053402829392, - "csi": 0.0007944984033179186, - "f1": 0.0015877353534326437, - "f2": 0.003957505002983309, - "frequency_bias": 1002.307120085016, - "predicted_positive_rate": 0.23331379097093383 - }, - "0.5000": { - "threshold": 0.5, - "tp": 1178, - "fp": 1044975, - "fn": 704, - "precision": 0.0011260303225245256, - "recall": 0.6259298618490967, - "far": 0.9988739696774754, - "csi": 0.0011252730793221996, - "f1": 0.002248016526165634, - "f2": 0.005589927122155567, - "frequency_bias": 555.8730074388948, - "predicted_positive_rate": 0.1293943104514533 - }, - "0.7000": { - "threshold": 0.7, - "tp": 819, - "fp": 535799, - "fn": 1063, - "precision": 0.0015262253595667683, - "recall": 0.4351753453772582, - "far": 0.9984737746404332, - "csi": 0.0015232079987948244, - "f1": 0.0030417827298050144, - "f2": 0.007525553803574776, - "frequency_bias": 285.1317747077577, - "predicted_positive_rate": 0.06637204700061843 - }, - "0.9000": { - "threshold": 0.9, - "tp": 476, - "fp": 160341, - "fn": 1406, - "precision": 0.0029598860816953433, - "recall": 0.25292242295430395, - "far": 0.9970401139183046, - "csi": 0.002934232507104418, - "f1": 0.005851295951419493, - "f2": 0.01413763402536458, - "frequency_bias": 85.4500531349628, - "predicted_positive_rate": 0.01989078540507112 - } - }, - "val_topk_area_metrics": { - "0.0100": { - "fraction": 0.01, - "cells": 80850, - "precision": 0.0037847866769880056, - "recall": 0.16259299218654633, - "lift": 16.259299027724918 - }, - "0.0500": { - "fraction": 0.05, - "cells": 404250, - "precision": 0.0017884972039610147, - "recall": 0.38416579365730286, - "lift": 7.6833156875816755 - }, - "0.1000": { - "fraction": 0.1, - "cells": 808500, - "precision": 0.001277674688026309, - "recall": 0.5488841533660889, - "lift": 5.488841666845874 - } - }, - "val_fss": { - "0.1000": { - "1": 0.0008760009616006759, - "2": 0.0009054218131856473, - "4": 0.0009476146471629754, - "8": 0.0009927278131620643 - }, - "0.2000": { - "1": 0.001305241106265398, - "2": 0.001353720436246486, - "4": 0.0014217692397042736, - "8": 0.0014697761316361246 - }, - "0.3000": { - "1": 0.0017153860355670902, - "2": 0.00177860122228235, - "4": 0.001884522312124659, - "8": 0.0019437440305635212 - }, - "0.5000": { - "1": 0.0024635323345161275, - "2": 0.0025797735042438763, - "4": 0.0028109999434535737, - "8": 0.002947214909635787 - }, - "0.7000": { - "1": 0.003567303280486267, - "2": 0.0038553882148504304, - "4": 0.004379270130838386, - "8": 0.004750678685008225 - }, - "0.9000": { - "1": 0.007761321421334344, - "2": 0.009021772361332503, - "4": 0.011815395230424186, - "8": 0.015149874578981093 - } - }, - "minutes": 0.32094937562942505 - }, - { - "epoch": 8, - "train_loss": 0.4501411836177224, - "train_main_loss": 0.42512462227910447, - "train_aux_spatial_loss": 0.0500331219766449, - "val_loss": 0.2779005537430445, - "val_pr_auc": 0.0033511312666555885, - "val_auroc": 0.8727357646440189, - "val_brier": 0.07455155998468399, - "val_brier_skill_score": -319.34232782334607, - "val_ece": 0.1583937377225246, - "val_positive_rate": 0.00023277674335986376, - "val_precision": 0.0015191347299669793, - "val_positive_recall": 0.6211477151965994, - "val_far": 0.998480865270033, - "val_csi": 0.0015177284707165392, - "val_f1": 0.0030308569235894785, - "val_f2": 0.0075220868804252005, - "val_frequency_bias": 408.8825717321998, - "val_threshold_metrics": { - "0.1000": { - "threshold": 0.1, - "tp": 1718, - "fp": 3294835, - "fn": 164, - "precision": 0.0005211504259145842, - "recall": 0.9128586609989373, - "far": 0.9994788495740854, - "csi": 0.0005211245005258262, - "f1": 0.001041706142458469, - "f2": 0.0025998151982351524, - "frequency_bias": 1751.6222104144526, - "predicted_positive_rate": 0.4077369202226345 - }, - "0.2000": { - "threshold": 0.2, - "tp": 1536, - "fp": 1918537, - "fn": 346, - "precision": 0.0007999695844897564, - "recall": 0.8161530286928799, - "far": 0.9992000304155102, - "csi": 0.0007998254547575295, - "f1": 0.0015983724905109641, - "f2": 0.00398422702623624, - "frequency_bias": 1020.230074388948, - "predicted_positive_rate": 0.23748583797155226 - }, - "0.3000": { - "threshold": 0.3, - "tp": 1408, - "fp": 1333909, - "fn": 474, - "precision": 0.0010544312698782387, - "recall": 0.7481402763018066, - "far": 0.9989455687301217, - "csi": 0.0010540571092334055, - "f1": 0.002105894485413166, - "f2": 0.005242600597984131, - "frequency_bias": 709.5201912858661, - "predicted_positive_rate": 0.16515980210265924 - }, - "0.5000": { - "threshold": 0.5, - "tp": 1169, - "fp": 768348, - "fn": 713, - "precision": 0.0015191347299669793, - "recall": 0.6211477151965994, - "far": 0.998480865270033, - "csi": 0.0015177284707165392, - "f1": 0.0030308569235894785, - "f2": 0.0075220868804252005, - "frequency_bias": 408.8825717321998, - "predicted_positive_rate": 0.09517835497835497 - }, - "0.7000": { - "threshold": 0.7, - "tp": 867, - "fp": 440719, - "fn": 1015, - "precision": 0.0019633774621478036, - "recall": 0.46068012752391074, - "far": 0.9980366225378522, - "csi": 0.001958874923463797, - "f1": 0.003910090468759866, - "f2": 0.009652337713809855, - "frequency_bias": 234.6365568544102, - "predicted_positive_rate": 0.054617934446505874 - }, - "0.9000": { - "threshold": 0.9, - "tp": 599, - "fp": 158162, - "fn": 1283, - "precision": 0.0037729669125288958, - "recall": 0.31827842720510097, - "far": 0.9962270330874711, - "csi": 0.0037427207517932567, - "f1": 0.007457530051107114, - "f2": 0.018010812501127557, - "frequency_bias": 84.35759829968119, - "predicted_positive_rate": 0.019636487322201607 - } - }, - "val_topk_area_metrics": { - "0.0100": { - "fraction": 0.01, - "cells": 80850, - "precision": 0.005343228112906218, - "recall": 0.2295430451631546, - "lift": 22.954303921357795 - }, - "0.0500": { - "fraction": 0.05, - "cells": 404250, - "precision": 0.002095238072797656, - "recall": 0.4500531256198883, - "lift": 9.001062746025704 - }, - "0.1000": { - "fraction": 0.1, - "cells": 808500, - "precision": 0.0014743352076038718, - "recall": 0.633368730545044, - "lift": 6.333687748713827 - } - }, - "val_fss": { - "0.1000": { - "1": 0.0011050920489504845, - "2": 0.0011474319184742177, - "4": 0.001207237050294463, - "8": 0.001263614219587872 - }, - "0.2000": { - "1": 0.0017407768081587616, - "2": 0.0018298937606811005, - "4": 0.001941458698715981, - "8": 0.0019852967362636864 - }, - "0.3000": { - "1": 0.00230278489686786, - "2": 0.0024155683935852545, - "4": 0.002561184010625994, - "8": 0.0025929800638113676 - }, - "0.5000": { - "1": 0.0034017699482070674, - "2": 0.003590432517582065, - "4": 0.0038659187657282823, - "8": 0.003947451170566385 - }, - "0.7000": { - "1": 0.004745738291732748, - "2": 0.005164124226629263, - "4": 0.005773230761917714, - "8": 0.006170780849613977 - }, - "0.9000": { - "1": 0.009755112945959765, - "2": 0.011279501877038078, - "4": 0.014849528591105998, - "8": 0.020210872188599516 - } - }, - "minutes": 0.3122890591621399 - }, - { - "epoch": 9, - "train_loss": 0.4357784079228708, - "train_main_loss": 0.41138888008416336, - "train_aux_spatial_loss": 0.04877905696682909, - "val_loss": 0.2466988556087017, - "val_pr_auc": 0.0024715249735166854, - "val_auroc": 0.8593465923841782, - "val_brier": 0.06473856419324875, - "val_brier_skill_score": -277.17663852865024, - "val_ece": 0.14665763289695394, - "val_positive_rate": 0.00023277674335986376, - "val_precision": 0.0013977547551646229, - "val_positive_recall": 0.5042507970244421, - "val_far": 0.9986022452448354, - "val_csi": 0.001395836612103036, - "val_f1": 0.0027877819361130864, - "val_f2": 0.0069121335986504955, - "val_frequency_bias": 360.7577045696068, - "val_threshold_metrics": { - "0.1000": { - "threshold": 0.1, - "tp": 1671, - "fp": 3056649, - "fn": 211, - "precision": 0.0005463784038295535, - "recall": 0.88788522848034, - "far": 0.9994536215961705, - "csi": 0.0005463407106221908, - "f1": 0.0010920847708746023, - "f2": 0.002725184027388181, - "frequency_bias": 1625.0371944739638, - "predicted_positive_rate": 0.3782708719851577 - }, - "0.2000": { - "threshold": 0.2, - "tp": 1469, - "fp": 1790064, - "fn": 413, - "precision": 0.000819968150181995, - "recall": 0.7805526036131775, - "far": 0.999180031849818, - "csi": 0.0008197791674525906, - "f1": 0.001638215360081186, - "f2": 0.004082685356416487, - "frequency_bias": 951.9303931987248, - "predicted_positive_rate": 0.22158726035868892 - }, - "0.3000": { - "threshold": 0.3, - "tp": 1278, - "fp": 1237670, - "fn": 604, - "precision": 0.0010315202897942448, - "recall": 0.6790648246546227, - "far": 0.9989684797102057, - "csi": 0.0010310176579925651, - "f1": 0.0020599115108435482, - "f2": 0.005126452494873548, - "frequency_bias": 658.3145589798087, - "predicted_positive_rate": 0.1532403215831787 - }, - "0.5000": { - "threshold": 0.5, - "tp": 949, - "fp": 677997, - "fn": 933, - "precision": 0.0013977547551646229, - "recall": 0.5042507970244421, - "far": 0.9986022452448354, - "csi": 0.001395836612103036, - "f1": 0.0027877819361130864, - "f2": 0.0069121335986504955, - "frequency_bias": 360.7577045696068, - "predicted_positive_rate": 0.08397600494743351 - }, - "0.7000": { - "threshold": 0.7, - "tp": 780, - "fp": 337462, - "fn": 1102, - "precision": 0.002306041236747654, - "recall": 0.41445270988310307, - "far": 0.9976939587632524, - "csi": 0.0022985525012966194, - "f1": 0.004586562547776693, - "f2": 0.011279174017410416, - "frequency_bias": 179.72476089266738, - "predicted_positive_rate": 0.04183574520717378 - }, - "0.9000": { - "threshold": 0.9, - "tp": 575, - "fp": 126493, - "fn": 1307, - "precision": 0.004525136147574527, - "recall": 0.3055260361317747, - "far": 0.9954748638524255, - "csi": 0.0044790652385589096, - "f1": 0.008918185343156262, - "f2": 0.021360218728639784, - "frequency_bias": 67.51753453772582, - "predicted_positive_rate": 0.015716512059369202 - } - }, - "val_topk_area_metrics": { - "0.0100": { - "fraction": 0.01, - "cells": 80850, - "precision": 0.005256648175418377, - "recall": 0.22582359611988068, - "lift": 22.582359816297473 - }, - "0.0500": { - "fraction": 0.05, - "cells": 404250, - "precision": 0.002011131728067994, - "recall": 0.4319872558116913, - "lift": 8.63974510099088 - }, - "0.1000": { - "fraction": 0.1, - "cells": 808500, - "precision": 0.0012838590191677213, - "recall": 0.5515409111976624, - "lift": 5.515409317256945 - } - }, - "val_fss": { - "0.1000": { - "1": 0.0011651445757413503, - "2": 0.0012111852278434698, - "4": 0.0012755769277128515, - "8": 0.001326139591474207 - }, - "0.2000": { - "1": 0.0017847015451787351, - "2": 0.0018660988212818586, - "4": 0.001969892674721363, - "8": 0.0020256766859216846 - }, - "0.3000": { - "1": 0.0022836423061386446, - "2": 0.0024096079587074826, - "4": 0.0025866385088719746, - "8": 0.002677494067034991 - }, - "0.5000": { - "1": 0.003257597509393695, - "2": 0.0035462700407361547, - "4": 0.003995590535502691, - "8": 0.0042497358267622465 - }, - "0.7000": { - "1": 0.0056228500540735915, - "2": 0.006273475118964256, - "4": 0.00754737401459038, - "8": 0.008762055377144917 - }, - "0.9000": { - "1": 0.01119749514237256, - "2": 0.012904909248986685, - "4": 0.017159283576349882, - "8": 0.023161111656246902 - } - }, - "minutes": 0.3215821862220764 - }, - { - "epoch": 10, - "train_loss": 0.43129921461825527, - "train_main_loss": 0.4073479366069134, - "train_aux_spatial_loss": 0.04790255693317092, - "val_loss": 0.31038872922460237, - "val_pr_auc": 0.003182743786360499, - "val_auroc": 0.8601447316055663, - "val_brier": 0.08384887129068375, - "val_brier_skill_score": -359.29216048780205, - "val_ece": 0.17778525591728558, - "val_positive_rate": 0.00023277674335986376, - "val_precision": 0.0013001670051705871, - "val_positive_recall": 0.6200850159404888, - "val_far": 0.9986998329948295, - "val_csi": 0.0012991321307548103, - "val_f1": 0.002594893152439411, - "val_f2": 0.006446765844846732, - "val_frequency_bias": 476.9272051009564, - "val_threshold_metrics": { - "0.1000": { - "threshold": 0.1, - "tp": 1708, - "fp": 3565572, - "fn": 174, - "precision": 0.00047879616963064295, - "recall": 0.9075451647183846, - "far": 0.9995212038303694, - "csi": 0.00047877281669224045, - "f1": 0.0009570874059513128, - "f2": 0.00238893948989708, - "frequency_bias": 1895.4729011689692, - "predicted_positive_rate": 0.44122201607915895 - }, - "0.2000": { - "threshold": 0.2, - "tp": 1545, - "fp": 2195984, - "fn": 337, - "precision": 0.0007030623941709075, - "recall": 0.8209351753453773, - "far": 0.999296937605829, - "csi": 0.0007029545932281585, - "f1": 0.0014049215903712403, - "f2": 0.0035033107987684675, - "frequency_bias": 1167.6562167906482, - "predicted_positive_rate": 0.27180321583178724 - }, - "0.3000": { - "threshold": 0.3, - "tp": 1434, - "fp": 1603689, - "fn": 448, - "precision": 0.000893389478563325, - "recall": 0.7619553666312433, - "far": 0.9991066105214367, - "csi": 0.000893140197474917, - "f1": 0.001784686419768451, - "f2": 0.004446095280379946, - "frequency_bias": 852.8815090329437, - "predicted_positive_rate": 0.19853098330241187 - }, - "0.5000": { - "threshold": 0.5, - "tp": 1167, - "fp": 896410, - "fn": 715, - "precision": 0.0013001670051705871, - "recall": 0.6200850159404888, - "far": 0.9986998329948295, - "csi": 0.0012991321307548103, - "f1": 0.002594893152439411, - "f2": 0.006446765844846732, - "frequency_bias": 476.9272051009564, - "predicted_positive_rate": 0.11101756338899196 - }, - "0.7000": { - "threshold": 0.7, - "tp": 924, - "fp": 456494, - "fn": 958, - "precision": 0.0020200341919207375, - "recall": 0.4909670563230606, - "far": 0.9979799658080792, - "csi": 0.002015812346196136, - "f1": 0.0040235140431090785, - "f2": 0.009936637803099713, - "frequency_bias": 243.0488841657811, - "predicted_positive_rate": 0.05657612863327149 - }, - "0.9000": { - "threshold": 0.9, - "tp": 734, - "fp": 175677, - "fn": 1148, - "precision": 0.004160738275957848, - "recall": 0.3900106269925611, - "far": 0.9958392617240421, - "csi": 0.004133837203408444, - "f1": 0.008233637888195275, - "f2": 0.0199522667840969, - "frequency_bias": 93.73591923485654, - "predicted_positive_rate": 0.021819542362399504 - } - }, - "val_topk_area_metrics": { - "0.0100": { - "fraction": 0.01, - "cells": 80850, - "precision": 0.006431663408875465, - "recall": 0.2763018012046814, - "lift": 27.630180386759534 - }, - "0.0500": { - "fraction": 0.05, - "cells": 404250, - "precision": 0.002209029160439968, - "recall": 0.4744952321052551, - "lift": 9.489904912987356 - }, - "0.1000": { - "fraction": 0.1, - "cells": 808500, - "precision": 0.0013927025720477104, - "recall": 0.5982996821403503, - "lift": 5.9829970638202745 - } - }, - "val_fss": { - "0.1000": { - "1": 0.001019473269986193, - "2": 0.0010557422939804217, - "4": 0.0011052621270040142, - "8": 0.0011453423660473527 - }, - "0.2000": { - "1": 0.0015099904182619417, - "2": 0.0015686609647825867, - "4": 0.0016378638383718336, - "8": 0.0016768349605363886 - }, - "0.3000": { - "1": 0.0019186672475260957, - "2": 0.0019898217973708388, - "4": 0.002076219424972159, - "8": 0.0021161819887003386 - }, - "0.5000": { - "1": 0.002904085129755374, - "2": 0.0030756505922767685, - "4": 0.0033142291130332513, - "8": 0.003457418818581859 - }, - "0.7000": { - "1": 0.004749695098450801, - "2": 0.005118148709556514, - "4": 0.005725380054942164, - "8": 0.006258756516320685 - }, - "0.9000": { - "1": 0.010015907694898685, - "2": 0.011072008375778264, - "4": 0.013502674103421874, - "8": 0.017073185913380362 - } - }, - "minutes": 0.31329720417658485 - } -] diff --git a/models/wildfire_fm/metrics/reference_fireprone_seed_7_20260504/run_summary.json b/models/wildfire_fm/metrics/reference_fireprone_seed_7_20260504/run_summary.json deleted file mode 100644 index 5b3e2fd02cb4fc1889b50952fb25ac63d69605b4..0000000000000000000000000000000000000000 --- a/models/wildfire_fm/metrics/reference_fireprone_seed_7_20260504/run_summary.json +++ /dev/null @@ -1,172 +0,0 @@ -{ - "run_name": "reference_fireprone_seed_7_20260504", - "device": "cuda", - "index_root": "", - "input_channels": 16, - "tile_size": 32, - "num_train_tiles": 27177, - "positive_train_tiles": 6724, - "negative_train_tiles": 20453, - "train_positive_rate": 0.0018423063389998895, - "metric_reference_positive_rate": 0.0002787771234976204, - "pos_weight": 50.0, - "use_aux_spatial_head": true, - "aux_spatial_radius": 2, - "aux_spatial_loss_weight": 0.5, - "aux_positive_rate": 0.004560296846011132, - "init_checkpoint": "", - "best_val_pr_auc": 0.0033511312666555885, - "best_val_auroc": 0.8727357646440189, - "best_test_pr_auc": 0.0016046388753321321, - "best_test_auroc": 0.8223907600143328, - "best_test_brier": 0.09998881816864014, - "best_test_ece": 0.19384167839217642, - "best_test_topk_area_metrics": { - "0.0100": { - "fraction": 0.01, - "cells": 80850, - "precision": 0.003426097799092531, - "recall": 0.1412544548511505, - "lift": 14.125446318034083 - }, - "0.0500": { - "fraction": 0.05, - "cells": 404250, - "precision": 0.0017266542417928576, - "recall": 0.3559408485889435, - "lift": 7.11881657572967 - }, - "0.1000": { - "fraction": 0.1, - "cells": 808500, - "precision": 0.0011168831260874867, - "recall": 0.46047934889793396, - "lift": 4.60479343153765 - } - }, - "best_test_threshold_metrics": { - "0.1000": { - "threshold": 0.1, - "tp": 1768, - "fp": 3684053, - "fn": 193, - "precision": 0.000479676034186142, - "recall": 0.901580826109128, - "far": 0.9995203239658138, - "csi": 0.000479650918309046, - "f1": 0.0009588419272071939, - "f2": 0.0023932868844359196, - "frequency_bias": 1879.5619581845997, - "predicted_positive_rate": 0.4558838589981447 - }, - "0.2000": { - "threshold": 0.2, - "tp": 1571, - "fp": 2394479, - "fn": 390, - "precision": 0.0006556624444398072, - "recall": 0.8011218765935747, - "far": 0.9993443375555602, - "csi": 0.0006555557410158402, - "f1": 0.0013102525384579137, - "f2": 0.0032676149613918085, - "frequency_bias": 1221.8510963793983, - "predicted_positive_rate": 0.2963574520717378 - }, - "0.3000": { - "threshold": 0.3, - "tp": 1390, - "fp": 1792642, - "fn": 571, - "precision": 0.0007747910851088498, - "recall": 0.7088220295767466, - "far": 0.9992252089148912, - "csi": 0.0007745445650096428, - "f1": 0.0015478902200621052, - "f2": 0.0038570911649858258, - "frequency_bias": 914.8556858745538, - "predicted_positive_rate": 0.22189635126777985 - }, - "0.5000": { - "threshold": 0.5, - "tp": 1063, - "fp": 1120631, - "fn": 898, - "precision": 0.0009476737862554315, - "recall": 0.5420703722590515, - "far": 0.9990523262137446, - "csi": 0.0009469157093583421, - "f1": 0.001892039816491717, - "f2": 0.0047054636497399814, - "frequency_bias": 572.0010198878124, - "predicted_positive_rate": 0.13873766233766233 - }, - "0.7000": { - "threshold": 0.7, - "tp": 807, - "fp": 603572, - "fn": 1154, - "precision": 0.0013352548649109251, - "recall": 0.41152473227944925, - "far": 0.9986647451350891, - "csi": 0.0013327101908566503, - "f1": 0.002661872876603886, - "f2": 0.006590735728647894, - "frequency_bias": 308.1993880673126, - "predicted_positive_rate": 0.07475312306740878 - }, - "0.9000": { - "threshold": 0.9, - "tp": 547, - "fp": 202779, - "fn": 1414, - "precision": 0.0026902609602313526, - "recall": 0.27893931667516575, - "far": 0.9973097390397686, - "csi": 0.0026716811565888443, - "f1": 0.005329124591425663, - "f2": 0.01295165032911872, - "frequency_bias": 103.68485466598675, - "predicted_positive_rate": 0.025148546691403834 - } - }, - "best_test_fss": { - "0.1000": { - "1": 0.001006653180254391, - "2": 0.001036172017983894, - "4": 0.0010764843206666486, - "8": 0.0011088189513154267 - }, - "0.2000": { - "1": 0.0013931809574536658, - "2": 0.0014436935674144724, - "4": 0.0015037854705144404, - "8": 0.0015046820466139144 - }, - "0.3000": { - "1": 0.0016631737390239154, - "2": 0.0017265932312411358, - "4": 0.0017994774155269955, - "8": 0.0017754472066875415 - }, - "0.5000": { - "1": 0.00209939111569335, - "2": 0.002211811539862274, - "4": 0.0023188574757858627, - "8": 0.0022717093175803615 - }, - "0.7000": { - "1": 0.0031227648502994976, - "2": 0.0033730806856239193, - "4": 0.003653859775083146, - "8": 0.003690592345308774 - }, - "0.9000": { - "1": 0.0066600329138472025, - "2": 0.0075581235730202145, - "4": 0.009383123791337655, - "8": 0.011475141202696215 - } - }, - "epochs": 10 -} diff --git a/models/wildfire_fm/metrics/reference_fireprone_seed_7_20260504/tile_summary.json b/models/wildfire_fm/metrics/reference_fireprone_seed_7_20260504/tile_summary.json deleted file mode 100644 index 7ef825f81c3bcef8a18cffe05b9d41285027d3b1..0000000000000000000000000000000000000000 --- a/models/wildfire_fm/metrics/reference_fireprone_seed_7_20260504/tile_summary.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "index_root": "", - "tile_size": 32, - "num_train_tiles": 27177, - "positive_train_tiles": 6724, - "negative_train_tiles": 20453, - "input_channels": 16, - "train_positive_rate": 0.0018423063389998895, - "metric_reference_positive_rate": 0.0002787771234976204, - "pos_weight_source": "tiles", - "positive_rate_source": "tiles", - "pos_weight": 50.0, - "loss_type": "bce", - "train_target_mode": "dilate_max", - "train_target_radius": 2 -} diff --git a/models/wildfire_fm/metrics/reference_fireprone_seed_99_20260504/history.json b/models/wildfire_fm/metrics/reference_fireprone_seed_99_20260504/history.json deleted file mode 100644 index 5a495399872b53c3de3f81fcacef0a49636ca48d..0000000000000000000000000000000000000000 --- a/models/wildfire_fm/metrics/reference_fireprone_seed_99_20260504/history.json +++ /dev/null @@ -1,1682 +0,0 @@ -[ - { - "epoch": 1, - "train_loss": 0.5778533777167002, - "train_main_loss": 0.5341317258517807, - "train_aux_spatial_loss": 0.08744330333986021, - "val_loss": 0.5268240878979366, - "val_pr_auc": 0.001801559151297299, - "val_auroc": 0.8300020902264806, - "val_brier": 0.16021513938903809, - "val_brier_skill_score": -687.4321497091457, - "val_ece": 0.3149934093473418, - "val_positive_rate": 0.00023277674335986376, - "val_precision": 0.0007911901010445063, - "val_positive_recall": 0.7454835281615303, - "val_far": 0.9992088098989554, - "val_csi": 0.0007909764415306043, - "val_f1": 0.0015807025845557582, - "val_f2": 0.003939227518325972, - "val_frequency_bias": 942.230605738576, - "val_threshold_metrics": { - "0.1000": { - "threshold": 0.1, - "tp": 1861, - "fp": 6674626, - "fn": 21, - "precision": 0.00027873940292252495, - "recall": 0.9888416578108395, - "far": 0.9997212605970774, - "csi": 0.0002787385261876418, - "f1": 0.0005573217053445235, - "f2": 0.0013921273366382329, - "frequency_bias": 3547.548884165781, - "predicted_positive_rate": 0.825786889301175 - }, - "0.2000": { - "threshold": 0.2, - "tp": 1722, - "fp": 4433958, - "fn": 160, - "precision": 0.0003882155610864625, - "recall": 0.9149840595111584, - "far": 0.9996117844389135, - "csi": 0.0003882015582167075, - "f1": 0.0007761018324927066, - "f2": 0.0019377890929256523, - "frequency_bias": 2356.8969181721573, - "predicted_positive_rate": 0.5486307977736549 - }, - "0.3000": { - "threshold": 0.3, - "tp": 1631, - "fp": 3209108, - "fn": 251, - "precision": 0.0005079827416678839, - "recall": 0.8666312433581297, - "far": 0.9994920172583321, - "csi": 0.0005079430331455407, - "f1": 0.001015370316013, - "f2": 0.0025339724764912295, - "frequency_bias": 1706.0249734325187, - "predicted_positive_rate": 0.3971229437229437 - }, - "0.5000": { - "threshold": 0.5, - "tp": 1403, - "fp": 1771875, - "fn": 479, - "precision": 0.0007911901010445063, - "recall": 0.7454835281615303, - "far": 0.9992088098989554, - "csi": 0.0007909764415306043, - "f1": 0.0015807025845557582, - "f2": 0.003939227518325972, - "frequency_bias": 942.230605738576, - "predicted_positive_rate": 0.21932937538651825 - }, - "0.7000": { - "threshold": 0.7, - "tp": 1007, - "fp": 873767, - "fn": 875, - "precision": 0.0011511544696115796, - "recall": 0.5350690754516472, - "far": 0.9988488455303884, - "csi": 0.0011500041683368564, - "f1": 0.0022973663557883596, - "f2": 0.0057066627979988705, - "frequency_bias": 464.81083953241233, - "predicted_positive_rate": 0.10819715522572665 - }, - "0.9000": { - "threshold": 0.9, - "tp": 585, - "fp": 243959, - "fn": 1297, - "precision": 0.002392207537293902, - "recall": 0.3108395324123273, - "far": 0.9976077924627061, - "csi": 0.002379586806106386, - "f1": 0.004747875630006573, - "f2": 0.011603827477863468, - "frequency_bias": 129.9383634431456, - "predicted_positive_rate": 0.030246629560915277 - } - }, - "val_topk_area_metrics": { - "0.0100": { - "fraction": 0.01, - "cells": 80850, - "precision": 0.0034137291368097067, - "recall": 0.146652489900589, - "lift": 14.665250005376244 - }, - "0.0500": { - "fraction": 0.05, - "cells": 404250, - "precision": 0.00172912806738168, - "recall": 0.37141337990760803, - "lift": 7.4282681440753535 - }, - "0.1000": { - "fraction": 0.1, - "cells": 808500, - "precision": 0.0011861472157761455, - "recall": 0.5095642805099487, - "lift": 5.095643141387231 - } - }, - "val_fss": { - "0.1000": { - "1": 0.0005767564103924538, - "2": 0.0005890134794819257, - "4": 0.0006078227289899596, - "8": 0.0006326282137129047 - }, - "0.2000": { - "1": 0.00081986363926001, - "2": 0.0008498172851243435, - "4": 0.0008980338142677846, - "8": 0.0009442365839407163 - }, - "0.3000": { - "1": 0.001086040351521933, - "2": 0.0011281453218618953, - "4": 0.0011947800946939813, - "8": 0.0012519072968685263 - }, - "0.5000": { - "1": 0.001722969678203623, - "2": 0.0017971917246983882, - "4": 0.001911957912350437, - "8": 0.0020062945625652917 - }, - "0.7000": { - "1": 0.002627765306140284, - "2": 0.0028429364479983787, - "4": 0.0031580131392605626, - "8": 0.003416834699769833 - }, - "0.9000": { - "1": 0.005918451186751605, - "2": 0.006697606371421805, - "4": 0.00873029919299595, - "8": 0.01141916176600577 - } - }, - "minutes": 0.4064649740854899 - }, - { - "epoch": 2, - "train_loss": 0.5286303263821187, - "train_main_loss": 0.49828399360398123, - "train_aux_spatial_loss": 0.06069266609487724, - "val_loss": 0.2564812108874321, - "val_pr_auc": 0.0016945588134657274, - "val_auroc": 0.8576418217604199, - "val_brier": 0.06716381013393402, - "val_brier_skill_score": -287.597734081082, - "val_ece": 0.15346273963817214, - "val_positive_rate": 0.00023277674335986376, - "val_precision": 0.0013487785097515384, - "val_positive_recall": 0.4957492029755579, - "val_far": 0.9986512214902484, - "val_csi": 0.0013469306438992558, - "val_f1": 0.0026902377241684555, - "val_f2": 0.006671290569383565, - "val_frequency_bias": 367.55419766206165, - "val_threshold_metrics": { - "0.1000": { - "threshold": 0.1, - "tp": 1706, - "fp": 3331373, - "fn": 176, - "precision": 0.0005118390533197683, - "recall": 0.9064824654622742, - "far": 0.9994881609466802, - "csi": 0.0005118120275826482, - "f1": 0.0010231004200648825, - "f2": 0.0025534281644024577, - "frequency_bias": 1771.0302869287991, - "predicted_positive_rate": 0.41225466914038345 - }, - "0.2000": { - "threshold": 0.2, - "tp": 1510, - "fp": 1809417, - "fn": 372, - "precision": 0.0008338270951838479, - "recall": 0.8023379383634431, - "far": 0.9991661729048161, - "csi": 0.0008336558458873991, - "f1": 0.001665922885422568, - "f2": 0.004151876180603864, - "frequency_bias": 962.2353878852285, - "predicted_positive_rate": 0.22398602350030922 - }, - "0.3000": { - "threshold": 0.3, - "tp": 1315, - "fp": 1226566, - "fn": 567, - "precision": 0.0010709506865893357, - "recall": 0.6987247608926673, - "far": 0.9989290493134106, - "csi": 0.0010704563807340645, - "f1": 0.0021386234583411603, - "f2": 0.005322124090078671, - "frequency_bias": 652.4341126461211, - "predicted_positive_rate": 0.15187149041434755 - }, - "0.5000": { - "threshold": 0.5, - "tp": 933, - "fp": 690804, - "fn": 949, - "precision": 0.0013487785097515384, - "recall": 0.4957492029755579, - "far": 0.9986512214902484, - "csi": 0.0013469306438992558, - "f1": 0.0026902377241684555, - "f2": 0.006671290569383565, - "frequency_bias": 367.55419766206165, - "predicted_positive_rate": 0.08555807050092765 - }, - "0.7000": { - "threshold": 0.7, - "tp": 701, - "fp": 351788, - "fn": 1181, - "precision": 0.0019887145414466834, - "recall": 0.37247608926673753, - "far": 0.9980112854585533, - "csi": 0.001982073684508157, - "f1": 0.003956305679640828, - "f2": 0.009735651372018544, - "frequency_bias": 187.29489904357067, - "predicted_positive_rate": 0.043597897340754484 - }, - "0.9000": { - "threshold": 0.9, - "tp": 356, - "fp": 142050, - "fn": 1526, - "precision": 0.0024998946673595213, - "recall": 0.1891604675876727, - "far": 0.9975001053326404, - "csi": 0.0024733902120445766, - "f1": 0.004934575293856731, - "f2": 0.01187189029839796, - "frequency_bias": 75.6673751328374, - "predicted_positive_rate": 0.017613605442176872 - } - }, - "val_topk_area_metrics": { - "0.0100": { - "fraction": 0.01, - "cells": 80850, - "precision": 0.002770562656223774, - "recall": 0.11902231723070145, - "lift": 11.90223136656136 - }, - "0.0500": { - "fraction": 0.05, - "cells": 404250, - "precision": 0.001855287584476173, - "recall": 0.39851221442222595, - "lift": 7.970244611627591 - }, - "0.1000": { - "fraction": 0.1, - "cells": 808500, - "precision": 0.0012615955201908946, - "recall": 0.5419766306877136, - "lift": 5.4197661758697135 - } - }, - "val_fss": { - "0.1000": { - "1": 0.0010905469804065593, - "2": 0.001132628631186794, - "4": 0.0012021196677818313, - "8": 0.0012747952224614334 - }, - "0.2000": { - "1": 0.0018261282348317476, - "2": 0.0019093635460114555, - "4": 0.0020469217446071886, - "8": 0.0021685131925710532 - }, - "0.3000": { - "1": 0.002352647456016599, - "2": 0.0024661544218815834, - "4": 0.0026671970273386414, - "8": 0.0028544633354051774 - }, - "0.5000": { - "1": 0.0031238784505325246, - "2": 0.0033962414112024275, - "4": 0.0039018832969852646, - "8": 0.004433729474684234 - }, - "0.7000": { - "1": 0.004722874230304819, - "2": 0.005319378699667143, - "4": 0.006670035900161353, - "8": 0.008381867678164179 - }, - "0.9000": { - "1": 0.00686789744070182, - "2": 0.008074405364309811, - "4": 0.011220289911567538, - "8": 0.016138933098775343 - } - }, - "minutes": 0.3335001548131307 - }, - { - "epoch": 3, - "train_loss": 0.5119181981534846, - "train_main_loss": 0.48301050484862534, - "train_aux_spatial_loss": 0.05781538778911272, - "val_loss": 0.24042269537846248, - "val_pr_auc": 0.0024715155000571394, - "val_auroc": 0.8559554621686548, - "val_brier": 0.06131188943982124, - "val_brier_skill_score": -262.4524802758656, - "val_ece": 0.15086109867404504, - "val_positive_rate": 0.00023277674335986376, - "val_precision": 0.0017179984871622876, - "val_positive_recall": 0.5116896918172157, - "val_far": 0.9982820015128377, - "val_csi": 0.0017151864352441424, - "val_f1": 0.003424499215885694, - "val_f2": 0.008476157616043265, - "val_frequency_bias": 297.8405951115834, - "val_threshold_metrics": { - "0.1000": { - "threshold": 0.1, - "tp": 1692, - "fp": 3465302, - "fn": 190, - "precision": 0.0004880308417032161, - "recall": 0.8990435706695006, - "far": 0.9995119691582968, - "csi": 0.00048800409785001314, - "f1": 0.0009755321320220153, - "f2": 0.0024348672997321647, - "frequency_bias": 1842.1859723698194, - "predicted_positive_rate": 0.42881805813234386 - }, - "0.2000": { - "threshold": 0.2, - "tp": 1457, - "fp": 2011263, - "fn": 425, - "precision": 0.0007238960213045033, - "recall": 0.7741764080765143, - "far": 0.9992761039786955, - "csi": 0.0007237431978322475, - "f1": 0.0014464395448828104, - "f2": 0.0036059929276009675, - "frequency_bias": 1069.4580233793836, - "predicted_positive_rate": 0.24894495980210266 - }, - "0.3000": { - "threshold": 0.3, - "tp": 1286, - "fp": 1194152, - "fn": 596, - "precision": 0.0010757563336618043, - "recall": 0.6833156216790648, - "far": 0.9989242436663381, - "csi": 0.001075220269657886, - "f1": 0.002148130825510306, - "f2": 0.0053451219735221114, - "frequency_bias": 635.1955366631244, - "predicted_positive_rate": 0.14785875077303648 - }, - "0.5000": { - "threshold": 0.5, - "tp": 963, - "fp": 559573, - "fn": 919, - "precision": 0.0017179984871622876, - "recall": 0.5116896918172157, - "far": 0.9982820015128377, - "csi": 0.0017151864352441424, - "f1": 0.003424499215885694, - "f2": 0.008476157616043265, - "frequency_bias": 297.8405951115834, - "predicted_positive_rate": 0.06933036487322201 - }, - "0.7000": { - "threshold": 0.7, - "tp": 755, - "fp": 299786, - "fn": 1127, - "precision": 0.002512136447273417, - "recall": 0.40116896918172157, - "far": 0.9974878635527266, - "csi": 0.0025027513690547222, - "f1": 0.0049930064842951755, - "f2": 0.012253748348584247, - "frequency_bias": 159.692348565356, - "predicted_positive_rate": 0.03717266542980829 - }, - "0.9000": { - "threshold": 0.9, - "tp": 487, - "fp": 107029, - "fn": 1395, - "precision": 0.004529558391309201, - "recall": 0.2587672688629118, - "far": 0.9954704416086908, - "csi": 0.004471540983004472, - "f1": 0.008903270626519681, - "f2": 0.021165814818678075, - "frequency_bias": 57.12858660998937, - "predicted_positive_rate": 0.013298206555349412 - } - }, - "val_topk_area_metrics": { - "0.0100": { - "fraction": 0.01, - "cells": 80850, - "precision": 0.004947433713823557, - "recall": 0.21253985166549683, - "lift": 21.253986297827947 - }, - "0.0500": { - "fraction": 0.05, - "cells": 404250, - "precision": 0.0020754483994096518, - "recall": 0.44580233097076416, - "lift": 8.916047064895523 - }, - "0.1000": { - "fraction": 0.1, - "cells": 808500, - "precision": 0.001385281328111887, - "recall": 0.595111608505249, - "lift": 5.95111568328068 - } - }, - "val_fss": { - "0.1000": { - "1": 0.0010285527774587244, - "2": 0.0010653427122698789, - "4": 0.001122755300464795, - "8": 0.0011787351272731783 - }, - "0.2000": { - "1": 0.0015860647314260845, - "2": 0.0016723213614568966, - "4": 0.0017956498996868353, - "8": 0.0018829515791326523 - }, - "0.3000": { - "1": 0.0023788924877019557, - "2": 0.0025265549899461393, - "4": 0.002761744000741362, - "8": 0.002927857083833363 - }, - "0.5000": { - "1": 0.003953657459880633, - "2": 0.004350779441774422, - "4": 0.005033153433681603, - "8": 0.0055588513866320355 - }, - "0.7000": { - "1": 0.006268758494930693, - "2": 0.007051153593185755, - "4": 0.008535853615263655, - "8": 0.01020087247019108 - }, - "0.9000": { - "1": 0.012090833894108921, - "2": 0.013963613985153223, - "4": 0.018853160860571982, - "8": 0.02710607924875852 - } - }, - "minutes": 0.3254421591758728 - }, - { - "epoch": 4, - "train_loss": 0.4973915634129131, - "train_main_loss": 0.46941266194494213, - "train_aux_spatial_loss": 0.0559578032103662, - "val_loss": 0.2442722608645757, - "val_pr_auc": 0.0015287010547867983, - "val_auroc": 0.861693387637483, - "val_brier": 0.06107700988650322, - "val_brier_skill_score": -261.4432208736014, - "val_ece": 0.14442989307660722, - "val_positive_rate": 0.00023277674335986376, - "val_precision": 0.0016247604629118164, - "val_positive_recall": 0.5063761955366631, - "val_far": 0.9983752395370882, - "val_csi": 0.0016221911666329065, - "val_f1": 0.0032391278486820865, - "val_f2": 0.008020859283997334, - "val_frequency_bias": 311.66206163655687, - "val_threshold_metrics": { - "0.1000": { - "threshold": 0.1, - "tp": 1686, - "fp": 3178518, - "fn": 196, - "precision": 0.0005301546693231, - "recall": 0.895855472901169, - "far": 0.999469845330677, - "csi": 0.0005301219972330525, - "f1": 0.0010596822336039944, - "f2": 0.002644513403259747, - "frequency_bias": 1689.8002125398511, - "predicted_positive_rate": 0.3933461966604824 - }, - "0.2000": { - "threshold": 0.2, - "tp": 1503, - "fp": 1763618, - "fn": 379, - "precision": 0.0008514996988874984, - "recall": 0.7986184909670563, - "far": 0.9991485003011125, - "csi": 0.0008513169073916737, - "f1": 0.0017011855667477644, - "f2": 0.004239417955838973, - "frequency_bias": 937.8963868225293, - "predicted_positive_rate": 0.2183204700061843 - }, - "0.3000": { - "threshold": 0.3, - "tp": 1346, - "fp": 1144434, - "fn": 536, - "precision": 0.0011747455881582851, - "recall": 0.7151965993623804, - "far": 0.9988252544118417, - "csi": 0.0011741962949134445, - "f1": 0.00234563834996715, - "f2": 0.005835388291765947, - "frequency_bias": 608.8097768331562, - "predicted_positive_rate": 0.14171675943104514 - }, - "0.5000": { - "threshold": 0.5, - "tp": 953, - "fp": 585595, - "fn": 929, - "precision": 0.0016247604629118164, - "recall": 0.5063761955366631, - "far": 0.9983752395370882, - "csi": 0.0016221911666329065, - "f1": 0.0032391278486820865, - "f2": 0.008020859283997334, - "frequency_bias": 311.66206163655687, - "predicted_positive_rate": 0.07254768089053804 - }, - "0.7000": { - "threshold": 0.7, - "tp": 640, - "fp": 321536, - "fn": 1242, - "precision": 0.001986491855383393, - "recall": 0.34006376195536664, - "far": 0.9980135081446166, - "csi": 0.0019788632667322166, - "f1": 0.003949910201260268, - "f2": 0.009705675393686459, - "frequency_bias": 171.18809776833157, - "predicted_positive_rate": 0.03984860853432282 - }, - "0.9000": { - "threshold": 0.9, - "tp": 318, - "fp": 123828, - "fn": 1564, - "precision": 0.0025615001691556716, - "recall": 0.1689691817215728, - "far": 0.9974384998308443, - "csi": 0.0025296316919894996, - "f1": 0.0050464976037071126, - "f2": 0.012075276820025214, - "frequency_bias": 65.96493092454836, - "predicted_positive_rate": 0.015355102040816327 - } - }, - "val_topk_area_metrics": { - "0.0100": { - "fraction": 0.01, - "cells": 80850, - "precision": 0.0020408162381500006, - "recall": 0.08767268806695938, - "lift": 8.767268622686151 - }, - "0.0500": { - "fraction": 0.05, - "cells": 404250, - "precision": 0.0018330240854993463, - "recall": 0.3937300741672516, - "lift": 7.874601470240361 - }, - "0.1000": { - "fraction": 0.1, - "cells": 808500, - "precision": 0.0014223870821297169, - "recall": 0.6110520958900452, - "lift": 6.110520585515546 - } - }, - "val_fss": { - "0.1000": { - "1": 0.001124098184984046, - "2": 0.001166554522149088, - "4": 0.0012322749538367095, - "8": 0.0012949795537359199 - }, - "0.2000": { - "1": 0.0018536996365665637, - "2": 0.0019364847225513682, - "4": 0.0020742894927048283, - "8": 0.0021806406439595083 - }, - "0.3000": { - "1": 0.002565767874730618, - "2": 0.0026879762763847292, - "4": 0.002921815146029516, - "8": 0.0031007743459819936 - }, - "0.5000": { - "1": 0.003750084728529779, - "2": 0.004078126887792521, - "4": 0.004664788826923449, - "8": 0.0051470259043820965 - }, - "0.7000": { - "1": 0.004897024677909645, - "2": 0.005528912754412585, - "4": 0.006720608815280804, - "8": 0.007938644694797992 - }, - "0.9000": { - "1": 0.006531676310613244, - "2": 0.0076338310683981, - "4": 0.010603661046574286, - "8": 0.01568061290465117 - } - }, - "minutes": 0.33346703052520754 - }, - { - "epoch": 5, - "train_loss": 0.4875196878293424, - "train_main_loss": 0.4603805759908716, - "train_aux_spatial_loss": 0.05427822470582733, - "val_loss": 0.29524567822615305, - "val_pr_auc": 0.0014329584864629465, - "val_auroc": 0.8570874266988384, - "val_brier": 0.0725969523191452, - "val_brier_skill_score": -310.9435287950134, - "val_ece": 0.1495643223012075, - "val_positive_rate": 0.00023277674335986376, - "val_precision": 0.0013688255199105052, - "val_positive_recall": 0.5233793836344315, - "val_far": 0.9986311744800895, - "val_csi": 0.0013671213559623145, - "val_f1": 0.0027305097737003402, - "val_f2": 0.00677326944684737, - "val_frequency_bias": 382.3565356004251, - "val_threshold_metrics": { - "0.1000": { - "threshold": 0.1, - "tp": 1662, - "fp": 2908424, - "fn": 220, - "precision": 0.0005711171422425317, - "recall": 0.8831030818278427, - "far": 0.9994288828577574, - "csi": 0.0005710739695413472, - "f1": 0.001141496060396268, - "f2": 0.0028482177560157033, - "frequency_bias": 1546.2731137088203, - "predicted_positive_rate": 0.3599364254792826 - }, - "0.2000": { - "threshold": 0.2, - "tp": 1500, - "fp": 1678289, - "fn": 382, - "precision": 0.000892969295548429, - "recall": 0.7970244420828906, - "far": 0.9991070307044516, - "csi": 0.0008927662720044567, - "f1": 0.00178393990263256, - "f2": 0.004444926472026299, - "frequency_bias": 892.5552603613178, - "predicted_positive_rate": 0.2077661100803958 - }, - "0.3000": { - "threshold": 0.3, - "tp": 1327, - "fp": 1182582, - "fn": 555, - "precision": 0.0011208631744500633, - "recall": 0.7051009564293305, - "far": 0.99887913682555, - "csi": 0.001120337975658188, - "f1": 0.002238168446210167, - "f2": 0.005568905447791197, - "frequency_bias": 629.0696068012752, - "predicted_positive_rate": 0.14643277674706245 - }, - "0.5000": { - "threshold": 0.5, - "tp": 985, - "fp": 718610, - "fn": 897, - "precision": 0.0013688255199105052, - "recall": 0.5233793836344315, - "far": 0.9986311744800895, - "csi": 0.0013671213559623145, - "f1": 0.0027305097737003402, - "f2": 0.00677326944684737, - "frequency_bias": 382.3565356004251, - "predicted_positive_rate": 0.08900371057513914 - }, - "0.7000": { - "threshold": 0.7, - "tp": 718, - "fp": 439397, - "fn": 1164, - "precision": 0.0016313917953262216, - "recall": 0.38150903294367694, - "far": 0.9983686082046738, - "csi": 0.0016270885312919944, - "f1": 0.0032488908295757665, - "f2": 0.008019783622216812, - "frequency_bias": 233.85494155154092, - "predicted_positive_rate": 0.05443599257884972 - }, - "0.9000": { - "threshold": 0.9, - "tp": 505, - "fp": 234301, - "fn": 1377, - "precision": 0.0021507116513206645, - "recall": 0.2683315621679065, - "far": 0.9978492883486794, - "csi": 0.002138172518767227, - "f1": 0.004267220982897315, - "f2": 0.010419503660237521, - "frequency_bias": 124.76408076514346, - "predicted_positive_rate": 0.0290421768707483 - } - }, - "val_topk_area_metrics": { - "0.0100": { - "fraction": 0.01, - "cells": 80850, - "precision": 0.0025108226109296083, - "recall": 0.10786397755146027, - "lift": 10.786398051148842 - }, - "0.0500": { - "fraction": 0.05, - "cells": 404250, - "precision": 0.0016870747786015272, - "recall": 0.3623804450035095, - "lift": 7.247608821442164 - }, - "0.1000": { - "fraction": 0.1, - "cells": 808500, - "precision": 0.00131601735483855, - "recall": 0.5653560161590576, - "lift": 5.653560299209267 - } - }, - "val_fss": { - "0.1000": { - "1": 0.0012350949695771885, - "2": 0.0012982024046330753, - "4": 0.0013891551578664174, - "8": 0.0014700961990706807 - }, - "0.2000": { - "1": 0.0019807045037021354, - "2": 0.0020895005644333553, - "4": 0.002273115825633387, - "8": 0.0024347403778096632 - }, - "0.3000": { - "1": 0.0024896008041010997, - "2": 0.002650367743360804, - "4": 0.0029762248158465843, - "8": 0.003287343877128479 - }, - "0.5000": { - "1": 0.003163524553950414, - "2": 0.00349047890930243, - "4": 0.004130384099124695, - "8": 0.004813550356828178 - }, - "0.7000": { - "1": 0.003977167875739984, - "2": 0.0045831981839108415, - "4": 0.005821352827483972, - "8": 0.007370126727222481 - }, - "0.9000": { - "1": 0.005249887506590167, - "2": 0.006010307870879972, - "4": 0.008156507820926917, - "8": 0.011534026564462896 - } - }, - "minutes": 0.34152236382166545 - }, - { - "epoch": 6, - "train_loss": 0.4729130111568113, - "train_main_loss": 0.44654953663776475, - "train_aux_spatial_loss": 0.052726947147827274, - "val_loss": 0.3501560365160306, - "val_pr_auc": 0.0013937340471345924, - "val_auroc": 0.86562865600496, - "val_brier": 0.09354882687330246, - "val_brier_skill_score": -400.97212468650577, - "val_ece": 0.19507035652011379, - "val_positive_rate": 0.00023277674335986376, - "val_precision": 0.0012431519742859437, - "val_positive_recall": 0.6333687566418703, - "val_far": 0.9987568480257141, - "val_csi": 0.0012422580332512457, - "val_f1": 0.002481433485820752, - "val_f2": 0.006167339796622656, - "val_frequency_bias": 509.48618490967056, - "val_threshold_metrics": { - "0.1000": { - "threshold": 0.1, - "tp": 1802, - "fp": 4151030, - "fn": 80, - "precision": 0.0004339207557637776, - "recall": 0.9574920297555791, - "far": 0.9995660792442362, - "csi": 0.0004339123968916269, - "f1": 0.000867448397170058, - "f2": 0.0021656779701756577, - "frequency_bias": 2206.605738575983, - "predicted_positive_rate": 0.5136465058750773 - }, - "0.2000": { - "threshold": 0.2, - "tp": 1629, - "fp": 2545183, - "fn": 253, - "precision": 0.000639623183807835, - "recall": 0.8655685441020191, - "far": 0.9993603768161922, - "csi": 0.0006395596500285623, - "f1": 0.0012783017498373677, - "f2": 0.0031886906206691355, - "frequency_bias": 1353.2476089266738, - "predicted_positive_rate": 0.31500457637600493 - }, - "0.3000": { - "threshold": 0.3, - "tp": 1501, - "fp": 1707825, - "fn": 381, - "precision": 0.0008781238921071814, - "recall": 0.7975557917109458, - "far": 0.9991218761078928, - "csi": 0.0008779282064119759, - "f1": 0.0017543162491058947, - "f2": 0.004371367629396559, - "frequency_bias": 908.249734325186, - "predicted_positive_rate": 0.21141941867656153 - }, - "0.5000": { - "threshold": 0.5, - "tp": 1192, - "fp": 957661, - "fn": 690, - "precision": 0.0012431519742859437, - "recall": 0.6333687566418703, - "far": 0.9987568480257141, - "csi": 0.0012422580332512457, - "f1": 0.002481433485820752, - "f2": 0.006167339796622656, - "frequency_bias": 509.48618490967056, - "predicted_positive_rate": 0.1185965367965368 - }, - "0.7000": { - "threshold": 0.7, - "tp": 855, - "fp": 519311, - "fn": 1027, - "precision": 0.0016437060476847776, - "recall": 0.4543039319872476, - "far": 0.9983562939523152, - "csi": 0.001640467158998682, - "f1": 0.003275560867966164, - "f2": 0.008101285972552275, - "frequency_bias": 276.39001062699253, - "predicted_positive_rate": 0.06433716759431045 - }, - "0.9000": { - "threshold": 0.9, - "tp": 460, - "fp": 219580, - "fn": 1422, - "precision": 0.002090528994728231, - "recall": 0.24442082890541977, - "far": 0.9979094710052717, - "csi": 0.002077105778869513, - "f1": 0.0041456007065545555, - "f2": 0.010106869155593053, - "frequency_bias": 116.91817215727949, - "predicted_positive_rate": 0.027215831787260357 - } - }, - "val_topk_area_metrics": { - "0.0100": { - "fraction": 0.01, - "cells": 80850, - "precision": 0.0018676561303436756, - "recall": 0.08023379743099213, - "lift": 8.023379412333956 - }, - "0.0500": { - "fraction": 0.05, - "cells": 404250, - "precision": 0.001798392040655017, - "recall": 0.38629117608070374, - "lift": 7.7258235281467655 - }, - "0.1000": { - "fraction": 0.1, - "cells": 808500, - "precision": 0.0013593073235824704, - "recall": 0.5839532613754272, - "lift": 5.839532351739428 - } - }, - "val_fss": { - "0.1000": { - "1": 0.0009193882112880081, - "2": 0.0009544238669535465, - "4": 0.0010101364252731448, - "8": 0.0010646885958944807 - }, - "0.2000": { - "1": 0.0014043880513923135, - "2": 0.0014935574288226228, - "4": 0.0016362737109093528, - "8": 0.0017557443359328406 - }, - "0.3000": { - "1": 0.001958875397916504, - "2": 0.0020919404579550216, - "4": 0.002317532927398114, - "8": 0.002500376612063371 - }, - "0.5000": { - "1": 0.0028749173593225485, - "2": 0.0031265486723459635, - "4": 0.0035980207015140486, - "8": 0.004012266867574943 - }, - "0.7000": { - "1": 0.003977407204865502, - "2": 0.0044937585845883365, - "4": 0.005528768396049211, - "8": 0.006620101577705317 - }, - "0.9000": { - "1": 0.00520751276135345, - "2": 0.006124343725859349, - "4": 0.008435537832026108, - "8": 0.011484696864351607 - } - }, - "minutes": 0.33149927854537964 - }, - { - "epoch": 7, - "train_loss": 0.46297245338802046, - "train_main_loss": 0.43753425775094246, - "train_aux_spatial_loss": 0.050876390929891935, - "val_loss": 0.22136119877298674, - "val_pr_auc": 0.0020734326588658226, - "val_auroc": 0.8708066066652015, - "val_brier": 0.0556364431977272, - "val_brier_skill_score": -238.065523638041, - "val_ece": 0.12623748259798678, - "val_positive_rate": 0.00023277674335986376, - "val_precision": 0.0018673864945086484, - "val_positive_recall": 0.5462274176408076, - "val_far": 0.9981326135054913, - "val_csi": 0.0018644940836773337, - "val_f1": 0.003722048430077627, - "val_f2": 0.00921097432037704, - "val_frequency_bias": 292.5090329436769, - "val_threshold_metrics": { - "0.1000": { - "threshold": 0.1, - "tp": 1637, - "fp": 2555675, - "fn": 245, - "precision": 0.0006401252565193453, - "recall": 0.8698193411264612, - "far": 0.9993598747434806, - "csi": 0.0006400639360139383, - "f1": 0.0012793090324531864, - "f2": 0.003191232201618814, - "frequency_bias": 1358.826780021254, - "predicted_positive_rate": 0.31630327767470623 - }, - "0.2000": { - "threshold": 0.2, - "tp": 1451, - "fp": 1472886, - "fn": 431, - "precision": 0.0009841711901688692, - "recall": 0.7709883103081828, - "far": 0.9990158288098311, - "csi": 0.0009838835667711802, - "f1": 0.0019658329827755908, - "f2": 0.004895857584867718, - "frequency_bias": 783.3884165781084, - "predicted_positive_rate": 0.18235460729746444 - }, - "0.3000": { - "threshold": 0.3, - "tp": 1316, - "fp": 1011772, - "fn": 566, - "precision": 0.0012989987049496193, - "recall": 0.6992561105207227, - "far": 0.9987010012950504, - "csi": 0.0012982733753332005, - "f1": 0.0025931800939929255, - "f2": 0.006447086857348895, - "frequency_bias": 538.3039319872477, - "predicted_positive_rate": 0.12530463821892393 - }, - "0.5000": { - "threshold": 0.5, - "tp": 1028, - "fp": 549474, - "fn": 854, - "precision": 0.0018673864945086484, - "recall": 0.5462274176408076, - "far": 0.9981326135054913, - "csi": 0.0018644940836773337, - "f1": 0.003722048430077627, - "f2": 0.00921097432037704, - "frequency_bias": 292.5090329436769, - "predicted_positive_rate": 0.06808930117501547 - }, - "0.7000": { - "threshold": 0.7, - "tp": 769, - "fp": 311654, - "fn": 1113, - "precision": 0.002461406490559274, - "recall": 0.4086078639744952, - "far": 0.9975385935094407, - "csi": 0.0024526689120228616, - "f1": 0.004893336090739887, - "f2": 0.012017465174354824, - "frequency_bias": 166.0058448459086, - "predicted_positive_rate": 0.03864230055658627 - }, - "0.9000": { - "threshold": 0.9, - "tp": 413, - "fp": 132533, - "fn": 1469, - "precision": 0.0031065244535375264, - "recall": 0.21944739638682254, - "far": 0.9968934755464625, - "csi": 0.0030725737454897145, - "f1": 0.006126323908980331, - "f2": 0.014700229223913322, - "frequency_bias": 70.64080765143464, - "predicted_positive_rate": 0.016443537414965986 - } - }, - "val_topk_area_metrics": { - "0.0100": { - "fraction": 0.01, - "cells": 80850, - "precision": 0.003426097799092531, - "recall": 0.1471838504076004, - "lift": 14.718385306198385 - }, - "0.0500": { - "fraction": 0.05, - "cells": 404250, - "precision": 0.002142238663509488, - "recall": 0.46014878153800964, - "lift": 9.202975488825663 - }, - "0.1000": { - "fraction": 0.1, - "cells": 808500, - "precision": 0.001512677758000791, - "recall": 0.6498405933380127, - "lift": 6.4984058809614424 - } - }, - "val_fss": { - "0.1000": { - "1": 0.0014021766263225821, - "2": 0.001488854806077189, - "4": 0.001617522044606745, - "8": 0.0017209109810044643 - }, - "0.2000": { - "1": 0.0022323244436344147, - "2": 0.0024036083494088922, - "4": 0.0026669563437204946, - "8": 0.0028496477215000926 - }, - "0.3000": { - "1": 0.0030061288845402157, - "2": 0.0032706824694820336, - "4": 0.0037414303607975485, - "8": 0.004090089080580284 - }, - "0.5000": { - "1": 0.004510319885638658, - "2": 0.005095488079194688, - "4": 0.006224726885447285, - "8": 0.007265877228674111 - }, - "0.7000": { - "1": 0.0061981922002846135, - "2": 0.0071663426719156265, - "4": 0.009389004622336272, - "8": 0.011978850515245498 - }, - "0.9000": { - "1": 0.00831885386783937, - "2": 0.009977206958581508, - "4": 0.01488323805412095, - "8": 0.022524780923326215 - } - }, - "minutes": 0.33211313088734945 - }, - { - "epoch": 8, - "train_loss": 0.45174517886643994, - "train_main_loss": 0.4269228792569537, - "train_aux_spatial_loss": 0.04964459997631244, - "val_loss": 0.44725767920414605, - "val_pr_auc": 0.0027931174903348286, - "val_auroc": 0.8702263757214033, - "val_brier": 0.12641970813274384, - "val_brier_skill_score": -542.2157770315075, - "val_ece": 0.24778940444824657, - "val_positive_rate": 0.00023277674335986376, - "val_precision": 0.0010423633917768275, - "val_positive_recall": 0.7369819341126461, - "val_far": 0.9989576366082231, - "val_csi": 0.0010419757723729927, - "val_f1": 0.002081782377944814, - "val_f2": 0.005182497134120186, - "val_frequency_bias": 707.0297555791711, - "val_threshold_metrics": { - "0.1000": { - "threshold": 0.1, - "tp": 1809, - "fp": 4950872, - "fn": 73, - "precision": 0.00036525671651374276, - "recall": 0.961211477151966, - "far": 0.9996347432834862, - "csi": 0.00036525133289478946, - "f1": 0.0007302359461369248, - "f2": 0.0018235118721811926, - "frequency_bias": 2631.605207226355, - "predicted_positive_rate": 0.6125764996907854 - }, - "0.2000": { - "threshold": 0.2, - "tp": 1728, - "fp": 3373367, - "fn": 154, - "precision": 0.0005119855885538036, - "recall": 0.9181721572794899, - "far": 0.9994880144114462, - "csi": 0.0005119622285644703, - "f1": 0.0010234005147207101, - "f2": 0.002554230843933835, - "frequency_bias": 1793.355472901169, - "predicted_positive_rate": 0.4174514533085962 - }, - "0.3000": { - "threshold": 0.3, - "tp": 1624, - "fp": 2396644, - "fn": 258, - "precision": 0.0006771553471088302, - "recall": 0.8629117959617428, - "far": 0.9993228446528911, - "csi": 0.0006770825081737701, - "f1": 0.001353248755286128, - "f2": 0.0033751822681557368, - "frequency_bias": 1274.3188097768332, - "predicted_positive_rate": 0.2966317872603587 - }, - "0.5000": { - "threshold": 0.5, - "tp": 1387, - "fp": 1329243, - "fn": 495, - "precision": 0.0010423633917768275, - "recall": 0.7369819341126461, - "far": 0.9989576366082231, - "csi": 0.0010419757723729927, - "f1": 0.002081782377944814, - "f2": 0.005182497134120186, - "frequency_bias": 707.0297555791711, - "predicted_positive_rate": 0.16458008658008658 - }, - "0.7000": { - "threshold": 0.7, - "tp": 1109, - "fp": 713032, - "fn": 773, - "precision": 0.0015529146205021137, - "recall": 0.5892667375132837, - "far": 0.9984470853794979, - "csi": 0.0015512355332249752, - "f1": 0.0030976658571023555, - "f2": 0.007683577928385451, - "frequency_bias": 379.45855472901167, - "predicted_positive_rate": 0.0883291280148423 - }, - "0.9000": { - "threshold": 0.9, - "tp": 772, - "fp": 287094, - "fn": 1110, - "precision": 0.0026818033390535873, - "recall": 0.410201912858661, - "far": 0.9973181966609465, - "csi": 0.0026715021316649134, - "f1": 0.005328768447064346, - "f2": 0.013067293174539767, - "frequency_bias": 152.95749202975557, - "predicted_positive_rate": 0.035604947433518865 - } - }, - "val_topk_area_metrics": { - "0.0100": { - "fraction": 0.01, - "cells": 80850, - "precision": 0.0053803338669240475, - "recall": 0.23113708198070526, - "lift": 23.113708823592663 - }, - "0.0500": { - "fraction": 0.05, - "cells": 404250, - "precision": 0.0021595547441393137, - "recall": 0.46386823058128357, - "lift": 9.27736470993035 - }, - "0.1000": { - "fraction": 0.1, - "cells": 808500, - "precision": 0.0014545454178005457, - "recall": 0.6248671412467957, - "lift": 6.248671567467869 - } - }, - "val_fss": { - "0.1000": { - "1": 0.0007674856988975076, - "2": 0.000791355414216266, - "4": 0.0008260328883219481, - "8": 0.0008700381800748058 - }, - "0.2000": { - "1": 0.0010883550972506928, - "2": 0.0011302905318852918, - "4": 0.0011919848085577556, - "8": 0.001259132880186864 - }, - "0.3000": { - "1": 0.0014676516650308224, - "2": 0.0015372280258556525, - "4": 0.0016419540607538474, - "8": 0.0017449860353127011 - }, - "0.5000": { - "1": 0.0023237192035810983, - "2": 0.002464085798703608, - "4": 0.002708399276045159, - "8": 0.0029286220283636366 - }, - "0.7000": { - "1": 0.0035873573947993176, - "2": 0.003932072661929298, - "4": 0.004541864058129508, - "8": 0.005095613653573028 - }, - "0.9000": { - "1": 0.006636955504364939, - "2": 0.007636333009147123, - "4": 0.009658326859772859, - "8": 0.012242707807434083 - } - }, - "minutes": 0.3327596187591553 - }, - { - "epoch": 9, - "train_loss": 0.44174556909476165, - "train_main_loss": 0.41717497324598773, - "train_aux_spatial_loss": 0.04914119221815917, - "val_loss": 0.35248868018388746, - "val_pr_auc": 0.0014017929970181718, - "val_auroc": 0.826237549108265, - "val_brier": 0.09342223405838013, - "val_brier_skill_score": -400.42816508289366, - "val_ece": 0.21254881950345395, - "val_positive_rate": 0.00023277674335986376, - "val_precision": 0.0011612270298949224, - "val_positive_recall": 0.5547290116896918, - "val_far": 0.998838772970105, - "val_csi": 0.00116014566273321, - "val_f1": 0.002317602568898173, - "val_f2": 0.00575792238276506, - "val_frequency_bias": 477.70935175345375, - "val_threshold_metrics": { - "0.1000": { - "threshold": 0.1, - "tp": 1744, - "fp": 4810537, - "fn": 138, - "precision": 0.0003624061022205478, - "recall": 0.926673751328374, - "far": 0.9996375938977794, - "csi": 0.00036239570993298794, - "f1": 0.000724528853717666, - "f2": 0.0018092003230833422, - "frequency_bias": 2557.003719447396, - "predicted_positive_rate": 0.5952110080395795 - }, - "0.2000": { - "threshold": 0.2, - "tp": 1569, - "fp": 2798322, - "fn": 313, - "precision": 0.0005603789576094213, - "recall": 0.8336875664187035, - "far": 0.9994396210423906, - "csi": 0.0005603163198109853, - "f1": 0.0011200050824959767, - "f2": 0.0027943816010363976, - "frequency_bias": 1487.7210414452709, - "predicted_positive_rate": 0.34630686456400744 - }, - "0.3000": { - "threshold": 0.3, - "tp": 1407, - "fp": 1861031, - "fn": 475, - "precision": 0.0007554613898556623, - "recall": 0.7476089266737513, - "far": 0.9992445386101443, - "csi": 0.0007552687645638847, - "f1": 0.0015093975283213183, - "f2": 0.003762100487388541, - "frequency_bias": 989.605738575983, - "predicted_positive_rate": 0.23035720470006185 - }, - "0.5000": { - "threshold": 0.5, - "tp": 1044, - "fp": 898005, - "fn": 838, - "precision": 0.0011612270298949224, - "recall": 0.5547290116896918, - "far": 0.998838772970105, - "csi": 0.00116014566273321, - "f1": 0.002317602568898173, - "f2": 0.00575792238276506, - "frequency_bias": 477.70935175345375, - "predicted_positive_rate": 0.11119962894248608 - }, - "0.7000": { - "threshold": 0.7, - "tp": 749, - "fp": 463515, - "fn": 1133, - "precision": 0.0016133062223217824, - "recall": 0.39798087141339, - "far": 0.9983866937776782, - "csi": 0.0016093786595100528, - "f1": 0.0032135854431873277, - "f2": 0.007937820056295994, - "frequency_bias": 246.6865037194474, - "predicted_positive_rate": 0.057422881880024736 - }, - "0.9000": { - "threshold": 0.9, - "tp": 453, - "fp": 189367, - "fn": 1429, - "precision": 0.0023864713939521654, - "recall": 0.24070138150903295, - "far": 0.9976135286060478, - "csi": 0.0023686398360252864, - "f1": 0.004726085278192194, - "f2": 0.011477187506333988, - "frequency_bias": 100.86078639744952, - "predicted_positive_rate": 0.02347804576376005 - } - }, - "val_topk_area_metrics": { - "0.0100": { - "fraction": 0.01, - "cells": 80850, - "precision": 0.0027334571350365877, - "recall": 0.1174282655119896, - "lift": 11.742827464558045 - }, - "0.0500": { - "fraction": 0.05, - "cells": 404250, - "precision": 0.0017217068234458566, - "recall": 0.3698193430900574, - "lift": 7.396386763535759 - }, - "0.1000": { - "fraction": 0.1, - "cells": 808500, - "precision": 0.0012269634753465652, - "recall": 0.5270988345146179, - "lift": 5.270988233776119 - } - }, - "val_fss": { - "0.1000": { - "1": 0.0007761889316962867, - "2": 0.0008088700048963782, - "4": 0.0008539254426118692, - "8": 0.0008953386395748053 - }, - "0.2000": { - "1": 0.0012289524564135235, - "2": 0.0012947806940213358, - "4": 0.0013901778317579883, - "8": 0.0014569956840254639 - }, - "0.3000": { - "1": 0.001693693219550152, - "2": 0.0017980729206314994, - "4": 0.001959161549363153, - "8": 0.0020775186421173597 - }, - "0.5000": { - "1": 0.002685166736102307, - "2": 0.0029421415350803226, - "4": 0.0034770871818433635, - "8": 0.003947886788339461 - }, - "0.7000": { - "1": 0.003915058495870372, - "2": 0.00452728637874078, - "4": 0.005929248077012117, - "8": 0.007441641374286712 - }, - "0.9000": { - "1": 0.006463210719885204, - "2": 0.00788266562330997, - "4": 0.011779975914961827, - "8": 0.01823609329615261 - } - }, - "minutes": 0.32817530234654746 - }, - { - "epoch": 10, - "train_loss": 0.43139428173367955, - "train_main_loss": 0.4073724075672085, - "train_aux_spatial_loss": 0.048043749015301965, - "val_loss": 0.2240987864633401, - "val_pr_auc": 0.003116102503753257, - "val_auroc": 0.874721890484618, - "val_brier": 0.05645371973514557, - "val_brier_skill_score": -241.57729815390343, - "val_ece": 0.1303528208549411, - "val_positive_rate": 0.00023277674335986376, - "val_precision": 0.0018323169746849528, - "val_positive_recall": 0.5425079702444209, - "val_far": 0.998167683025315, - "val_csi": 0.0018294900901126901, - "val_f1": 0.003652298336612413, - "val_f2": 0.009039461988221253, - "val_frequency_bias": 296.07757704569605, - "val_threshold_metrics": { - "0.1000": { - "threshold": 0.1, - "tp": 1658, - "fp": 2737917, - "fn": 224, - "precision": 0.0006052033618353212, - "recall": 0.8809776833156217, - "far": 0.9993947966381647, - "csi": 0.0006051538817263602, - "f1": 0.0012095757839718077, - "f2": 0.0030177244901265082, - "frequency_bias": 1455.67215727949, - "predicted_positive_rate": 0.33884662956091527 - }, - "0.2000": { - "threshold": 0.2, - "tp": 1447, - "fp": 1498772, - "fn": 435, - "precision": 0.0009645258458931663, - "recall": 0.7688629117959618, - "far": 0.9990354741541069, - "csi": 0.0009642462552993562, - "f1": 0.0019266347602458157, - "f2": 0.004798550419931196, - "frequency_bias": 797.1408076514347, - "predicted_positive_rate": 0.18555584415584417 - }, - "0.3000": { - "threshold": 0.3, - "tp": 1304, - "fp": 1006923, - "fn": 578, - "precision": 0.0012933595311373331, - "recall": 0.6928799149840595, - "far": 0.9987066404688627, - "csi": 0.0012926184941589307, - "f1": 0.0025818995771743447, - "f2": 0.0064188706922437, - "frequency_bias": 535.721041445271, - "predicted_positive_rate": 0.12470340136054421 - }, - "0.5000": { - "threshold": 0.5, - "tp": 1021, - "fp": 556197, - "fn": 861, - "precision": 0.0018323169746849528, - "recall": 0.5425079702444209, - "far": 0.998167683025315, - "csi": 0.0018294900901126901, - "f1": 0.003652298336612413, - "f2": 0.009039461988221253, - "frequency_bias": 296.07757704569605, - "predicted_positive_rate": 0.0689199752628324 - }, - "0.7000": { - "threshold": 0.7, - "tp": 820, - "fp": 311338, - "fn": 1062, - "precision": 0.0026268748518378515, - "recall": 0.4357066950053135, - "far": 0.9973731251481621, - "csi": 0.002617968201264287, - "f1": 0.0052222646796586435, - "f2": 0.012825084614277761, - "frequency_bias": 165.86503719447396, - "predicted_positive_rate": 0.03860952380952381 - }, - "0.9000": { - "threshold": 0.9, - "tp": 560, - "fp": 134308, - "fn": 1322, - "precision": 0.004152208084942314, - "recall": 0.2975557917109458, - "far": 0.9958477919150577, - "csi": 0.004111902489169542, - "f1": 0.008190127970749542, - "f2": 0.019663473693081267, - "frequency_bias": 71.66206163655686, - "predicted_positive_rate": 0.01668126159554731 - } - }, - "val_topk_area_metrics": { - "0.0100": { - "fraction": 0.01, - "cells": 80850, - "precision": 0.004922696389257908, - "recall": 0.2114771455526352, - "lift": 21.147715696183667 - }, - "0.0500": { - "fraction": 0.05, - "cells": 404250, - "precision": 0.0022238714154809713, - "recall": 0.47768330574035645, - "lift": 9.553666673834993 - }, - "0.1000": { - "fraction": 0.1, - "cells": 808500, - "precision": 0.0014669140800833702, - "recall": 0.6301806569099426, - "lift": 6.301806868290009 - } - }, - "val_fss": { - "0.1000": { - "1": 0.0013048518501964645, - "2": 0.0013652965653525584, - "4": 0.0014515110692925326, - "8": 0.0015328291894178836 - }, - "0.2000": { - "1": 0.0021372784576736414, - "2": 0.002257479647874039, - "4": 0.002436579126223748, - "8": 0.0025690244682270436 - }, - "0.3000": { - "1": 0.00287961415383009, - "2": 0.0030574405787181513, - "4": 0.0033325979533164185, - "8": 0.00352065311784866 - }, - "0.5000": { - "1": 0.004248566368253326, - "2": 0.004625449177877106, - "4": 0.005259187972128654, - "8": 0.00579578478616205 - }, - "0.7000": { - "1": 0.006428046413327948, - "2": 0.007130644667450725, - "4": 0.008581068668154113, - "8": 0.010327699790465106 - }, - "0.9000": { - "1": 0.01054567134777984, - "2": 0.01212271654616559, - "4": 0.015815071561349092, - "8": 0.02133905715867812 - } - }, - "minutes": 0.32832499742507937 - } -] diff --git a/models/wildfire_fm/metrics/reference_fireprone_seed_99_20260504/run_summary.json b/models/wildfire_fm/metrics/reference_fireprone_seed_99_20260504/run_summary.json deleted file mode 100644 index 7dd55d9d4d721878a50a4d04faecf601d81fd41e..0000000000000000000000000000000000000000 --- a/models/wildfire_fm/metrics/reference_fireprone_seed_99_20260504/run_summary.json +++ /dev/null @@ -1,172 +0,0 @@ -{ - "run_name": "reference_fireprone_seed_99_20260504", - "device": "cuda", - "index_root": "", - "input_channels": 16, - "tile_size": 32, - "num_train_tiles": 27177, - "positive_train_tiles": 6724, - "negative_train_tiles": 20453, - "train_positive_rate": 0.0018334667181808147, - "metric_reference_positive_rate": 0.0002787771234976204, - "pos_weight": 50.0, - "use_aux_spatial_head": true, - "aux_spatial_radius": 2, - "aux_spatial_loss_weight": 0.5, - "aux_positive_rate": 0.004560296846011132, - "init_checkpoint": "", - "best_val_pr_auc": 0.003116102503753257, - "best_val_auroc": 0.874721890484618, - "best_test_pr_auc": 0.001365278660864068, - "best_test_auroc": 0.82665163680193, - "best_test_brier": 0.07600810378789902, - "best_test_ece": 0.15982505350344936, - "best_test_topk_area_metrics": { - "0.0100": { - "fraction": 0.01, - "cells": 80850, - "precision": 0.0023623995948582888, - "recall": 0.0973992869257927, - "lift": 9.73992880989997 - }, - "0.0500": { - "fraction": 0.05, - "cells": 404250, - "precision": 0.001677179941907525, - "recall": 0.34574195742607117, - "lift": 6.914839162318504 - }, - "0.1000": { - "fraction": 0.1, - "cells": 808500, - "precision": 0.0012034631799906492, - "recall": 0.4961754083633423, - "lift": 4.961754024999269 - } - }, - "best_test_threshold_metrics": { - "0.1000": { - "threshold": 0.1, - "tp": 1701, - "fp": 3149821, - "fn": 260, - "precision": 0.0005397392117205592, - "recall": 0.8674145843957165, - "far": 0.9994602607882794, - "csi": 0.0005396946870056368, - "f1": 0.0010788071475254504, - "f2": 0.0026919957991571727, - "frequency_bias": 1607.0994390617032, - "predicted_positive_rate": 0.3897986394557823 - }, - "0.2000": { - "threshold": 0.2, - "tp": 1452, - "fp": 1964811, - "fn": 509, - "precision": 0.0007384566561034816, - "recall": 0.7404385517593065, - "far": 0.9992615433438965, - "csi": 0.0007382655437437588, - "f1": 0.001475441819630286, - "f2": 0.0036776122064305534, - "frequency_bias": 1002.6838347781744, - "predicted_positive_rate": 0.24319888682745824 - }, - "0.3000": { - "threshold": 0.3, - "tp": 1287, - "fp": 1409052, - "fn": 674, - "precision": 0.000912546557955215, - "recall": 0.6562978072412035, - "far": 0.9990874534420447, - "csi": 0.0009121106609223303, - "f1": 0.00182255894639949, - "f2": 0.00453749621875315, - "frequency_bias": 719.1937786843447, - "predicted_positive_rate": 0.17443896103896103 - }, - "0.5000": { - "threshold": 0.5, - "tp": 968, - "fp": 802322, - "fn": 993, - "precision": 0.0012050442554992593, - "recall": 0.49362570117287097, - "far": 0.9987949557445007, - "csi": 0.001203556459604393, - "f1": 0.0024042193055333057, - "f2": 0.005966954905108157, - "frequency_bias": 409.6328403875574, - "predicted_positive_rate": 0.09935559678416821 - }, - "0.7000": { - "threshold": 0.7, - "tp": 707, - "fp": 432996, - "fn": 1254, - "precision": 0.001630147820051971, - "recall": 0.36053034166241715, - "far": 0.998369852179948, - "csi": 0.0016254480327940464, - "f1": 0.00324562047816666, - "f2": 0.008005942742222232, - "frequency_bias": 221.16420193778686, - "predicted_positive_rate": 0.05364291898577613 - }, - "0.9000": { - "threshold": 0.9, - "tp": 409, - "fp": 163803, - "fn": 1552, - "precision": 0.002490682775923806, - "recall": 0.2085670576236614, - "far": 0.9975093172240762, - "csi": 0.0024673632393040707, - "f1": 0.004922580683985966, - "f2": 0.011885665132282513, - "frequency_bias": 83.7389087200408, - "predicted_positive_rate": 0.020310698824984538 - } - }, - "best_test_fss": { - "0.1000": { - "1": 0.001143681210237446, - "2": 0.0011847434725281714, - "4": 0.001236591045744806, - "8": 0.0012702633695319543 - }, - "0.2000": { - "1": 0.0015924077611619847, - "2": 0.0016590472832728187, - "4": 0.0017424757489505893, - "8": 0.0017626104493195793 - }, - "0.3000": { - "1": 0.0019615416657199347, - "2": 0.002042614896579975, - "4": 0.0021496045193630797, - "8": 0.0021595197888091278 - }, - "0.5000": { - "1": 0.0026900758912903333, - "2": 0.0028522394616976587, - "4": 0.0030564017994608728, - "8": 0.003127168867652075 - }, - "0.7000": { - "1": 0.0038445992739185586, - "2": 0.004171973927427075, - "4": 0.00463101415464473, - "8": 0.004986515076567888 - }, - "0.9000": { - "1": 0.0064130574398052476, - "2": 0.007473703614246263, - "4": 0.00958738894872413, - "8": 0.012579819257860225 - } - }, - "epochs": 10 -} diff --git a/models/wildfire_fm/metrics/reference_fireprone_seed_99_20260504/tile_summary.json b/models/wildfire_fm/metrics/reference_fireprone_seed_99_20260504/tile_summary.json deleted file mode 100644 index 38db43a1e57d30bf3cbb4d9873a51294d8ef8055..0000000000000000000000000000000000000000 --- a/models/wildfire_fm/metrics/reference_fireprone_seed_99_20260504/tile_summary.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "index_root": "", - "tile_size": 32, - "num_train_tiles": 27177, - "positive_train_tiles": 6724, - "negative_train_tiles": 20453, - "input_channels": 16, - "train_positive_rate": 0.0018334667181808147, - "metric_reference_positive_rate": 0.0002787771234976204, - "pos_weight_source": "tiles", - "positive_rate_source": "tiles", - "pos_weight": 50.0, - "loss_type": "bce", - "train_target_mode": "dilate_max", - "train_target_radius": 2 -} diff --git a/models/wildfire_fm/training_scope.json b/models/wildfire_fm/training_scope.json deleted file mode 100644 index 173d6c3c8b38decad9a779a76288188263d202f7..0000000000000000000000000000000000000000 --- a/models/wildfire_fm/training_scope.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "released_checkpoint_scope": { - "region": "California", - "crs": "EPSG:5070", - "resolution_m": 5000, - "target_lead_hours": 12, - "status": "released checkpoints are California regional weights" - }, - "nationwide_status": { - "conus_checkpoint_released": false, - "conus_training_recipe_released": true, - "conus_cache_config": "training/configs/stage1_cache_conus_hrrr_us_5km_l12_template.json", - "conus_train_config": "training/configs/train_firewx_fm_conus_seed7_template.json" - }, - "notes": [ - "The current model weights should not be described as nationwide-trained.", - "The same input channel contract can be used for CONUS retraining after preparing local provider data.", - "Raw provider data are not redistributed in this repository." - ] -} diff --git a/paper/Does_Your_Wildfire_Prediction_Model_ICLR_revision.pdf b/paper/Does_Your_Wildfire_Prediction_Model_ICLR_revision.pdf new file mode 100644 index 0000000000000000000000000000000000000000..ca5653ca489cba40f6140d826e483fe71a9e9ac1 --- /dev/null +++ b/paper/Does_Your_Wildfire_Prediction_Model_ICLR_revision.pdf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4b5777c523412787b9b498870e54111976870b31b1a8e8c5a5c22a5ce76e502f +size 1767112 diff --git a/paper/iclr_source/.latexmkrc b/paper/iclr_source/.latexmkrc new file mode 100644 index 0000000000000000000000000000000000000000..66e1ede36493bd72d1a2ff565882a6a21e4f75fb --- /dev/null +++ b/paper/iclr_source/.latexmkrc @@ -0,0 +1,3 @@ +$pdf_mode = 1; +$pdflatex = 'pdflatex -interaction=nonstopmode -halt-on-error %O %S'; +$bibtex_use = 2; diff --git a/paper/iclr_source/figures/fig_fireprone_contract_progression_compact.pdf b/paper/iclr_source/figures/fig_fireprone_contract_progression_compact.pdf new file mode 100644 index 0000000000000000000000000000000000000000..a0a9b64655cbd8309f089665de8a2ecdcd4ecb54 Binary files /dev/null and b/paper/iclr_source/figures/fig_fireprone_contract_progression_compact.pdf differ diff --git a/paper/iclr_source/figures/fig_primary_rank_change_map.pdf b/paper/iclr_source/figures/fig_primary_rank_change_map.pdf new file mode 100644 index 0000000000000000000000000000000000000000..b9421a38ca826b131f5e6c9efb0f3e084bca5804 Binary files /dev/null and b/paper/iclr_source/figures/fig_primary_rank_change_map.pdf differ diff --git a/paper/iclr_source/figures/fig_rank_heatmap1.pdf b/paper/iclr_source/figures/fig_rank_heatmap1.pdf new file mode 100644 index 0000000000000000000000000000000000000000..d5dff753c7000e426230a9a6aef18ef15150c0f3 Binary files /dev/null and b/paper/iclr_source/figures/fig_rank_heatmap1.pdf differ diff --git a/paper_outputs/figures/fig_selection_regret_scatter.pdf b/paper/iclr_source/figures/fig_selection_regret_scatter.pdf similarity index 100% rename from paper_outputs/figures/fig_selection_regret_scatter.pdf rename to paper/iclr_source/figures/fig_selection_regret_scatter.pdf diff --git a/paper_outputs/figures/fig_task_contract_tiles.pdf b/paper/iclr_source/figures/fig_task_contract_tiles.pdf similarity index 100% rename from paper_outputs/figures/fig_task_contract_tiles.pdf rename to paper/iclr_source/figures/fig_task_contract_tiles.pdf diff --git a/paper_outputs/figures/matching.pdf b/paper/iclr_source/figures/matching.pdf similarity index 100% rename from paper_outputs/figures/matching.pdf rename to paper/iclr_source/figures/matching.pdf diff --git a/paper_outputs/figures/overview_wildfire.pdf b/paper/iclr_source/figures/overview_wildfire.pdf similarity index 100% rename from paper_outputs/figures/overview_wildfire.pdf rename to paper/iclr_source/figures/overview_wildfire.pdf diff --git a/paper/iclr_source/main.tex b/paper/iclr_source/main.tex new file mode 100644 index 0000000000000000000000000000000000000000..e8c29899750265c26b8c30480bf80cc30b5d812c --- /dev/null +++ b/paper/iclr_source/main.tex @@ -0,0 +1,144 @@ +% !TeX root = main.tex +% !TeX program = pdflatex +\documentclass{article} +\usepackage[preprint]{neurips_2026} +\usepackage[utf8]{inputenc} % allow utf-8 input +\usepackage[T1]{fontenc} % use 8-bit T1 fonts +\usepackage{hyperref} % hyperlinks +\usepackage{url} % simple URL typesetting +\usepackage{booktabs} % professional-quality tables +\usepackage{amsfonts} % blackboard math symbols +\usepackage{nicefrac} % compact symbols for 1/2, etc. +\usepackage{microtype} % microtypography +\usepackage[table]{xcolor} % colors +\usepackage{placeins} +\usepackage[utf8]{inputenc} +\usepackage[T1]{fontenc} +\usepackage{hyperref} +\setcitestyle{numbers,square} +\definecolor{tocneutral}{RGB}{0, 0, 0} +\hypersetup{ + hidelinks, + pdfauthor={Yangshuang Xu, Yuyang Dai, Liling Chang, Qi Wang, Yushun Dong}, + pdftitle={Does Your Wildfire Prediction Model Actually Work, or Just Score Well?}, + pdfsubject={}, + pdfkeywords={} +} + +\usepackage{url} +\usepackage{booktabs} +\usepackage{amsfonts} +\usepackage{nicefrac} +\usepackage{microtype} +\usepackage{amsmath} +\usepackage{amssymb} +\usepackage{graphicx} +\usepackage{tabularx} +\usepackage{longtable} +\usepackage{multirow} +\usepackage{array} +\usepackage{float} +\usepackage{adjustbox} +\usepackage{placeins} +\usepackage{enumitem} +\usepackage{siunitx} +\usepackage{tikz} +\usepackage{subcaption} +\usepackage{wrapfig} +\usepackage[normalem]{ulem} +\usepackage{pifont} +\usepackage{hyperref} +\usepackage{xcolor} +\usepackage{tabularx} +\usepackage{xspace} + + +\sisetup{detect-all} + +\definecolor{wfneutral}{RGB}{80,80,80} +\definecolor{wforange}{RGB}{231,111,81} +\definecolor{wfgreen}{RGB}{42,157,143} +\definecolor{wfgold}{RGB}{233,196,106} +\definecolor{wfslate}{RGB}{38,70,83} +\definecolor{wfgray}{RGB}{108,117,125} +\definecolor{wfpurple}{RGB}{116,81,164} +\definecolor{wfindigo}{RGB}{90,90,90} +\definecolor{wfrose}{RGB}{188,80,144} +\definecolor{wfolive}{RGB}{120,143,64} +\definecolor{primarybg}{RGB}{245,245,245} +\definecolor{primaryrule}{RGB}{190,190,190} +\definecolor{headerbg}{RGB}{80,80,80} +\definecolor{regbg}{RGB}{240,240,240} +\definecolor{retrbg}{RGB}{232,232,232} +\definecolor{refbg}{RGB}{255,243,205} % amber – reference row +\definecolor{alphabg}{RGB}{220,237,220} % green – AlphaEarth row +\definecolor{subheadbg}{RGB}{241,245,249} % near-white – column subheader +\definecolor{bestval}{RGB}{0,100,0} % dark green – best frozen value +\definecolor{warnval}{RGB}{180,0,0} % dark red – anomalous value + +\newcolumntype{L}[1]{>{\raggedright\arraybackslash}p{#1}} +\newcolumntype{Y}{>{\raggedright\arraybackslash}X} +\newcommand{\ms}[2] +{\ensuremath{#1{\mkern1mu}_{\scriptscriptstyle \pm #2}}} +\newcommand{\msb}[2]{\ensuremath{\mathbf{#1 \pm #2}}} + +\newcommand{\best}[1]{\textbf{#1}} +\newcommand{\ourfm}{\textsc{Wild}{\textbf{FIRE}}\textsc{-FM}\xspace} +% \title{Ranking Is Not Decision Quality: Evaluation Contracts for Wildfire-Centric Transfer} +% \title{Does Your Wildfire Prediction Model Actually Work,\\ or Just Score Well?} +\title{Does Your Wildfire Prediction Model Actually Work,\\ or Just Score Well?} +% : Does Your Wildfire Prediction Model Just Score Well? + + + +\author{% + Yangshuang Xu\thanks{Equal contribution.} \\ + Florida State University \\ + \texttt{yx21e@fsu.edu} \\ + \And + Yuyang Dai\footnotemark[1] \\ + Florida State University \\ + \texttt{yd26@fsu.edu} \\ + \And + Liling Chang \\ + Florida State University \\ + \texttt{liling.chang@fsu.edu} \\ + \And + Qi Wang \\ + Northeastern University \\ + \texttt{wangqi@vt.edu} \\ + \And + Yushun Dong\thanks{Corresponding author.} \\ + Florida State University \\ + \texttt{yd24f@fsu.edu} \\ +} + +\begin{document} + +\maketitle + +\input{sections/0_abstract} + +\begin{center} +\small +\textbf{Model card:}~\texttt{https://huggingface.co/RAI-Lab/Wildfire-FM} +\end{center} + +\input{sections/1_intro} +\input{sections/2_backbone} +% \input{sections/background} +\input{sections/3_prelim} +% \input{sections/methodology} +\input{sections/4_experiments} +\input{sections/5_conclusion} + +% \bibliographystyle{abbrvnat} +\bibliographystyle{plain} +\bibliography{references} + +\newpage +\input{sections/appendix} + + + +\end{document} diff --git a/paper/iclr_source/neurips_2026.sty b/paper/iclr_source/neurips_2026.sty new file mode 100644 index 0000000000000000000000000000000000000000..1fb049c55bc7d548beb2797f09cdddd62806f208 --- /dev/null +++ b/paper/iclr_source/neurips_2026.sty @@ -0,0 +1,437 @@ +% partial rewrite of the LaTeX2e package for submissions to the +% Conference on Neural Information Processing Systems (NeurIPS): +% +% - uses more LaTeX conventions +% - line numbers at submission time replaced with aligned numbers from +% lineno package +% - \nipsfinalcopy replaced with [final] package option +% - automatically loads times package for authors +% - loads natbib automatically; this can be suppressed with the +% [nonatbib] package option +% - adds foot line to first page identifying the conference +% - adds preprint option for submission to e.g. arXiv +% - conference acronym modified +% - update foot line to display the track name +% +% Roman Garnett (garnett@wustl.edu) and the many authors of +% nips15submit_e.sty, including MK and drstrip@sandia +% +% last revision: January 2026 + +\NeedsTeXFormat{LaTeX2e} +\ProvidesPackage{neurips_2026}[2026-01-29 NeurIPS 2026 submission/camera-ready style file] + +% declare final option, which creates camera-ready copy +\newif\if@neuripsfinal\@neuripsfinalfalse +\DeclareOption{final}{ + \@neuripsfinaltrue + \@anonymousfalse +} + +% declare nonatbib option, which does not load natbib in case of +% package clash (users can pass options to natbib via +% \PassOptionsToPackage) +\newif\if@natbib\@natbibtrue +\DeclareOption{nonatbib}{ + \@natbibfalse +} + +% declare preprint option, which creates a preprint version ready for +% upload to, e.g., arXiv +\newif\if@preprint\@preprintfalse +\DeclareOption{preprint}{ + \@preprinttrue + \@anonymousfalse +} + +% determine the track of the paper in camera-ready mode +\newif\if@main\@maintrue +\DeclareOption{main}{ + \@maintrue + \newcommand{\@trackname}{\@neuripsordinal\ Conference on Neural Information Processing Systems (NeurIPS \@neuripsyear).} +} +\newif\if@position\@positionfalse +\DeclareOption{position}{ + \@positiontrue + \newcommand{\@trackname}{\@neuripsordinal\ Conference on Neural Information Processing Systems (NeurIPS \@neuripsyear). Position Paper Track.} +} +\newif\if@eandd\@eanddfalse +\DeclareOption{eandd}{ + \@eanddtrue +\if@neuripsfinal\@anonymousfalse\else\if@preprint\@anonymousfalse\else\@anonymoustrue\fi\fi + \newcommand{\@trackname}{\@neuripsordinal\ Conference on Neural Information Processing Systems (NeurIPS \@neuripsyear). Track on Evaluations and Datasets.} +} +\newif\if@creativeai\@creativeaifalse +\DeclareOption{creativeai}{ + \@creativeaitrue + \@anonymousfalse + \newcommand{\@trackname}{\@neuripsordinal\ Conference on Neural Information Processing Systems (NeurIPS \@neuripsyear). Creative AI Track.} +} + +% For anonymous or non-anonymous +\newif\if@anonymous\@anonymoustrue + +% For workshop papers +\newcommand{\@workshoptitle}{} +\newcommand{\workshoptitle}[1]{\renewcommand{\@workshoptitle}{#1}} + +\newif\if@workshop\@workshopfalse +\DeclareOption{sglblindworkshop}{ + \@workshoptrue + \@anonymousfalse + \newcommand{\@trackname}{\@neuripsordinal\ Conference on Neural Information Processing Systems (NeurIPS \@neuripsyear). Workshop: \@workshoptitle.} +} +\DeclareOption{dblblindworkshop}{ + \@workshoptrue + \newcommand{\@trackname}{\@neuripsordinal\ Conference on Neural Information Processing Systems (NeurIPS \@neuripsyear). Workshop: \@workshoptitle.} +} +\DeclareOption{nonanonymous}{ + \@anonymousfalse +} + +\ProcessOptions\relax + +% fonts +\renewcommand{\rmdefault}{ptm} +\renewcommand{\sfdefault}{phv} + +% change this every year for notice string at bottom +\newcommand{\@neuripsordinal}{40th} +\newcommand{\@neuripsyear}{2026} +\newcommand{\@neuripslocation}{Sydney} + +% acknowledgments +\usepackage{environ} +\newcommand{\acksection}{\section*{Acknowledgments and Disclosure of Funding}} +\NewEnviron{ack}{% + \acksection + \BODY +} + + +% load natbib unless told otherwise +\if@natbib + \RequirePackage{natbib} +\fi + + + + + +% set page geometry +\usepackage[verbose=true,letterpaper]{geometry} +\AtBeginDocument{ + \newgeometry{ + textheight=9in, + textwidth=5.5in, + top=1in, + headheight=12pt, + headsep=25pt, + footskip=30pt + } + \@ifpackageloaded{fullpage} + {\PackageWarning{neurips_2026}{fullpage package not allowed! Overwriting formatting.}} + {} +} + +\widowpenalty=10000 +\clubpenalty=10000 +\flushbottom +\sloppy + + +% font sizes with reduced leading +\renewcommand{\normalsize}{% + \@setfontsize\normalsize\@xpt\@xipt + \abovedisplayskip 7\p@ \@plus 2\p@ \@minus 5\p@ + \abovedisplayshortskip \z@ \@plus 3\p@ + \belowdisplayskip \abovedisplayskip + \belowdisplayshortskip 4\p@ \@plus 3\p@ \@minus 3\p@ +} +\normalsize +\renewcommand{\small}{% + \@setfontsize\small\@ixpt\@xpt + \abovedisplayskip 6\p@ \@plus 1.5\p@ \@minus 4\p@ + \abovedisplayshortskip \z@ \@plus 2\p@ + \belowdisplayskip \abovedisplayskip + \belowdisplayshortskip 3\p@ \@plus 2\p@ \@minus 2\p@ +} +\renewcommand{\footnotesize}{\@setfontsize\footnotesize\@ixpt\@xpt} +\renewcommand{\scriptsize}{\@setfontsize\scriptsize\@viipt\@viiipt} +\renewcommand{\tiny}{\@setfontsize\tiny\@vipt\@viipt} +\renewcommand{\large}{\@setfontsize\large\@xiipt{14}} +\renewcommand{\Large}{\@setfontsize\Large\@xivpt{16}} +\renewcommand{\LARGE}{\@setfontsize\LARGE\@xviipt{20}} +\renewcommand{\huge}{\@setfontsize\huge\@xxpt{23}} +\renewcommand{\Huge}{\@setfontsize\Huge\@xxvpt{28}} + + +% Force \tiny to be no smaller than 6pt +\renewcommand{\tiny}{\fontsize{6pt}{7pt}\selectfont} + +% Force \scriptsize to be no smaller than 7pt +\renewcommand{\scriptsize}{\fontsize{7pt}{8pt}\selectfont} + +% Force \footnotesize to be no smaller than 8pt +\renewcommand{\footnotesize}{\fontsize{8pt}{9.5pt}\selectfont} + +% sections with less space +\providecommand{\section}{} +\renewcommand{\section}{% + \@startsection{section}{1}{\z@}% + {-2.0ex \@plus -0.5ex \@minus -0.2ex}% + { 1.5ex \@plus 0.3ex \@minus 0.2ex}% + {\large\bf\raggedright}% +} +\providecommand{\subsection}{} +\renewcommand{\subsection}{% + \@startsection{subsection}{2}{\z@}% + {-1.8ex \@plus -0.5ex \@minus -0.2ex}% + { 0.8ex \@plus 0.2ex}% + {\normalsize\bf\raggedright}% +} +\providecommand{\subsubsection}{} +\renewcommand{\subsubsection}{% + \@startsection{subsubsection}{3}{\z@}% + {-1.5ex \@plus -0.5ex \@minus -0.2ex}% + { 0.5ex \@plus 0.2ex}% + {\normalsize\bf\raggedright}% +} +\providecommand{\paragraph}{} +\renewcommand{\paragraph}{% + \@startsection{paragraph}{4}{\z@}% + {1.5ex \@plus 0.5ex \@minus 0.2ex}% + {-1em}% + {\normalsize\bf}% +} +\providecommand{\subparagraph}{} +\renewcommand{\subparagraph}{% + \@startsection{subparagraph}{5}{\z@}% + {1.5ex \@plus 0.5ex \@minus 0.2ex}% + {-1em}% + {\normalsize\bf}% +} +\providecommand{\subsubsubsection}{} +\renewcommand{\subsubsubsection}{% + \vskip5pt{\noindent\normalsize\rm\raggedright}% +} + +% float placement +\renewcommand{\topfraction }{0.85} +\renewcommand{\bottomfraction }{0.4} +\renewcommand{\textfraction }{0.1} +\renewcommand{\floatpagefraction}{0.7} + +\newlength{\@neuripsabovecaptionskip}\setlength{\@neuripsabovecaptionskip}{7\p@} +\newlength{\@neuripsbelowcaptionskip}\setlength{\@neuripsbelowcaptionskip}{\z@} + +\setlength{\abovecaptionskip}{\@neuripsabovecaptionskip} +\setlength{\belowcaptionskip}{\@neuripsbelowcaptionskip} + +% swap above/belowcaptionskip lengths for tables +\renewenvironment{table} + {\setlength{\abovecaptionskip}{\@neuripsbelowcaptionskip}% + \setlength{\belowcaptionskip}{\@neuripsabovecaptionskip}% + \@float{table}} + {\end@float} + +% footnote formatting +\setlength{\footnotesep }{6.65\p@} +\setlength{\skip\footins}{9\p@ \@plus 4\p@ \@minus 2\p@} +\renewcommand{\footnoterule}{\kern-3\p@ \hrule width 12pc \kern 2.6\p@} +\setcounter{footnote}{0} + +% paragraph formatting +\setlength{\parindent}{\z@} +\setlength{\parskip }{5.5\p@} + +% list formatting +\setlength{\topsep }{4\p@ \@plus 1\p@ \@minus 2\p@} +\setlength{\partopsep }{1\p@ \@plus 0.5\p@ \@minus 0.5\p@} +\setlength{\itemsep }{2\p@ \@plus 1\p@ \@minus 0.5\p@} +\setlength{\parsep }{2\p@ \@plus 1\p@ \@minus 0.5\p@} +\setlength{\leftmargin }{3pc} +\setlength{\leftmargini }{\leftmargin} +\setlength{\leftmarginii }{2em} +\setlength{\leftmarginiii}{1.5em} +\setlength{\leftmarginiv }{1.0em} +\setlength{\leftmarginv }{0.5em} +\def\@listi {\leftmargin\leftmargini} +\def\@listii {\leftmargin\leftmarginii + \labelwidth\leftmarginii + \advance\labelwidth-\labelsep + \topsep 2\p@ \@plus 1\p@ \@minus 0.5\p@ + \parsep 1\p@ \@plus 0.5\p@ \@minus 0.5\p@ + \itemsep \parsep} +\def\@listiii{\leftmargin\leftmarginiii + \labelwidth\leftmarginiii + \advance\labelwidth-\labelsep + \topsep 1\p@ \@plus 0.5\p@ \@minus 0.5\p@ + \parsep \z@ + \partopsep 0.5\p@ \@plus 0\p@ \@minus 0.5\p@ + \itemsep \topsep} +\def\@listiv {\leftmargin\leftmarginiv + \labelwidth\leftmarginiv + \advance\labelwidth-\labelsep} +\def\@listv {\leftmargin\leftmarginv + \labelwidth\leftmarginv + \advance\labelwidth-\labelsep} +\def\@listvi {\leftmargin\leftmarginvi + \labelwidth\leftmarginvi + \advance\labelwidth-\labelsep} + +% create title +\providecommand{\maketitle}{} +\renewcommand{\maketitle}{% + \par + \begingroup + \renewcommand{\thefootnote}{\fnsymbol{footnote}} + % for perfect author name centering + \renewcommand{\@makefnmark}{\hbox to \z@{$^{\@thefnmark}$\hss}} + % The footnote-mark was overlapping the footnote-text, + % added the following to fix this problem (MK) + \long\def\@makefntext##1{% + \parindent 1em\noindent + \hbox to 1.8em{\hss $\m@th ^{\@thefnmark}$}##1 + } + \thispagestyle{empty} + \@maketitle + \@thanks + \@notice + \endgroup + \let\maketitle\relax + \let\thanks\relax +} + +% rules for title box at top of first page +\newcommand{\@toptitlebar}{ + \hrule height 4\p@ + \vskip 0.25in + \vskip -\parskip% +} +\newcommand{\@bottomtitlebar}{ + \vskip 0.29in + \vskip -\parskip + \hrule height 1\p@ + \vskip 0.09in% +} + +% create title (includes both anonymized and non-anonymized versions) +\providecommand{\@maketitle}{} +\renewcommand{\@maketitle}{% + \vbox{% + \hsize\textwidth + \linewidth\hsize + \vskip 0.1in + \@toptitlebar + \centering + {\LARGE\bf \@title\par} + \@bottomtitlebar + \if@anonymous + \begin{tabular}[t]{c}\bf\rule{\z@}{24\p@} + Anonymous Author(s) \\ + Affiliation \\ + Address \\ + \texttt{email} \\ + \end{tabular}% + \else + \def\And{% + \end{tabular}\hfil\linebreak[0]\hfil% + \begin{tabular}[t]{c}\bf\rule{\z@}{24\p@}\ignorespaces% + } + \def\AND{% + \end{tabular}\hfil\linebreak[4]\hfil% + \begin{tabular}[t]{c}\bf\rule{\z@}{24\p@}\ignorespaces% + } + \begin{tabular}[t]{c}\bf\rule{\z@}{24\p@}\@author\end{tabular}% + \fi + \vskip 0.3in \@minus 0.1in + } +} + +% add conference notice to bottom of first page +\newcommand{\ftype@noticebox}{8} +\newcommand{\@notice}{% + % give a bit of extra room back to authors on first page + \enlargethispage{2\baselineskip}% + \@float{noticebox}[b]% + \footnotesize\@noticestring% + \end@float% +} + +% abstract styling +\renewenvironment{abstract}% +{% + \vskip 0.075in% + \centerline% + {\large\bf Abstract}% + \vspace{0.5ex}% + \begin{quote}% +} +{ + \par% + \end{quote}% + \vskip 1ex% +} + +% For the paper checklist +\newcommand{\answerYes}[1][]{\textcolor{black}{[Yes]#1}} +\newcommand{\answerNo}[1][]{\textcolor{orange}{[No]#1}} +\newcommand{\answerNA}[1][]{\textcolor{gray}{[N/A]#1}} +\newcommand{\answerTODO}[1][]{\textcolor{red}{\bf [TODO]}} +\newcommand{\justificationTODO}[1][]{\textcolor{red}{\bf [TODO]}} + +% handle tweaks for camera-ready copy vs. submission copy +\if@preprint + \newcommand{\@noticestring}{% + Preprint.% + } +\else + \if@neuripsfinal + \newcommand{\@noticestring}{ + \@trackname + } + \else + \newcommand{\@noticestring}{% + Submitted to \@neuripsordinal\/ Conference on Neural Information Processing Systems (NeurIPS \@neuripsyear). Do not distribute.% + } + + % hide the acknowledgements + \NewEnviron{hide}{} + \let\ack\hide + \let\endack\endhide + + % line numbers for submission + \RequirePackage{lineno} + \linenumbers + + % fix incompatibilities between lineno and amsmath, if required, by + % transparently wrapping linenomath environments around amsmath + % environments + \AtBeginDocument{% + \@ifpackageloaded{amsmath}{% + \newcommand*\patchAmsMathEnvironmentForLineno[1]{% + \expandafter\let\csname old#1\expandafter\endcsname\csname #1\endcsname + \expandafter\let\csname oldend#1\expandafter\endcsname\csname end#1\endcsname + \renewenvironment{#1}% + {\linenomath\csname old#1\endcsname}% + {\csname oldend#1\endcsname\endlinenomath}% + }% + \newcommand*\patchBothAmsMathEnvironmentsForLineno[1]{% + \patchAmsMathEnvironmentForLineno{#1}% + \patchAmsMathEnvironmentForLineno{#1*}% + }% + \patchBothAmsMathEnvironmentsForLineno{equation}% + \patchBothAmsMathEnvironmentsForLineno{align}% + \patchBothAmsMathEnvironmentsForLineno{flalign}% + \patchBothAmsMathEnvironmentsForLineno{alignat}% + \patchBothAmsMathEnvironmentsForLineno{gather}% + \patchBothAmsMathEnvironmentsForLineno{multline}% + } + {} + } + \fi +\fi + + +\endinput diff --git a/paper/iclr_source/references.bib b/paper/iclr_source/references.bib new file mode 100644 index 0000000000000000000000000000000000000000..ab486de31637739b4f2c5c5f3e38dfa2bc897f46 --- /dev/null +++ b/paper/iclr_source/references.bib @@ -0,0 +1,465 @@ +@article{lam2023graphcast, + title = {Learning skillful medium-range global weather forecasting}, + author = {Lam, Remi and Sanchez-Gonzalez, Alvaro and Willson, Matthew and Wirnsberger, Peter and Fortunato, Meire and Alet, Ferran and Ravuri, Suman and Ewalds, Timo and Eaton-Rosen, Zach and Hu, Weihua and others}, + journal = {Science}, + volume = {382}, + number = {6677}, + pages = {1416--1421}, + year = {2023} +} + +@article{ebert2009neighborhood, + title = {Neighborhood Verification: A Strategy for Rewarding Close Forecasts}, + author = {Ebert, Elizabeth E.}, + journal = {Weather and Forecasting}, + volume = {24}, + number = {6}, + pages = {1498--1510}, + year = {2009}, + doi = {10.1175/2009WAF2222251.1} +} + +@inproceedings{ronneberger2015unet, + title = {U-Net: Convolutional Networks for Biomedical Image Segmentation}, + author = {Ronneberger, Olaf and Fischer, Philipp and Brox, Thomas}, + booktitle = {Medical Image Computing and Computer-Assisted Intervention}, + pages = {234--241}, + year = {2015}, + doi = {10.1007/978-3-319-24574-4_28} +} + +@misc{noaa_hrrr_ncei, + title = {{Rapid Refresh / High-Resolution Rapid Refresh}}, + author = {{NOAA National Centers for Environmental Information}}, + year = {2026}, + howpublished = {\url{https://www.ncei.noaa.gov/products/weather-climate-models/rapid-refresh-update}}, + note = {Accessed: 2026-05-05} +} + +@misc{noaa_hrrr_emc, + title = {{High-Resolution Rapid Refresh (HRRR)}}, + author = {{NOAA National Centers for Environmental Prediction Environmental Modeling Center}}, + year = {2026}, + howpublished = {\url{https://rapidrefresh.noaa.gov/hrrr/}}, + note = {Accessed: 2026-05-05} +} + +@misc{nasa_firms, + title = {{Fire Information for Resource Management System (FIRMS)}}, + author = {{NASA Earthdata}}, + year = {2026}, + howpublished = {\url{https://www.earthdata.nasa.gov/data/tools/firms}}, + note = {Accessed: 2026-05-05} +} + +@misc{landfire_fbfm40, + title = {{LANDFIRE 40 Fire Behavior Fuel Models}}, + author = {{LANDFIRE}}, + year = {2026}, + howpublished = {\url{https://landfire.gov/fuel/fbfm40}}, + note = {Accessed: 2026-05-05} +} + +@misc{landfire_canopy_cover, + title = {{LANDFIRE Forest Canopy Cover}}, + author = {{LANDFIRE}}, + year = {2026}, + howpublished = {\url{https://landfire.gov/fuel/cc}}, + note = {Accessed: 2026-05-05} +} + +@misc{usfs_wrc_housing_density, + title = {{Wildfire Risk to Communities: Housing Unit Density Image Service}}, + author = {{USDA Forest Service}}, + year = {2026}, + howpublished = {\url{https://catalog.data.gov/dataset/wildfire-risk-to-communities-housing-unit-density-image-service-fac22}}, + note = {Accessed: 2026-05-05} +} + +@misc{ornl_landscan_2024, + title = {{LandScan Global 2024}}, + author = {{Oak Ridge National Laboratory}}, + year = {2024}, + howpublished = {\url{https://landscan.ornl.gov/}}, + note = {Accessed: 2026-05-05} +} + +@misc{nifc_wfigs_perimeters, + title = {{Wildland Fire Interagency Geospatial Services (WFIGS): Current Perimeters}}, + author = {{National Interagency Fire Center}}, + year = {2026}, + howpublished = {\url{https://data-nifc.opendata.arcgis.com/datasets/nifc::wfigs-current-perimeters/about}}, + note = {Accessed: 2026-05-05} +} + +@misc{mtbs_usgs_2025, + title = {{Monitoring Trends in Burn Severity (MTBS)}}, + author = {{U.S. Geological Survey and USDA Forest Service}}, + year = {2025}, + howpublished = {\url{https://www.mtbs.gov/}}, + note = {Accessed: 2026-05-05} +} + +@article{pickell2017early, + title={An early warning system to forecast the close of the spring burning window from satellite-observed greenness}, + author={Pickell, Paul D and Coops, Nicholas C and Ferster, Colin J and Bater, Christopher W and Blouin, Karen D and Flannigan, Mike D and Zhang, Jinkai}, + journal={Scientific Reports}, + volume={7}, + number={1}, + pages={14190}, + year={2017}, + publisher={Nature Publishing Group} +} + +@article{khosravi2025assessing, + title={Assessing Pan-Canada wildfire susceptibility by integrating satellite data with novel hybrid deep learning and black widow optimizer algorithms}, + author={Khosravi, Khabat and Mosallanejad, Ashkan and Bateni, Sayed M. and Kim, Dongkyun and Jun, Changhyun and Shahvaran, Ali Reza and Farooque, Aitazaz A. and Karbasi, Massoud and Ali, Mumtaz}, + journal={Science of the Total Environment}, + volume={977}, + year={2025}, + publisher={Elsevier}, + doi={10.1016/j.scitotenv.2025.179369} +} + +@article{vetrita2025drought, + title={Drought and Fine Fuel Moisture Code Evaluation: An Early Warning System for Forest/Land Fire Using Remote Sensing Approach}, + author={Vetrita, Yenni and Prasasti, Indah and Haryani, Nanik Suryo and Priyatna, M. and Khomarudin, M. Rokhis}, + journal={International Journal of Remote Sensing and Earth Sciences}, + volume={9}, + number={2}, + year={2025}, + doi={10.30536/ijreses.v9i2.13954} +} + +@inproceedings{xu2026bcwildfire, + title={BCWildfire: A Long-term Multi-factor Dataset and Deep Learning Benchmark for Boreal Wildfire Risk Prediction}, + author={Xu, Zhengsen and Cheng, Sibo and Wang, Lanying and He, Hongjie and Sun, Wentao and Li, Jonathan and Xu, Lincoln Linlin}, + booktitle={Proceedings of the AAAI Conference on Artificial Intelligence}, + volume={40}, + number={46}, + pages={39486--39494}, + year={2026}, + doi={10.1609/aaai.v40i46.41299} +} + +@article{krstinic2026spatio, + title={Spatio-temporal data model for early wildfire detection}, + author={Krstini{\'c}, Damir and Bejo, Jakov and Sikora, Toma and Bugari{\'c}, Marin}, + journal={Fire}, + volume={9}, + number={4}, + pages={175}, + year={2026}, + doi={10.3390/fire9040175} +} + +@article{kotroni2020disarm, + title={DISARM Early Warning System for Wildfires in the Eastern Mediterranean}, + author={Kotroni, Vassiliki and Cartalis, Constantinos and Michaelides, Silas and Stoyanova, Julia and Tymvios, Filippos and Bezes, Antonis and Christoudias, Theodoros and Dafis, Stavros and Giannakopoulos, Christos and Giannaros, Theodore M. and others}, + journal={Sustainability}, + volume={12}, + number={16}, + pages={6670}, + year={2020}, + publisher={MDPI}, + doi={10.3390/su12166670} +} + +@article{ts2025satfire, + title={TS-SatFire: A Multi-Task Satellite Image Time-Series Dataset for Wildfire Detection and Prediction}, + author={TS-SatFire Consortium}, + journal={Scientific Data}, + volume={12}, + pages={1817}, + year={2025}, + doi={10.1038/s41597-025-06271-3} +} + +@article{nujjoo2025modelling, + title={Modelling Spatial-Temporal Wildfire Susceptibility Using Geospatial Techniques Over the Table Mountain Nature Reserve, South Africa}, + author={Nujjoo, Syed Tanweer Raza and Odera, Patroba Achola}, + journal={Geoplanning: Journal of Geomatics and Planning}, + volume={12}, + number={2}, + pages={197--214}, + year={2025}, + doi={10.14710/geoplanning.12.2.197-214} +} + +@article{singh2025benchmarking, + title={Benchmarking Artificial Neural Networks and U-Net Convolutional Architectures for Wildfire Susceptibility Prediction: Innovations in Geospatial Intelligence}, + author={Singh, Harikesh and Ang, Li-Minn and Srivastava, Sanjeev Kumar}, + journal={IEEE Transactions on Geoscience and Remote Sensing}, + volume={63}, + year={2025}, + publisher={IEEE}, + doi={10.1109/TGRS.2025.3529134} +} + +@inproceedings{goldammer1999early, + title={Early warning systems for the prediction of and appropriate response to wildfires and related environmental hazards}, + author={Goldammer, Johann Georg}, + booktitle={Early Warning Systems for Natural Disaster Reduction}, + year={1999} +} + +@article{huot2022nextday, + title={Next day wildfire prediction using deep learning}, + author={Huot, Fantine and Hu, R Lily and Goyal, Nita and Sankar, Tharun and Ihme, Matthias and Chen, Yi-Fan}, + journal={arXiv preprint arXiv:2206.08930}, + year={2022} +} + +@inproceedings{radke2019firecast, + title={FireCast: Leveraging deep learning to predict wildfire spread}, + author={Radke, David and Hessler, Anna and Ellsworth, David}, + booktitle={Proceedings of the 28th International Joint Conference on Artificial Intelligence}, + pages={4575--4581}, + year={2019} +} + +@article{nguyen2023climax, + title={ClimaX: A foundation model for weather and climate}, + author={Nguyen, Tung and Brandstetter, Johannes and Kapoor, Aditya and Gupta, Jayesh K and Grover, Aditya}, + journal={arXiv preprint arXiv:2301.10343}, + year={2023} +} + +@article{reed2023scalemae, + title={Scale-MAE: A scale-aware masked autoencoder for multiscale geospatial representation learning}, + author={Reed, Colorado J and Gupta, Ritwik and Li, Shufan and Brockman, Sarah and Funk, Christopher and Clipp, Brian and Keutzer, Kurt and Ermon, Stefano and Salakhutdinov, Ruslan}, + journal={arXiv preprint arXiv:2212.14532}, + year={2023} +} + +@article{bi2023panguweather, + title={Pangu-Weather: A 3D high-resolution model for fast and accurate global weather forecast}, + author={Bi, Kaifeng and Xie, Lingxi and Zhang, Hengheng and Chen, Xin and Gu, Xiaotao and Tian, Qi}, + journal={Nature}, + volume={619}, + number={7970}, + pages={533--538}, + year={2023}, + publisher={Nature Publishing Group} +} + +@article{cong2022satmae, + title={SatMAE: Pre-training transformers for temporal and multi-spectral satellite imagery}, + author={Cong, Yezhen and Khanna, Samar and Meng, Chenlin and Liu, Patrick and Rozi, Erik and He, Yutong and Burke, Marshall and Lobell, David and Ermon, Stefano}, + journal={Advances in Neural Information Processing Systems}, + volume={35}, + pages={197--211}, + year={2022} +} + +@article{guo2024skysense, + title={SkySense: A multi-modal remote sensing foundation model towards universal interpretation}, + author={Guo, Xin and Lao, Jianwei and Dang, Bo and Zhang, Yingying and Yu, Lei and Zhang, Ruixiang and Zhan, Siyu and Li, Wei and Hao, Yonggang and Zhang, Shuai and others}, + journal={arXiv preprint arXiv:2403.11916}, + year={2024} +} + +@article{bodnar2025aurora, + title={Aurora: A foundation model of the atmosphere}, + author={Bodnar, Cristian and others}, + journal={arXiv preprint arXiv:2405.13063}, + year={2024} +} + +@article{schmude2024prithviwxc, + title={Prithvi WxC: Foundation model for weather and climate}, + author={Schmude, Johannes and Roy, Sujit and Trofimova, Paulina and Ramesh, Karthik and Lusch, Bethany and Kesa, Harikumar and Singh, Shraddha and Chen, Phil and Liu, Zhuohan and Parashar, Shubhankar and others}, + journal={arXiv preprint arXiv:2409.13598}, + year={2024} +} + +@article{farahmand2020fdeo, + title={Introducing spatially distributed fire danger from earth observations (FDEO) using satellite-based data in the contiguous United States}, + author={Farahmand, Alireza and Stavros, E Natasha and Reager, John T and Behrangi, Ali}, + journal={Remote Sensing}, + volume={12}, + number={8}, + pages={1252}, + year={2020}, + publisher={MDPI} +} + +@article{gilleland2009intercomparison, + title={Intercomparison of spatial forecast verification methods}, + author={Gilleland, Eric and Ahijevych, David and Brown, Barbara G and Ebert, Elizabeth E}, + journal={Weather and Forecasting}, + volume={24}, + number={5}, + pages={1416--1430}, + year={2009}, + publisher={American Meteorological Society} +} + +@article{gilleland2009spatialverification, + title = {Intercomparison of Spatial Forecast Verification Methods}, + author = {Gilleland, Eric and Ahijevych, David and Brown, Barbara G. and Casati, Barbara and Ebert, Elizabeth E.}, + journal = {Weather and Forecasting}, + volume = {24}, + number = {5}, + pages = {1416--1430}, + year = {2009}, + doi = {10.1175/2009WAF2222269.1} +} + +@article{bi2023pangu, + title = {Accurate medium-range global weather forecasting with 3D neural networks}, + author = {Bi, Kaifeng and Xie, Lingxi and Zhang, Hengheng and Chen, Xin and Gu, Xiaotao and Tian, Qi}, + journal = {Nature}, + volume = {619}, + pages = {533--538}, + year = {2023} +} + +@inproceedings{reed2023scale, + title = {Scale-{MAE}: A Scale-Aware Masked Autoencoder for Multiscale Geospatial Representation Learning}, + author = {Reed, Colorado J. and Gupta, Ritwik and Li, Shufan and Brockman, Sarah and Funk, Christopher and Clipp, Brian and Keutzer, Kurt and Candido, Salvatore and Uyttendaele, Matt and Darrell, Trevor}, + booktitle = {Proceedings of the IEEE/CVF International Conference on Computer Vision}, + pages = {4088--4099}, + year = {2023} +} + +@inproceedings{mcdermott2024aurocauprc, + title = {A Closer Look at {AUROC} and {AUPRC} under Class Imbalance}, + author = {McDermott, Matthew B. and Zhang, Haoran and Hansen, Lasse Hyldig and Angelotti, Giovanni and Gallifant, Jack}, + booktitle = {Advances in Neural Information Processing Systems}, + year = {2024} +} + +@inproceedings{lacoste2023geobench, + title = {{GEO-Bench}: Toward Foundation Models for Earth Monitoring}, + author = {Lacoste, Alexandre and Lehmann, Nils and Rodriguez, Pau and Sherwin, Evan and Kerner, Hannah and L{\"u}tjens, Bj{\"o}rn and Irvin, Jeremy and Dao, David and Alemohammad, Hamed and Drouin, Alexandre and others}, + booktitle = {Advances in Neural Information Processing Systems}, + year = {2023} +} + +@article{rasp2024weatherbench2, + title = {{WeatherBench} 2: A benchmark for the next generation of data-driven global weather models}, + author = {Rasp, Stephan and Hoyer, Stephan and Merose, Alex and Langmore, Ian and Battaglia, Peter and Russell, Tyler and Sanchez-Gonzalez, Alvaro and Yang, Vivian and Carver, Rob and Agrawal, Shreya and others}, + journal = {Journal of Advances in Modeling Earth Systems}, + volume = {16}, + number = {6}, + pages = {e2023MS004019}, + year = {2024} +} + +@inproceedings{koh2021wilds, + title = {{WILDS}: A benchmark of in-the-wild distribution shifts}, + author = {Koh, Pang Wei and Sagawa, Shiori and Marklund, Henrik and Xie, Sang Michael and Zhang, Marvin and Balsubramani, Akshay and Hu, Weihua and Yasunaga, Michihiro and Phillips, Richard Lanas and Gao, Irena and others}, + booktitle = {Proceedings of the International Conference on Machine Learning}, + pages = {5637--5664}, + year = {2021} +} + +@inproceedings{yeh2021sustainbench, + title = {{SustainBench}: Benchmarks for Monitoring the Sustainable Development Goals with Machine Learning}, + author = {Yeh, Christopher and Meng, Chenlin and Wang, Sijing and Driscoll, Anne and Rozi, Erik and Liu, Peng and Lee, Jae Yong and Burke, Marshall and Lobell, David B. and Ermon, Stefano}, + booktitle = {Advances in Neural Information Processing Systems}, + year = {2021} +} + +@inproceedings{torchgeo2022, + title = {TorchGeo: Deep Learning With Geospatial Data}, + author = {Stewart, Adam J. and Robinson, Caleb and Corley, Isaac A. and Ortiz, Anthony and Lavista Ferres, Juan M. and Banerjee, Arindam}, + booktitle = {Proceedings of the ACM SIGSPATIAL International Conference on Advances in Geographic Information Systems}, + year = {2022} +} + +@inproceedings{schaeffer2023mirage, + title = {Are Emergent Abilities of Large Language Models a Mirage?}, + author = {Schaeffer, Rylan and Miranda, Brando and Koyejo, Sanmi}, + booktitle = {Advances in Neural Information Processing Systems}, + year = {2023} +} + +@inproceedings{luth2023activelearning, + title = {Navigating the Pitfalls of Active Learning Evaluation: A Systematic Framework for Meaningful Performance Assessment}, + author = {L{\"u}th, Carsten T. and Bungert, Till J. and Klein, Lukas and J{\"a}ger, Paul F.}, + booktitle = {Advances in Neural Information Processing Systems}, + year = {2023} +} + +@inproceedings{traub2024selectiveclassification, + title = {Overcoming Common Flaws in the Evaluation of Selective Classification Systems}, + author = {Traub, Jeremias and Bungert, Till J. and L{\"u}th, Carsten T. and Baumgartner, Michael and Maier-Hein, Klaus H. and Maier-Hein, Lena and J{\"a}ger, Paul F.}, + booktitle = {Advances in Neural Information Processing Systems}, + year = {2024} +} + + +# new---4.30 +@article{pathak2024stormcast, + title={Kilometer-scale convection-allowing model emulation using generative diffusion modeling}, + author={Pathak, Jaideep and Cohen, Yair and Garg, Piyush and Harrington, Peter and Brenowitz, Noah and Durran, Dale and Mardani, Morteza and Vahdat, Arash and Xu, Shaoming and Kashinath, Karthik and others}, + journal={Science Advances}, + volume={12}, + number={5}, + pages={eadv0423}, + year={2026}, + publisher={American Association for the Advancement of Science} +} + +@article{brown2025alphaearth, + title={Alphaearth foundations: An embedding field model for accurate and efficient global mapping from sparse label data}, + author={Brown, Christopher F and Kazmierski, Michal R and Pasquarella, Valerie J and Rucklidge, William J and Samsikova, Masha and Zhang, Chenhui and Shelhamer, Evan and Lahera, Estefania and Wiles, Olivia and Ilyushchenko, Simon and others}, + journal={arXiv preprint arXiv:2507.22291}, + year={2025} +} + +@article{weyn2020dlwp, + title={Improving data-driven global weather prediction using deep convolutional neural networks on a cubed sphere}, + author={Weyn, Jonathan A and Durran, Dale R and Caruana, Rich}, + journal={Journal of Advances in Modeling Earth Systems}, + volume={12}, + number={9}, + pages={e2020MS002109}, + year={2020}, + publisher={Wiley Online Library} +} + +@article{pathak2022fourcastnet, + title={Fourcastnet: A global data-driven high-resolution weather model using adaptive fourier neural operators}, + author={Pathak, Jaideep and Subramanian, Shashank and Harrington, Peter and Raja, Sanjeev and Chattopadhyay, Ashesh and Mardani, Morteza and Kurth, Thorsten and Hall, David and Li, Zongyi and Azizzadenesheli, Kamyar and others}, + journal={arXiv preprint arXiv:2202.11214}, + year={2022} +} + +@article{chen2023fengwu, + title={Fengwu: Pushing the skillful global medium-range weather forecast beyond 10 days lead}, + author={Chen, Kang and Han, Tao and Gong, Junchao and Bai, Lei and Ling, Fenghua and Luo, Jing-Jia and Chen, Xi and Ma, Leiming and Zhang, Tianning and Su, Rui and others}, + journal={arXiv preprint arXiv:2304.02948}, + year={2023} +} + +@article{chen2023fuxi, + title={FuXi: A cascade machine learning forecasting system for 15-day global weather forecast}, + author={Chen, Lei and Zhong, Xiaohui and Zhang, Feng and Cheng, Yuan and Xu, Yinghui and Qi, Yuan and Li, Hao}, + journal={npj climate and atmospheric science}, + volume={6}, + number={1}, + pages={190}, + year={2023}, + publisher={Nature Publishing Group UK London} +} + +@article{marsocci2024pangaea, + title={Pangaea: A global and inclusive benchmark for geospatial foundation models}, + author={Marsocci, Valerio and Jia, Yuru and Bellier, Georges Le and Kerekes, David and Zeng, Liang and Hafner, Sebastian and Gerard, Sebastian and Brune, Eric and Yadav, Ritu and Shibli, Ali and others}, + journal={arXiv preprint arXiv:2412.04204}, + year={2024} +} + +@article{gerard2023wildfirespreadts, + title={Wildfirespreadts: A dataset of multi-modal time series for wildfire spread prediction}, + author={Gerard, Sebastian and Zhao, Yu and Sullivan, Josephine}, + journal={Advances in Neural Information Processing Systems}, + volume={36}, + pages={74515--74529}, + year={2023} +} + + diff --git a/paper/iclr_source/sections/0_abstract.tex b/paper/iclr_source/sections/0_abstract.tex new file mode 100644 index 0000000000000000000000000000000000000000..aa547939b73856b32027b8e922e6f84fa3302f81 --- /dev/null +++ b/paper/iclr_source/sections/0_abstract.tex @@ -0,0 +1,4 @@ +\begin{abstract} +Wildfire prediction is important for early warning and resource allocation, yet existing Earth foundation models (Earth FMs) are pretrained for general atmospheric and geophysical objectives rather than wildfire forecasting. To address this gap, we study \ourfm, a wildfire-specialized reference backbone trained on a Lower-48 5 km input contract that combines weather, active-fire supervision, fuel, canopy, validity, and static environmental context. However, introducing a domain-specific backbone alone does not solve the evaluation problem: wildfire events are sparse in space and time, making transfer conclusions highly sensitive to matching rules and evaluation settings. +To address this problem, we introduce a fixed-contract evaluation framework with two controlled checks: a fixed-output check for matching-rule effects and a fixed-feature check for head-selection effects. Under matched contracts, we compare \ourfm\ with ten Earth-FM baselines across occupancy, spread, retrieval, and regression tasks. Our results show that wildfire transfer conclusions depend strongly on evaluation design and task formulation. We hope this framework and \ourfm\ provide a basis for future wildfire-specific Earth-FM research and benchmarking. +\end{abstract} diff --git a/paper/iclr_source/sections/1_intro.tex b/paper/iclr_source/sections/1_intro.tex new file mode 100644 index 0000000000000000000000000000000000000000..c49745a0a41291a631802839a872838174e58f08 --- /dev/null +++ b/paper/iclr_source/sections/1_intro.tex @@ -0,0 +1,77 @@ +\section{Introduction} + +Wildfire prediction is critical for early warning and resource allocation in disaster response~\cite{goldammer1999early, farahmand2020fdeo}. As extreme fire events grow more frequent and severe, accurate forecasting of wildfire occurrence and spread is becoming increasingly important~\cite{pickell2017early, kotroni2020disarm}. Recent Earth foundation models (Earth FMs), pretrained on large-scale atmospheric and geophysical data~\cite{bodnar2025aurora, schmude2024prithviwxc, nguyen2023climax}, provide transferable representations for Earth-system dynamics and have shown strong performance across weather and remote-sensing tasks. However, wildfire dynamics depend on complex interactions among weather, vegetation, topography, fuel conditions, and active-fire behavior, which are not explicitly modeled during pretraining in existing general-purpose Earth FMs. This mismatch raises a natural question: can representations learned for general atmospheric or geophysical objectives transfer reliably to wildfire forecasting, and how should we measure that transfer? + +Answering this question requires solving two intertwined problems. The first is that existing Earth FMs are not pretrained specifically for wildfire dynamics, but instead adapted to wildfire tasks after general-purpose pretraining. To address this limitation, we use \textbf{\ourfm} as a wildfire-specialized reference backbone trained directly for gridded active-fire occupancy using fire-relevant multimodal data, including Lower-48 HRRR weather, FIRMS active-fire supervision, fuel, canopy, validity masks, and static environmental context. By learning from a fixed wildfire input contract rather than from general atmospheric or geophysical objectives alone, \ourfm\ provides an in-domain anchor for evaluating transfer to wildfire decision tasks. + +The second problem is evaluation: even with a domain-specific model, reliably comparing \ourfm\ against transferred general-purpose Earth FMs remains difficult. Wildfire events are sparse in space and time~\cite{ebert2009neighborhood, gilleland2009intercomparison}, making transfer conclusions highly sensitive to three sources of evaluation variability. +\textit{First,} matching rules determine what counts as a correct prediction. Early warning systems tolerate spatial offsets that post-fire damage assessment cannot, so different matching rules can produce substantially different F1 scores from the same model outputs~\cite{ebert2009neighborhood, gilleland2009intercomparison}. +\textit{Second,} head-selection metrics determine which lightweight adapter is chosen on top of a frozen representation. Ranking metrics such as PR-AUC and decision metrics such as F1 can favor different heads from the same frozen features~\cite{mcdermott2024aurocauprc, traub2024selectiveclassification}. +\textit{Third,} wildfire task forms operate over different prediction units and metric families. Occupancy prediction, spread forecasting, burned-area regression, and analog retrieval therefore produce scores that are not directly comparable even under the same backbone~\cite{schaeffer2023mirage, gerard2023wildfirespreadts}. +Related protocol sensitivity has also been observed in active learning~\cite{luth2023activelearning} and selective classification~\cite{traub2024selectiveclassification}. We show that these effects become particularly severe in wildfire transfer evaluation, where sparse events and heterogeneous task forms amplify evaluation instability~\cite{marsocci2024pangaea}. + +\begin{figure*} + \centering + \includegraphics[width=0.96\linewidth]{figures/overview_wildfire.pdf} + \caption{Overview of \textbf{\ourfm} and \textbf{Evaluation Protocol} in this paper.} + \vspace{-8mm} + \label{fig:overview} +\end{figure*} + + +This evaluation instability makes reliable comparison between +\ourfm\ and existing Earth FMs fundamentally difficult. Standard +geospatial benchmarks such as GEO-Bench~\cite{lacoste2023geobench}, +WeatherBench2~\cite{rasp2024weatherbench2}, WILDS~\cite{koh2021wilds}, +SustainBench~\cite{yeh2021sustainbench}, and +TorchGeo~\cite{torchgeo2022} standardize datasets, splits, and +metrics, but do so for tasks with dense, balanced labels where +matching-rule sensitivity is not a primary concern. Wildfire +studies such as FireCast~\cite{radke2019firecast} and Next Day +Wildfire Spread~\cite{huot2022nextday} apply the same +report-and-compare paradigm directly to sparse fire events +without controlling for matching-rule choice, head-selection +metric, or task-form comparability, the three sources of +instability identified above. As a result, scores reported under +different implicit protocol choices are not directly comparable, +even when the underlying predictions are identical. + +Based on the \textit{limitations of prior work}, our contributions are as follows (see Figure~\ref{fig:overview}). +\begin{itemize} + +\item \textbf{Wildfire-specialized reference backbone.} +We study \ourfm as an in-domain backbone trained directly +for wildfire prediction using multimodal wildfire data +spanning weather, active-fire supervision, fuel, canopy, +validity, and static environmental context. Unlike general +Earth FMs adapted after pretraining, \ourfm exposes a fixed +wildfire input contract and serves as an empirical anchor for +same-contract transfer comparisons. + +\item \textbf{Fixed-contract evaluation framework.} +We formulate wildfire Earth-FM transfer as a fixed-contract +evaluation problem, defining a contract +$\mathcal{C} = (\mathcal{T}, M, \Lambda, \Omega, \mathcal{A})$ +that specifies the task, metric, matching rule, evaluation +scope, and lightweight-head family before comparison. We +introduce two controlled checks: a \emph{fixed-output check} +for matching-rule effects and a \emph{fixed-feature check} +for head-selection effects, enabling evaluation artifacts to +be separated from representation quality. + +\item \textbf{Systematic wildfire transfer benchmark.} +Under fixed contracts, we compare \ourfm\ against ten +general-purpose Earth FMs across six wildfire task forms. +Our results show that wildfire transfer conclusions are +highly sensitive to evaluation design and strongly +task-dependent across occupancy, spread, retrieval, and +regression settings. + +\end{itemize} + +% \begin{figure*} +% \centering +% \includegraphics[width=\linewidth]{figures/overview_wildfire.pdf} +% \caption{Overview of \textbf{\ourfm} and \textbf{Evaluation Protocol} in this paper.} +% \label{fig:overview} +% \end{figure*} diff --git a/paper/iclr_source/sections/2_backbone.tex b/paper/iclr_source/sections/2_backbone.tex new file mode 100644 index 0000000000000000000000000000000000000000..e2be21a5da8c4f39849b9155dd09138c2a42855e --- /dev/null +++ b/paper/iclr_source/sections/2_backbone.tex @@ -0,0 +1,41 @@ +\section{\ourfm Reference Backbone} +\label{Reference_backbone} + +\ourfm is a wildfire-specialized Lower-48 backbone trained on fire-relevant multimodal data for gridded active-fire occupancy prediction. Existing general-purpose Earth FMs are pretrained for atmospheric and geophysical objectives~\cite{lam2023graphcast}, or for remote-sensing objectives~\cite{reed2023scalemae}, so wildfire-relevant information enters only indirectly through those objectives. In contrast, \ourfm is trained with weather, active-fire supervision, fuel, canopy, validity, and static environmental context on a fixed 16-channel 5 km input contract. This design makes \ourfm a task-aligned in-domain reference whose features are shaped by signals directly relevant to fire occurrence and spread. +It also serves as an empirical anchor for interpreting how transferred Earth FMs behave under matched evaluation contracts. This section describes the data resources, input contract, training strategy, and full-map inference rule used to build \ourfm as an in-domain reference backbone. The fixed-contract protocol used to compare it with transferred Earth FMs is defined separately in Section~\ref{sec:eval}. + +\subsection{Data Resources} +We group the resources by their role in the study: dynamic weather inputs, occupancy supervision, static context, and event-level resources for supporting tasks. Source and terms-of-use notes for the external data and model assets used in this study are summarized in Appendix Table~\ref{tab:external_assets_licenses}. + + +\noindent\textbf{Dynamic weather inputs.} +The weather inputs come from NOAA High-Resolution Rapid Refresh (HRRR) fields over the Lower 48 United States~\cite{noaa_hrrr_ncei,noaa_hrrr_emc}. The data are placed on a projected 5 km grid in EPSG:5070. Each time map uses weather fields every 6 hours and predicts wildfire occupancy at a 12-hour lead. The variables include near-surface temperature and dew point, wind, CAPE, surface pressure, boundary-layer height, visibility, precipitation rate, and accumulated precipitation. In the reported input cache, the precipitation slots are retained for interface compatibility but are constant-zero fields. + +\noindent\textbf{Occupancy supervision.} +Wildfire supervision comes from NASA FIRMS active-fire detections~\cite{nasa_firms}. The detections are mapped to the same grid as the weather fields. \ourfm is trained on gridded occupancy labels derived from these detections; FIRMS detections supervise the target and are not used as input channels. This defines the occupancy target used by the reference backbone throughout the primary experiments. + +\noindent\textbf{Static context.} +Static context describes landscape and exposure factors that do not change at the weather time step. These variables are LANDFIRE fire-behavior fuel model~\cite{landfire_fbfm40}, LANDFIRE canopy cover~\cite{landfire_canopy_cover}, Wildfire Risk to Communities housing-unit density~\cite{usfs_wrc_housing_density}, and LandScan population~\cite{ornl_landscan_2024}. Static layers are aligned to the same projected grid using layer-appropriate resampling, with validity masks recording weather support and static-layer support after reprojection. Together, the occupancy input has 16 channels: 10 weather fields, two validity masks, and four static layers. + +\noindent\textbf{Event-level resources.} +Event-level resources are used for supporting burned-area and analog tasks, not as occupancy labels for \ourfm. These resources include WFIGS incident and perimeter attributes~\cite{nifc_wfigs_perimeters} and MTBS burned-area and burn-severity records~\cite{mtbs_usgs_2025}. They provide event-scale outcomes and incident metadata for supporting tasks in the experiments and appendix analyses. + + + +\subsection{Training Strategy} + +\noindent\textbf{Model and data split.} +\ourfm uses a compact U-Net~\cite{ronneberger2015unet} that maps gridded weather and static inputs to wildfire predictions. +The network has 7.77M parameters with group normalization, skip connections, a primary occupancy head, and an auxiliary spatial-support head used only during training. +Data are split by time over 2024 Lower-48 maps: June--August 2024 for training, September 2024 for validation, and October 2024 for testing. +This yields 368 training time maps, 120 validation time maps, and 120 test time maps. +Temporal splitting keeps later fire outcomes out of earlier training periods. + +\noindent\textbf{Fire-aware tile training.} +Training is performed on 32$\times$32 tiles sampled from the time maps. The tiles include positive crops whose active-fire cells can appear anywhere in the crop, plus non-fire context, so the model sees both sparse fire labels and surrounding background conditions. This sampling reduces the dominance of empty cells without removing non-fire examples from the training distribution. Continuous channels are normalized with train-split statistics before sampling, and class-weighted binary cross-entropy is used for the primary occupancy target to further balance sparse positives. + +\noindent\textbf{Spatial-support training objective.} +Wildfire labels can shift by a few grid cells because detections, weather fields, and static layers are aligned on a common grid. To reduce sensitivity to these small displacements during training, the occupancy target is dilated by two grid cells. An auxiliary spatial-support output is trained for the same neighborhood with loss weight 0.5, while held-out diagnostics are computed against the original occupancy labels. The primary loss uses positive-class weight 10 and AdamW optimization. At test time, \ourfm is scored under the same task-specific evaluation contracts as the transferred Earth-FM backbones in Section~\ref{sec:eval}, ensuring matched comparison conditions. + +\noindent\textbf{Full-map inference.} +The model is fully convolutional at inference time. Full-domain probability maps are produced with overlapping windows and phase averaging, rather than by stitching independent training crops. For the no-exposure serving rule, housing-density and population channels are zeroed after train-split normalization, keeping the fixed 16-channel interface while preventing exposure features from directly driving the served occupancy map. diff --git a/paper/iclr_source/sections/3_prelim.tex b/paper/iclr_source/sections/3_prelim.tex new file mode 100644 index 0000000000000000000000000000000000000000..d1f057562abfd6869380dc5e03bf36c94f061863 --- /dev/null +++ b/paper/iclr_source/sections/3_prelim.tex @@ -0,0 +1,84 @@ +\section{Evaluation Design} +\label{sec:eval} + +% Section~\ref{Reference_backbone} establishes \ourfm\ as a wildfire-specialized backbone. +% This section formalizes an evaluation contract and introduces two controlled checks to isolate evaluation effects. + +\subsection{Wildfire Output Records and Fire Sets} + +\paragraph{Output record.} +A wildfire prediction model produces scores over spatial units and forecast times, which are compared against observed fire activity to compute performance. We formalize this comparison as a \emph{wildfire output record} $\mathcal{O} = (S, Y)$, where the score field $S = \{s_{i,t}\}$ contains model scores over spatial units $i$ and times $t$, and the label field $Y = \{y_{i,t}\}$ contains the corresponding observations. For occupancy tasks, $y_{i,t} \in \{0,1\}$ indicates whether fire is observed at $(i,t)$. +\vspace{-0.5em} +\paragraph{Predicted and observed fire sets.} +To evaluate $\mathcal{O}$, the score field is thresholded at $\tau$ to produce a predicted fire set $\hat{P}_\tau = \{(i,t) : s_{i,t} \geq \tau\}$, while the observed fire set is $P = \{(i,t) : y_{i,t} = 1\}$. The pair $(\hat{P}_\tau, P)$ is evaluated under a matching rule. Given a matching rule, true positives (TP), false positives (FP), and false negatives (FN) are computed from matched and unmatched elements, and the decision F1 score is $\text{F1} = 2\text{TP}/(2\text{TP} + \text{FP} + \text{FN})$. The same $(\hat{P}_\tau, P)$ can yield different TP, FP, and FN counts under different matching rules without changing model outputs, motivating the fixed-output check in Section~\ref{sec:checks}. +\vspace{-0.5em} +\paragraph{Matching rules.} +A matching rule specifies when a predicted unit-time pair in $\hat{P}_\tau$ is considered a match to an observed pair in $P$~\cite{ebert2009neighborhood, gilleland2009intercomparison}. Because wildfire applications tolerate different levels of spatial and temporal error, we define three matching rules for occupancy outputs. \textit{(1) Exact matching}: requires agreement in both spatial unit and forecast time. \textit{(2) Tolerated matching}: accepts predictions within a fixed spatial or temporal neighborhood defined by the evaluation contract $\mathcal{C}$. \textit{(3) Union matching}: accepts predictions satisfying either exact or tolerated matching. +% \begin{itemize} +% \item \textbf{Exact matching}: requires agreement in both spatial unit and forecast time. +% \item \textbf{Tolerated matching}: accepts predictions within a fixed spatial or temporal neighborhood defined by the evaluation contract $\mathcal{C}$. +% \item \textbf{Union matching}: accepts predictions satisfying either exact or tolerated matching. +% \end{itemize} +% \vspace{-0.5em} +% +Figure~\ref{fig:toy_occupancy_contract} illustrates these rules for a fixed output. Because the output record is held constant, any score difference is attributed solely to the matching rule. + +\begin{figure} + \centering + \vspace{-2mm} + \includegraphics[width=0.8\linewidth]{figures/matching.pdf} + \vspace{-2mm} + \caption{Matching rules for one fixed occupancy output. + (a) Exact matching counts only same-time, same-cell overlap. + (b) Tolerated matching accepts bounded spatial or temporal offsets. + (c) The union reading counts matches accepted by either rule.} + \vspace{-5mm} + \label{fig:toy_occupancy_contract} +\end{figure} + +\subsection{Evaluation Contract} + +A wildfire transfer score depends not only on the model, but also on the evaluation choices used to compute it~\cite{luth2023activelearning}. Changing the matching rule $\Lambda$, metric $M$, or evaluation scope $\Omega$ changes what the score measures even when model outputs are fixed. + +We define an \emph{evaluation contract} as the tuple +$\mathcal{C} = (\mathcal{T}, M, \Lambda, \Omega, \mathcal{A})$, +where $\mathcal{T}$ denotes the task, $M$ the metric, +$\Lambda$ the matching rule, $\Omega$ the evaluation scope, +and $\mathcal{A}$ the allowed lightweight-head family. +Two transfer scores are comparable only when all five +components are identical. The evaluation scope $\Omega$ is particularly important in wildfire settings. A global scope evaluates the full spatial domain, including many fire-inactive regions that can mask differences between models. A fire-prone scope restricts evaluation to regions with higher historical fire activity. We report both scopes separately rather than averaging across them. Fixed matching-rule, task-form, and scope parameters are reported in Appendix Tables~\ref{tab:app_matching_rule_params}, \ref{tab:app_contract_params_full}, and~\ref{tab:app_scope_params}. + + +\subsection{Task-Form Contracts} +\label{sec:taskforms} +Contract components depend on task form. We distinguish \emph{primary} and \emph{supporting} tasks based on whether they directly evaluate wildfire decisions. Occupancy and fire spread are primary tasks because they evaluate spatial fire outputs under matching or overlap rules. +Retrieval, burned-area regression, smoke PM$_{2.5}$, and extreme heat are supporting tasks because they use different prediction units and metric families. Their results provide complementary evidence rather than direct substitutes for occupancy and spread evaluation~\cite{schaeffer2023mirage}. + +For primary tasks, multiple metrics are reported for the same output under different contracts. For occupancy, exact F1 requires same-cell same-time agreement, tolerated F1 accepts predictions within a spatial or temporal neighborhood, and union F1 accepts predictions satisfying either rule. For fire spread, exact F1 evaluates raster-cell agreement, spatial F1 evaluates region overlap between $\hat{B}$ and $B$~\cite{gilleland2009intercomparison}, and AP summarizes ranking quality across thresholds. These metrics are reported separately rather than aggregated because they measure different aspects of the same prediction task. Figure~\ref{fig:task_contract_tiles} summarizes the contract map across all six task forms. + +\subsection{Controlled Checks} +\label{sec:checks} +\begin{wrapfigure}[19]{r}{0.52\textwidth} +\vspace{-2em} +\centering +\includegraphics[width=\linewidth]{figures/fig_task_contract_tiles.pdf} +\vspace{-1.5em} +\caption{ +Evaluation contract map for the six fixed-contract tasks. +Yellow boxes denote \textcolor[RGB]{255,193,7}{\textbf{primary}} decision tasks; purple boxes denote \textcolor[RGB]{148,103,189}{\textbf{supporting}} tasks. +} +\label{fig:task_contract_tiles} +\vspace{-0.8em} +\end{wrapfigure} +We isolate the two instability sources with two checks. +Each check fixes all contract components except one, so any difference is attributed solely to that component. + +\paragraph{Fixed-output check.} +The fixed-output check isolates matching-rule effects by holding the output record $\mathcal{O} = (S, Y)$ and all other contract components fixed while varying only $\Lambda$. For the same occupancy record, we compute F1 under exact, tolerated, and union matching. Any score difference is therefore attributed solely to the matching rule. If matching rules alone shift F1 by tens of percentage points on the same output, then comparing models under different $\Lambda$ conflates model quality with evaluation design. + +\paragraph{Fixed-feature check and selection regret.} +The fixed-feature check isolates head-selection effects by holding the frozen feature source, $\mathcal{T}$, $\Omega$, $\Lambda$, and candidate head family $\mathcal{H} \subseteq \mathcal{A}$ fixed while varying only the selection metric. Let $R(h)$ denote the ranking score of head $h$ and $D(h)$ its decision score. Ranking-based selection chooses $h_R = \arg\max_{h \in \mathcal{H}} R(h)$, while decision-based selection chooses $h_D = \arg\max_{h \in \mathcal{H}} D(h)$. We define \emph{selection regret} as the decision-score gap incurred by using a ranking metric as a proxy for a decision metric during head selection: $\delta = D(h_D) - D(h_R) \geq 0$ ~\cite{mcdermott2024aurocauprc, traub2024selectiveclassification}. When $\delta > 0$, the ranking metric selects a head with lower decision performance under the same frozen representation, indicating that the observed gap arises from metric misalignment rather than from representation quality.The head family used in fixed-feature comparisons is summarized in Appendix Table~\ref{tab:app_head_architectures}. + + +\paragraph{Fixed-contract transfer comparison.} +After the controlled checks establish that matching-rule and selection-metric effects are non-trivial, Earth-FM backbones are evaluated under a shared contract $\mathcal{C}$. Entries are compared only when they satisfy the same $(\mathcal{T}, M, \Lambda, \Omega, \mathcal{A})$ tuple. Supporting tasks test whether occupancy and spread patterns generalize across task forms and provide additional evidence when transfer orderings are preserved. diff --git a/paper/iclr_source/sections/4_experiments.tex b/paper/iclr_source/sections/4_experiments.tex new file mode 100644 index 0000000000000000000000000000000000000000..292f11e9ee989da82768086565d10a8c79ce6da5 --- /dev/null +++ b/paper/iclr_source/sections/4_experiments.tex @@ -0,0 +1,437 @@ +\section{Experiments} +\label{sec:experiments} + +We address three research questions under the fixed-contract framework defined in Section~\ref{sec:eval}. \textbf{RQ1:} Under fixed outputs, does the matching rule determine whether a wildfire model appears usable? +\textbf{RQ2:} Under fixed features, does ranking-based head selection lose decision performance? +\textbf{RQ3:} Under fixed task contracts, do model comparisons remain consistent across task forms? +\vspace{-0.5em} +\subsection{Experimental Setup} +\paragraph{Task instances.} +We instantiate the six task-form contracts defined in Section~\ref{sec:taskforms}. +Occupancy and fire spread serve as primary tasks because they evaluate spatial fire outputs under matching or overlap rules and align with the decision structure of early warning systems~\cite{goldammer1999early, farahmand2020fdeo}. +The four supporting tasks, \textit{final burned area, analog retrieval, smoke PM$_{2.5}$, and extreme heat}, use different prediction units and metric families; their results bound rather than replace primary decision evidence. + +\paragraph{Compared backbones.} +The frozen Earth-FM comparator set includes Prithvi-WxC~\cite{schmude2024prithviwxc}, Aurora~\cite{bodnar2025aurora}, ClimaX~\cite{nguyen2023climax}, StormCast~\cite{pathak2024stormcast}, DLWP~\cite{weyn2020dlwp}, FCN~\cite{pathak2022fourcastnet}, FengWu~\cite{chen2023fengwu}, FuXi~\cite{chen2023fuxi}, Pangu-Weather~\cite{bi2023panguweather}, and AlphaEarth~\cite{brown2025alphaearth}. +\ourfm\ serves as the wildfire-specialized reference backbone. + +\paragraph{Protocol.} +For each comparison, the contract $\mathcal{C} = (\mathcal{T}, M, \Lambda, \Omega, \mathcal{A})$ is fixed before reporting test scores. +Thresholds and morphology parameters are selected on validation data and held fixed at test time. +Stochastic components are evaluated over five seeds and reported as mean $\pm$ standard deviation; deterministic fixed-output checks have zero seed variance by construction. +Entries outside a fixed contract are omitted from main tables and documented in the appendix. +For error metrics lower is better ($\downarrow$); for F1, AP, nDCG, and correlation metrics higher is better ($\uparrow$). +Appendix Table~\ref{tab:app_seed_robustness} summarizes the seed-level checks behind the reported mean-with-std convention. + +\subsection{Matching-Rule Sensitivity Under Fixed Output (RQ1)} +\label{sec:rq1} + +To answer RQ1, we conduct a fixed-output check on occupancy and fire spread tasks, holding the score field $S$, label field $Y$, threshold, and all other operating choices fixed while varying only the matching rule $\Lambda$ across exact, tolerated, and union settings. Primary rank changes are shown in Figure~\ref{fig:primary_ranking}, and same-contract metric values are reported in Table~\ref{tab:primary_results}. Complete occupancy sweeps and predicted-positive rates are reported in Appendix Tables~\ref{tab:fireprone_contract_progression} and~\ref{tab:app_occupancy_ppr_scope}. + +\begin{wrapfigure}[21]{r}{0.50\textwidth} +\centering +\vspace{-3mm} +\includegraphics[width=\linewidth]{figures/fig_primary_rank_change_map.pdf} +\caption{\textbf{Primary-task rank changes (RQ1).} +Cells show rank before\(\rightarrow\)after. Green/red/gray mark moving up/down/no change; darker green or red marks a larger move. Following Section~\ref{sec:taskforms}, Ex/Tol/Un are occupancy exact, tolerated, and union matching; Sp is spread spatial-overlap $F_1$.} +\label{fig:primary_ranking} +\vspace{-0.8em} +\end{wrapfigure} +Because both tasks involve spatially sparse targets, fire-active cells for occupancy, burned raster patches for spread, the operational assumptions encoded in $\Lambda$ directly govern what the model is being asked to get right, making matching-rule choice a substantive experimental setting rather than a post hoc evaluation detail. +The fixed-output results reveal a pattern that goes beyond score differences: matching-rule choice determines whether a model appears viable for wildfire decision tasks at all. Under exact matching, which requires same-cell same-time agreement, the majority of frozen Earth-FM backbones produce F1 scores that are effectively near zero, rendering them indistinguishable from an uninformative baseline and suggesting they have no practical utility for the task. As the matching rule relaxes to tolerated and then union matching, both of which reflect operationally realistic assumptions for early warning systems, where a prediction displaced by a few grid cells still triggers the correct response, the same frozen representations recover substantial decision performance, with several backbones crossing from near-zero to practically meaningful F1 levels. This transition is not a marginal score improvement: it is a qualitative change in whether a model can be considered usable. The same pattern holds for fire spread under region-level matching relaxation, where strict raster-cell agreement again suppresses performance for most backbones while spatial tolerance restores it. The implications for prior wildfire transfer claims are significant: papers that report model performance under a single implicit matching rule, which is common practice given that sparse decision targets almost always require some form of tolerance~\cite{ebert2009neighborhood, gilleland2009intercomparison}, may be drawing viability conclusions that are entirely dependent on an undisclosed protocol choice. A model claimed to perform well under one tolerance assumption may be completely unusable under a stricter one, and vice versa. Matching rule cannot be treated as an evaluation detail; it is an experimental setting that must be fixed, reported, and justified as part of any wildfire transfer claim. Additional spread AP values under fixed scopes are reported in Appendix Table~\ref{tab:app_spread_ap_by_scope}. + + +\begin{table}[t] +\centering +\small +\setlength{\tabcolsep}{4pt} +\renewcommand{\arraystretch}{1.20} +\caption{% + \textbf{Primary fixed-contract transfer results (RQ1).} + Occupancy metrics: exact, tolerated, union $F_1$ (\%). + Fire spread metrics: exact $F_1$ and spatial $F_1$ (\%). + Each block fixes $\mathcal{T}$, $\Lambda$, $\Omega$, and $\mathcal{A}$. + Upward arrows indicate that larger values are better. + \textbf{Bold} marks the best value per metric. \textbf{Tol.} = Tolerated +} +\label{tab:primary_results} +\setlength{\arrayrulewidth}{0.4pt} +\resizebox{\textwidth}{!}{% +\begin{tabular}{lccccc} +\toprule +& \multicolumn{3}{c}{\textbf{Occupancy}} +& \multicolumn{2}{c}{\textbf{Fire spread}} \\ +\cmidrule(lr){2-4}\cmidrule(lr){5-6} +\textbf{Comparator} +& \textbf{Exact $F_1\uparrow$} & \textbf{Tol.\ $F_1\uparrow$} & \textbf{Union $F_1\uparrow$} +& \textbf{Exact $F_1\uparrow$} & \textbf{Spatial $F_1\uparrow$} \\ +\midrule +\ourfm\ +& \ensuremath{\mathbf{5.31}{\mkern1mu}_{\scriptscriptstyle \boldsymbol{\pm}\mathbf{1.24}}} +& \ensuremath{\mathbf{44.97}{\mkern1mu}_{\scriptscriptstyle \boldsymbol{\pm}\mathbf{2.15}}} +& \ensuremath{\mathbf{66.74}{\mkern1mu}_{\scriptscriptstyle \boldsymbol{\pm}\mathbf{7.08}}} +& \ensuremath{\mathbf{37.98}{\mkern1mu}_{\scriptscriptstyle \boldsymbol{\pm}\mathbf{1.33}}} +& \ensuremath{\mathbf{82.20}{\mkern1mu}_{\scriptscriptstyle \boldsymbol{\pm}\mathbf{0.32}}} \\ +\midrule +Prithvi-WxC +& \ms{0.0552}{0.0039} & \ms{7.1649}{0.6557} & \ms{20.1853}{1.8299} +& \ms{22.14}{0.40} & \ms{58.18}{0.14} \\ +Aurora +& \ms{0.0656}{0.0094} & \ms{8.5009}{1.9594} & \ms{23.1037}{4.9418} +& \ms{31.08}{0.40} & \ms{71.82}{0.14} \\ +ClimaX +& \ms{0.3480}{0.0754} +& \ms{29.7535}{3.6073} +& \ms{60.1506}{7.5865} +& \ms{27.99}{<0.01} & \ms{69.06}{<0.01} \\ +StormCast +& \ms{0.0626}{0.0119} & \ms{8.1951}{2.1895} & \ms{22.3817}{5.4294} +& \ms{21.95}{0.37} & \ms{57.90}{0.23} \\ +DLWP +& \ms{0.1693}{0.0419} & \ms{14.9148}{3.2446} & \ms{28.1901}{6.9658} +& \ms{22.26}{1.41} & \ms{58.71}{1.91} \\ +FCN +& \ms{0.2829}{0.0839} & \ms{19.5061}{3.3412} & \ms{40.0604}{9.3701} +& \ms{20.60}{2.36} & \ms{57.15}{1.39} \\ +FengWu +& \ms{0.2613}{0.0757} & \ms{12.0050}{6.0239} & \ms{24.1022}{13.6293} +& \ms{22.50}{1.78} & \ms{58.20}{0.84} \\ +FuXi +& \ms{0.3774}{0.1212} & \ms{21.0323}{4.8211} & \ms{37.2888}{9.4470} +& \ms{25.16}{0.55} & \ms{59.57}{0.48} \\ +Pangu-Weather +& \ms{0.2755}{0.1089} & \ms{17.0909}{4.0477} & \ms{35.6386}{9.0327} +& \ms{22.29}{0.97} & \ms{58.43}{0.83} \\ +AlphaEarth +& \ms{2.0606}{0.4404} +& \ms{29.4476}{6.0064} & \ms{37.4286}{9.9458} +& \ms{16.19}{4.79} & \ms{43.47}{10.28} \\ +\bottomrule +\end{tabular} +} +\end{table} + +Table~\ref{tab:primary_results} shows that \ourfm remains the strongest same-contract entry on the primary spatial decision tasks under the reported Lower-48 input contract. The occupancy scores are \(5.31\), \(44.97\), and \(66.74\) for exact, tolerated, and union \(F_1\), respectively, while fire spread reaches \(37.98\) exact \(F_1\) and \(82.20\) spatial \(F_1\). The gap between exact and tolerated/union occupancy reinforces the central contract point: strict cell agreement and bounded-offset warning quality are different operating views and should not be collapsed into one score or one alarm policy. + + +% \begin{figure}[H] +% \centering +% \includegraphics[width=\textwidth]{figures/fig_fireprone_contract_progression_compact.pdf} +% \caption{ +% \textbf{Matching-rule sensitivity in fire-prone occupancy (RQ1).} +% Each row holds the score field \(S\), label field \(Y\), threshold, and \(\Omega\) fixed, and changes only \(\Lambda\). +% Legend: \textcolor[HTML]{404040}{$\blacksquare$} strict \(F_1\), +% \textcolor[HTML]{808080}{$\blacksquare$} added \(F_1\) from spatial tolerance, +% \textcolor[HTML]{BFBFBF}{$\blacksquare$} added \(F_1\) from union matching, +% red outline \ourfm, and dashed line original weather FMs vs.\ added baselines. +% The horizontal axis is \(F_1\) in percent. +% } +% \label{fig:fireprone_contract_progression} +% \end{figure} +\begin{wrapfigure}[14]{r}{0.50\textwidth} + \centering + \vspace{-1em} \includegraphics[width=0.50\textwidth]{figures/fig_selection_regret_scatter.pdf} +\caption{\textbf{Head-selection regret under fixed features (RQ2).} +Each point is one backbone; selection regret \(\delta\) follows Section~\ref{sec:checks} under global-scope union-\(F_1\).} + + \label{fig:selection_regret_diagnostic} + \vspace{-1.2em} +\end{wrapfigure} + +% \begin{wrapfigure}[17]{r}{0.46\textwidth} +% \vspace{-0.4em} +% \centering +% \resizebox{\linewidth}{!}{% +% \begin{tikzpicture}[x=1cm,y=1cm] +% \footnotesize +% \draw[black!12, line width=0.35pt] (2.450,-0.350) -- (2.450,4.530); +% \node[anchor=north, font=\scriptsize, text=black!70] at (2.450,-0.410) {-20}; +% \draw[black!12, line width=0.35pt] (3.243,-0.350) -- (3.243,4.530); +% \node[anchor=north, font=\scriptsize, text=black!70] at (3.243,-0.410) {-10}; +% \draw[wfgray, line width=0.55pt] (4.036,-0.350) -- (4.036,4.530); +% \node[anchor=north, font=\scriptsize, text=black!70] at (4.036,-0.410) {0}; +% \draw[black!12, line width=0.35pt] (4.829,-0.350) -- (4.829,4.530); +% \node[anchor=north, font=\scriptsize, text=black!70] at (4.829,-0.410) {10}; +% \draw[black!12, line width=0.35pt] (5.621,-0.350) -- (5.621,4.530); +% \node[anchor=north, font=\scriptsize, text=black!70] at (5.621,-0.410) {20}; +% \draw[black!12, line width=0.35pt] (6.414,-0.350) -- (6.414,4.530); +% \node[anchor=north, font=\scriptsize, text=black!70] at (6.414,-0.410) {30}; +% \draw[black!12, line width=0.35pt] (7.207,-0.350) -- (7.207,4.530); +% \node[anchor=north, font=\scriptsize, text=black!70] at (7.207,-0.410) {40}; +% \draw[black!12, line width=0.35pt] (8.000,-0.350) -- (8.000,4.530); +% \node[anchor=north, font=\scriptsize, text=black!70] at (8.000,-0.410) {50}; +% \draw[black!45, line width=0.4pt] (2.450,-0.350) -- (8.000,-0.350); +% \node[anchor=east, font=\scriptsize, text=black!82] at (2.320,4.350) {\textcolor{wfneutral}{\textbf{\ourfm}}}; +% \draw[wfslate, line width=0.72pt] (4.030,4.220) -- (5.212,4.220); +% \draw[wfslate, line width=0.72pt] (4.030,4.185) -- (4.030,4.255); +% \draw[wfslate, line width=0.72pt] (5.212,4.185) -- (5.212,4.255); +% \filldraw[wfslate] (4.621,4.220) circle[radius=0.045]; +% \draw[wforange, line width=0.72pt] (4.051,4.480) -- (4.487,4.480); +% \draw[wforange, line width=0.72pt] (4.051,4.445) -- (4.051,4.515); +% \draw[wforange, line width=0.72pt] (4.487,4.445) -- (4.487,4.515); +% \filldraw[wforange] (4.224,4.435) rectangle (4.314,4.525); +% \node[anchor=east, font=\scriptsize, text=black!82] at (2.320,3.940) {Prithvi-WxC}; +% \draw[wfslate, line width=0.72pt] (4.036,3.810) -- (4.036,3.810); +% \draw[wfslate, line width=0.72pt] (4.036,3.775) -- (4.036,3.845); +% \draw[wfslate, line width=0.72pt] (4.036,3.775) -- (4.036,3.845); +% \filldraw[wfslate] (4.036,3.810) circle[radius=0.045]; +% \draw[wforange, line width=0.72pt] (4.036,4.070) -- (4.036,4.070); +% \draw[wforange, line width=0.72pt] (4.036,4.035) -- (4.036,4.105); +% \draw[wforange, line width=0.72pt] (4.036,4.035) -- (4.036,4.105); +% \filldraw[wforange] (3.991,4.025) rectangle (4.081,4.115); +% \node[anchor=east, font=\scriptsize, text=black!82] at (2.320,3.530) {Aurora}; +% \draw[wfslate, line width=0.72pt] (3.580,3.400) -- (5.276,3.400); +% \draw[wfslate, line width=0.72pt] (3.580,3.365) -- (3.580,3.435); +% \draw[wfslate, line width=0.72pt] (5.276,3.365) -- (5.276,3.435); +% \filldraw[wfslate] (4.428,3.400) circle[radius=0.045]; +% \draw[wforange, line width=0.72pt] (2.627,3.660) -- (7.723,3.660); +% \draw[wforange, line width=0.72pt] (2.627,3.625) -- (2.627,3.695); +% \draw[wforange, line width=0.72pt] (7.723,3.625) -- (7.723,3.695); +% \filldraw[wforange] (5.130,3.615) rectangle (5.220,3.705); +% \node[anchor=east, font=\scriptsize, text=black!82] at (2.320,3.120) {ClimaX}; +% \draw[wfslate, line width=0.72pt] (4.032,2.990) -- (4.060,2.990); +% \draw[wfslate, line width=0.72pt] (4.032,2.955) -- (4.032,3.025); +% \draw[wfslate, line width=0.72pt] (4.060,2.955) -- (4.060,3.025); +% \filldraw[wfslate] (4.046,2.990) circle[radius=0.045]; +% \draw[wforange, line width=0.72pt] (4.036,3.250) -- (4.036,3.250); +% \draw[wforange, line width=0.72pt] (4.036,3.215) -- (4.036,3.285); +% \draw[wforange, line width=0.72pt] (4.036,3.215) -- (4.036,3.285); +% \filldraw[wforange] (3.991,3.205) rectangle (4.081,3.295); +% \node[anchor=east, font=\scriptsize, text=black!82] at (2.320,2.710) {StormCast}; +% \draw[wfslate, line width=0.72pt] (4.036,2.580) -- (4.036,2.580); +% \draw[wfslate, line width=0.72pt] (4.036,2.545) -- (4.036,2.615); +% \draw[wfslate, line width=0.72pt] (4.036,2.545) -- (4.036,2.615); +% \filldraw[wfslate] (4.036,2.580) circle[radius=0.045]; +% \draw[wforange, line width=0.72pt] (4.036,2.840) -- (4.036,2.840); +% \draw[wforange, line width=0.72pt] (4.036,2.805) -- (4.036,2.875); +% \draw[wforange, line width=0.72pt] (4.036,2.805) -- (4.036,2.875); +% \filldraw[wforange] (3.991,2.795) rectangle (4.081,2.885); +% \node[anchor=east, font=\scriptsize, text=black!82] at (2.320,2.300) {DLWP}; +% \draw[wfslate, line width=0.72pt] (4.036,2.170) -- (4.036,2.170); +% \draw[wfslate, line width=0.72pt] (4.036,2.135) -- (4.036,2.205); +% \draw[wfslate, line width=0.72pt] (4.036,2.135) -- (4.036,2.205); +% \filldraw[wfslate] (4.036,2.170) circle[radius=0.045]; +% \draw[wforange, line width=0.72pt] (4.044,2.430) -- (4.735,2.430); +% \draw[wforange, line width=0.72pt] (4.044,2.395) -- (4.044,2.465); +% \draw[wforange, line width=0.72pt] (4.735,2.395) -- (4.735,2.465); +% \filldraw[wforange] (4.345,2.385) rectangle (4.435,2.475); +% \node[anchor=east, font=\scriptsize, text=black!82] at (2.320,1.890) {FCN}; +% \draw[wfslate, line width=0.72pt] (4.036,1.760) -- (4.036,1.760); +% \draw[wfslate, line width=0.72pt] (4.036,1.725) -- (4.036,1.795); +% \draw[wfslate, line width=0.72pt] (4.036,1.725) -- (4.036,1.795); +% \filldraw[wfslate] (4.036,1.760) circle[radius=0.045]; +% \draw[wforange, line width=0.72pt] (3.971,2.020) -- (4.286,2.020); +% \draw[wforange, line width=0.72pt] (3.971,1.985) -- (3.971,2.055); +% \draw[wforange, line width=0.72pt] (4.286,1.985) -- (4.286,2.055); +% \filldraw[wforange] (4.083,1.975) rectangle (4.173,2.065); +% \node[anchor=east, font=\scriptsize, text=black!82] at (2.320,1.480) {FengWu}; +% \draw[wfslate, line width=0.72pt] (4.036,1.350) -- (4.036,1.350); +% \draw[wfslate, line width=0.72pt] (4.036,1.315) -- (4.036,1.385); +% \draw[wfslate, line width=0.72pt] (4.036,1.315) -- (4.036,1.385); +% \filldraw[wfslate] (4.036,1.350) circle[radius=0.045]; +% \draw[wforange, line width=0.72pt] (4.028,1.610) -- (4.127,1.610); +% \draw[wforange, line width=0.72pt] (4.028,1.575) -- (4.028,1.645); +% \draw[wforange, line width=0.72pt] (4.127,1.575) -- (4.127,1.645); +% \filldraw[wforange] (4.032,1.565) rectangle (4.122,1.655); +% \node[anchor=east, font=\scriptsize, text=black!82] at (2.320,1.070) {FuXi}; +% \draw[wfslate, line width=0.72pt] (4.036,0.940) -- (4.036,0.940); +% \draw[wfslate, line width=0.72pt] (4.036,0.905) -- (4.036,0.975); +% \draw[wfslate, line width=0.72pt] (4.036,0.905) -- (4.036,0.975); +% \filldraw[wfslate] (4.036,0.940) circle[radius=0.045]; +% \draw[wforange, line width=0.72pt] (4.029,1.200) -- (4.087,1.200); +% \draw[wforange, line width=0.72pt] (4.029,1.165) -- (4.029,1.235); +% \draw[wforange, line width=0.72pt] (4.087,1.165) -- (4.087,1.235); +% \filldraw[wforange] (4.013,1.155) rectangle (4.103,1.245); +% \node[anchor=east, font=\scriptsize, text=black!82] at (2.320,0.660) {Pangu-Weather}; +% \draw[wfslate, line width=0.72pt] (4.036,0.530) -- (4.036,0.530); +% \draw[wfslate, line width=0.72pt] (4.036,0.495) -- (4.036,0.565); +% \draw[wfslate, line width=0.72pt] (4.036,0.495) -- (4.036,0.565); +% \filldraw[wfslate] (4.036,0.530) circle[radius=0.045]; +% \draw[wforange, line width=0.72pt] (4.025,0.790) -- (4.076,0.790); +% \draw[wforange, line width=0.72pt] (4.025,0.755) -- (4.025,0.825); +% \draw[wforange, line width=0.72pt] (4.076,0.755) -- (4.076,0.825); +% \filldraw[wforange] (4.006,0.745) rectangle (4.096,0.835); +% \node[anchor=east, font=\scriptsize, text=black!82] at (2.320,0.250) {AlphaEarth}; +% \draw[wfslate, line width=0.72pt] (4.700,0.120) -- (6.103,0.120); +% \draw[wfslate, line width=0.72pt] (4.700,0.085) -- (4.700,0.155); +% \draw[wfslate, line width=0.72pt] (6.103,0.085) -- (6.103,0.155); +% \filldraw[wfslate] (5.401,0.120) circle[radius=0.045]; +% \draw[wforange, line width=0.72pt] (3.872,0.380) -- (4.815,0.380); +% \draw[wforange, line width=0.72pt] (3.872,0.345) -- (3.872,0.415); +% \draw[wforange, line width=0.72pt] (4.815,0.345) -- (4.815,0.415); +% \filldraw[wforange] (4.298,0.335) rectangle (4.388,0.425); +% \end{tikzpicture}% +% } +% \caption{\textbf{Fixed-feature selection-regret check (RQ2).} Fixed-feature selection regret \(\delta = D(h_D)-D(h_R)\) under union-\(F_1\). \textcolor{wfslate}{$\bullet$} uses the full grid; \textcolor{wforange}{$\blacksquare$} uses the top \(20\%\) fire-prone cells from training fire frequency. Horizontal intervals use the same colors and show mean \(\pm\) std over five seeds, in percentage points.} +% \vspace{+0.4em} +% \label{fig:selection_regret_diagnostic} +% \end{wrapfigure} + +\subsection{Head-Selection Sensitivity Under Fixed Features (RQ2)} +\label{sec:rq2} + +To answer RQ2, we conduct a fixed-feature check on occupancy and fire spread tasks, holding the frozen feature source, $\mathcal{T}$, $\Omega$, $\Lambda$, and candidate head family $\mathcal{H} \subseteq \mathcal{A}$ fixed while varying only the selection metric between PR-AUC-based and decision-F1-based selection. The resulting selection regret $\delta = D(h_D) - D(h_R)$ measures the decision-score loss induced by metric misalignment. Occupancy results are reported in Figure~\ref{fig:selection_regret_diagnostic} under both global and fire-prone scopes. Full per-seed and per-head details are reported in Appendix~\ref{sec:app_seeded_audits}, and the exact, tolerated, and union regret breakdown is provided in Appendix Table~\ref{tab:appendix_selection_regret_tolerance}. + +The fixed-feature results show that head-selection metrics introduce substantial backbone-dependent variation that is not explained by representation quality alone. Some backbones exhibit near-zero regret, indicating agreement between PR-AUC and decision-F1 selection, while others show large regret concentrated in specific scope-matching settings. Regret is generally larger under the global scope, where severe fire imbalance amplifies misalignment between ranking and decision metrics~\cite{mcdermott2024aurocauprc}. Restricting evaluation to fire-prone scopes typically reduces regret by concentrating evaluation on fire-relevant regions. A similar pattern appears for fire spread, where ranking and decision metrics can favor different heads under the same frozen representation. These results show that selection metrics must be aligned with the evaluation objective as part of the evaluation contract~\cite{traub2024selectiveclassification}. + +\subsection{Supporting Task Checks (RQ3)} +\label{sec:rq3} + +To answer RQ3, we evaluate all backbones across the four supporting task contracts, \textit{burned area, analog retrieval, smoke PM$_{2.5}$, }and\textit{ extreme heat}, and examine whether the reference-versus-frozen ordering established under primary tasks generalizes across task forms. A rank overview across all six contracts is provided in Figure~\ref{fig:task_comparator_normalized_map}, which maps backbone-by-task rank positions and makes cross-task ordering shifts directly visible. Native metric values are reported in Table~\ref{tab:supporting_results}. Additional supporting-task diagnostics are reported in Appendix Tables~\ref{tab:app_burned_area_median_acre}, \ref{tab:app_analog_rank_depth}, \ref{tab:app_smoke_high_event}, and~\ref{tab:app_heat_event_pr}. + + +The supporting task results produce three qualitatively distinct patterns relative to the primary findings. Burned area largely preserves the reference-versus-frozen ordering seen under occupancy and spread: \ourfm\ leads frozen entries on log-RMSE and Spearman $\rho$, suggesting that the wildfire-specific training signal transfers to event-scale regression under a different metric family and provides convergent evidence for the primary claim. Analog retrieval and smoke PM$_{2.5}$ show a different pattern, with AlphaEarth close to \ourfm\ on analog retrieval and competitive on smoke, indicating that retrieval and air-quality signals can also be captured by a general remote-sensing backbone. Extreme heat exhibits the largest variance across the comparator set, with atmospheric FMs ranging from near-reference performance to near-complete failure depending on backbone pretraining domain, while AlphaEarth again forms a near-tie with \ourfm. The scale of this variance is itself informative: aggregating scores across task forms without respecting contract boundaries would produce rankings dominated by scale artifacts in the extreme heat block rather than by transfer quality. Taken together, these results establish that supporting tasks bound rather than extend the primary claim, they provide useful evidence about where backbone families generalize and where they do not, but they cannot substitute for primary decision task evaluation, and their results must be interpreted within their own task-form contracts. + +\begin{figure}[t] + \centering +\vspace{-5mm} + + \includegraphics[width=\textwidth]{figures/fig_rank_heatmap1.pdf} + \vspace{-2mm} + \caption{{\textbf{Rank map for supporting task comparison (RQ3).} Each row fixes one task contract $\mathcal{C}$ and ranks the eligible backbones within that contract. The figure shows rank changes across task forms; native metric values are reported in Table~\ref{tab:supporting_results}.}} +\vspace{-6mm} + \label{fig:task_comparator_normalized_map} +\end{figure} + +\begin{table}[t] +\centering +\small +\setlength{\tabcolsep}{3.5pt} +\renewcommand{\arraystretch}{1.18} +\caption{% + \textbf{Supporting task-metric matrix (RQ3).} + Top: final burned area and analog retrieval. + Bottom: smoke PM$_{2.5}$ and extreme heat. + Each block fixes $\mathcal{T}$, $\Lambda$, and $\Omega$; backbone + column is shared across paired tasks. \ourfm\ row is + separated by a rule as the empirical anchor. \textbf{Bold} marks + the best value per metric. For error metrics + lower is better ($\downarrow$); for $F_1$, nDCG, and $r$ higher + is better ($\uparrow$). +} +\label{tab:supporting_results} +\resizebox{\textwidth}{!}{% +\begin{tabular}{lcccccc} +\toprule +& \multicolumn{3}{c}{\textbf{Burned area}} +& \multicolumn{3}{c}{\textbf{Analog retrieval}} \\ +\cmidrule(lr){2-4}\cmidrule(lr){5-7} +\textbf{Backbone} +& \textbf{log-RMSE$\downarrow$} & \textbf{log-MAE$\downarrow$} +& \textbf{Spearman$\uparrow$} +& \textbf{nDCG@10$\uparrow$} & \textbf{log-RMSE$\downarrow$} +& \textbf{log-MAE$\downarrow$} \\ +\midrule +\ourfm\ +& \ensuremath{\mathbf{1.1649}{\mkern1mu}_{\scriptscriptstyle \boldsymbol{\pm}\mathbf{0.0139}}} +& \ensuremath{\mathbf{1.0355}{\mkern1mu}_{\scriptscriptstyle \boldsymbol{\pm}\mathbf{0.0114}}} +& \ensuremath{\mathbf{0.5884}{\mkern1mu}_{\scriptscriptstyle \boldsymbol{\pm}\mathbf{0.0338}}} +& \ensuremath{\mathbf{0.5178}{\mkern1mu}_{\scriptscriptstyle \boldsymbol{\pm}<\mathbf{0.0001}}} +& \ensuremath{\mathbf{1.1769}{\mkern1mu}_{\scriptscriptstyle \boldsymbol{\pm}<\mathbf{0.0001}}} +& \ensuremath{\mathbf{0.9683}{\mkern1mu}_{\scriptscriptstyle \boldsymbol{\pm}<\mathbf{0.0001}}} \\ +\midrule +Prithvi-WxC +& \ms{1.3630}{0.0681} & \ms{1.2435}{0.0668} & \ms{0.1799}{0.3002} +& \ms{0.3857}{0.0189} & \ms{1.3908}{0.0938} & \ms{1.2585}{0.0865} \\ +Aurora +& \ms{1.8658}{0.2009} & \ms{1.6717}{0.1245} & \ms{-0.1156}{0.2982} +& \ms{0.4046}{0.0144} & \ms{1.3659}{0.0792} & \ms{1.2596}{0.0968} \\ +ClimaX +& \ms{2.0300}{0.2103} & \ms{1.8443}{0.1528} & \ms{-0.2515}{0.2688} +& \ms{0.4143}{0.0191} & \ms{1.4526}{0.0926} & \ms{1.2441}{0.1446} \\ +StormCast +& \ms{1.6679}{0.1438} & \ms{1.4745}{0.1134} & \ms{0.1830}{0.1969} +& \ms{0.4076}{0.0094} & \ms{1.3663}{0.0781} & \ms{1.2371}{0.1078} \\ +DLWP +& \ms{1.3070}{0.0980} & \ms{1.1769}{0.0834} & \ms{0.4888}{0.1368} +& \ms{0.3972}{0.0146} & \ms{1.5351}{0.0802} & \ms{1.3196}{0.0781} \\ +FCN +& \ms{1.3693}{0.0885} & \ms{1.2599}{0.0723} & \ms{0.3484}{0.1662} +& \ms{0.4316}{0.0134} & \ms{1.4604}{0.1035} & \ms{1.2351}{0.0586} \\ +FengWu +& \ms{1.3715}{0.1011} & \ms{1.2604}{0.0820} & \ms{0.3221}{0.2004} +& \ms{0.4246}{0.0237} & \ms{1.4179}{0.0986} & \ms{1.2233}{0.0915} \\ +FuXi +& \ms{1.4068}{0.1011} & \ms{1.3023}{0.0789} & \ms{0.2663}{0.2561} +& \ms{0.4279}{0.0212} & \ms{1.4290}{0.0929} & \ms{1.2236}{0.0961} \\ +Pangu-Weather +& \ms{1.3280}{0.0735} & \ms{1.2081}{0.0607} & \ms{0.4141}{0.1573} +& \ms{0.4017}{0.0245} & \ms{1.4235}{0.0731} & \ms{1.2225}{0.0847} \\ +AlphaEarth +& \ms{2.4068}{0.2841} & \ms{2.0822}{0.2371} & \ms{-0.3428}{0.1716} +& \ms{0.5036}{<0.0001} & \ms{1.2048}{<0.0001} & \ms{1.0188}{<0.0001} \\ +\bottomrule +\end{tabular} +} + +\vspace{4pt} + +\resizebox{\textwidth}{!}{% +\begin{tabular}{lcccccc} +\toprule +& \multicolumn{3}{c}{\textbf{Smoke PM$_{2.5}$}} +& \multicolumn{3}{c}{\textbf{Extreme heat}} \\ +\cmidrule(lr){2-4}\cmidrule(lr){5-7} +\textbf{Backbone} +& \textbf{RMSE$\downarrow$} & \textbf{MAE$\downarrow$} +& \textbf{Pearson $r\uparrow$} +& \textbf{RMSE-C$\downarrow$} & \textbf{MAE-C$\downarrow$} +& \textbf{Exceed.\ $F_1\uparrow$} \\ +\midrule +\ourfm\ +& \ensuremath{\mathbf{4.4684}{\mkern1mu}_{\scriptscriptstyle \boldsymbol{\pm}\mathbf{0.0060}}} +& \ms{2.4127}{0.0016} +& \ensuremath{\mathbf{0.6360}{\mkern1mu}_{\scriptscriptstyle \boldsymbol{\pm}\mathbf{0.0013}}} +& \ensuremath{\mathbf{0.2179}{\mkern1mu}_{\scriptscriptstyle \boldsymbol{\pm}\mathbf{0.0048}}} +& \ensuremath{\mathbf{0.1787}{\mkern1mu}_{\scriptscriptstyle \boldsymbol{\pm}\mathbf{0.0020}}} +& \ms{0.9541}{0.0164} \\ +\midrule +Prithvi-WxC +& \ms{6.0382}{0.0828} & \ms{3.7301}{0.0055} & \ms{0.0243}{0.0045} +& \ms{4.6225}{0.0192} & \ms{2.6315}{0.0128} & \ms{0.8693}{0.0023} \\ +Aurora +& \ms{6.0384}{0.0828} & \ms{3.7265}{0.0055} & \ms{0.0193}{0.0043} +& \ms{18.0474}{0.0708} & \ms{15.3747}{0.0594} & \ms{0.0951}{0.0038} \\ +ClimaX +& \ms{6.0402}{0.0828} & \ms{3.7290}{0.0055} & \ms{0.0004}{0.0029} +& \ms{17.6492}{0.0347} & \ms{14.4938}{0.0319} & \ms{0.7684}{0.0068} \\ +StormCast +& \ms{6.1230}{0.0830} & \ms{3.8182}{0.0073} & \ms{0.0183}{0.0041} +& \ms{1.7671}{0.2145} & \ms{1.3507}{0.1576} & \ms{0.9073}{0.0189} \\ +DLWP +& \ms{5.9289}{0.1031} & \ms{3.7331}{0.0088} & \ms{0.0303}{0.0060} +& \ms{2.2662}{0.1106} & \ms{1.7153}{0.0748} & \ms{0.9156}{0.0112} \\ +FCN +& \ms{5.9277}{0.1033} & \ms{3.7345}{0.0088} & \ms{0.0312}{0.0062} +& \ms{2.1657}{0.1800} & \ms{1.6033}{0.1039} & \ms{0.9257}{0.0096} \\ +FengWu +& \ms{5.9297}{0.1032} & \ms{3.7395}{0.0088} & \ms{0.0304}{0.0063} +& \ms{2.1266}{0.1589} & \ms{1.5801}{0.1004} & \ms{0.0481}{0.0459} \\ +FuXi +& \ms{5.9319}{0.1029} & \ms{3.7398}{0.0088} & \ms{0.0299}{0.0061} +& \ms{2.1282}{0.0969} & \ms{1.5759}{0.0719} & \ms{0.2268}{0.0623} \\ +Pangu-Weather +& \ms{5.9270}{0.1036} & \ms{3.7320}{0.0088} & \ms{0.0301}{0.0060} +& \ms{2.2045}{0.1483} & \ms{1.6307}{0.0889} & \ms{0.0199}{0.0062} \\ +AlphaEarth +& \ms{4.4899}{0.0042} +& \ensuremath{\mathbf{2.4009}{\mkern1mu}_{\scriptscriptstyle \boldsymbol{\pm}\mathbf{0.0020}}} +& \ms{0.6315}{0.0010} +& \ms{0.2187}{<0.0001} +& \ms{0.1795}{<0.0001} +& \ensuremath{\mathbf{0.9542}{\mkern1mu}_{\scriptscriptstyle \boldsymbol{\pm}\mathbf{0.0107}}} \\ +\bottomrule +\end{tabular} +} +\end{table} + + +\paragraph{Pattern 1: primary pattern preserved (burned area).} +\ourfm\ leads all frozen entries on log-RMSE and Spearman $\rho$. The ordering observed under occupancy and spread is preserved under burned-area regression despite the different prediction unit and metric family. + +\paragraph{Pattern 2: primary pattern bounded (analog retrieval and smoke PM$_{2.5}$).} +For analog retrieval, AlphaEarth approaches \ourfm\ (nDCG@10 $= 0.5036$ vs.\ $0.5178$). For smoke PM$_{2.5}$, \ourfm has lower RMSE and slightly higher Pearson $r$, while AlphaEarth has lower MAE and remains close. These results show that the occupancy-and-spread ordering does not fully extend to all supporting tasks once AlphaEarth is included. + +\paragraph{Pattern 3: primary pattern bounded with large variance (extreme heat).} \ourfm and AlphaEarth are near-tied on RMSE-C and exceedance F1, while atmospheric FMs range from RMSE-C $= 1.77$ (StormCast) to $18.05$ (Aurora). This large spread indicates that aggregated scores across task forms would be dominated by scale artifacts rather than transfer quality, reinforcing the need for per-contract reporting established in Section~\ref{sec:eval}. + +\textit{Answer to RQ3:} Figure~\ref{fig:task_comparator_normalized_map} and Table~\ref{tab:supporting_results} show that burned area preserves the primary reference-versus-frozen pattern under a different metric family. Analog retrieval, smoke PM$_{2.5}$, and extreme heat bound this pattern: AlphaEarth matches or approaches \ourfm on these tasks, indicating that the primary occupancy and spread claims do not extend uniformly across all task forms. diff --git a/paper/iclr_source/sections/5_conclusion.tex b/paper/iclr_source/sections/5_conclusion.tex new file mode 100644 index 0000000000000000000000000000000000000000..679693ddc7c24b0638daf97ff3e4b0717fdb368d --- /dev/null +++ b/paper/iclr_source/sections/5_conclusion.tex @@ -0,0 +1,31 @@ +\vspace{-4em} +\section{Conclusion} + +We introduced \ourfm as a wildfire-specialized reference +backbone trained directly on fire-relevant multimodal data. +Our results show that wildfire forecasting benefits from +in-domain weather, fuel, canopy, validity, and active-fire +supervision rather than relying only on transfer from general +atmospheric or geophysical pretraining. +At the same time, our study shows that reliable wildfire +transfer evaluation is substantially more difficult than +standard benchmark settings suggest. Wildfire transfer +conclusions depend strongly on matching rules, head-selection +metrics, and task form, and scores computed under different +evaluation settings are not directly comparable. These effects +become particularly pronounced in sparse spatiotemporal +prediction settings such as wildfire forecasting. +We therefore introduced a fixed-contract evaluation framework +for wildfire Earth-FM transfer. By explicitly specifying the +task, metric, matching rule, evaluation scope, and head family +before comparison, fixed-contract evaluation enables more +controlled and interpretable comparison across wildfire tasks +and models. +We hope \ourfm\ and the fixed-contract framework provide a +basis for future wildfire-specific Earth FMs, transfer +benchmarks, and decision-oriented evaluation protocols. +More broadly, our research provides a more cautious basis for interpreting wildfire-model evidence at the intersection of AI and environmental decision-making. + +\paragraph{Limitations.} The conclusions apply to the task forms, scopes, evaluation rules, and comparator eligibility decisions used in this study. +The evaluation covers selected wildfire decision tasks and supporting retrieval and regression task forms. +They provide task-form evidence rather than a single score across all wildfire-related prediction tasks. diff --git a/paper/iclr_source/sections/appendix.tex b/paper/iclr_source/sections/appendix.tex new file mode 100644 index 0000000000000000000000000000000000000000..f2936d03d919a91a11b12401b3cc92f4bf9d3314 --- /dev/null +++ b/paper/iclr_source/sections/appendix.tex @@ -0,0 +1,732 @@ +% ============================================================ +% APPENDIX +% ============================================================ +\appendix + +% Copy-paste safety: these definitions are no-ops when main.tex already defines them. +\providecommand{\ms}[2]{\ensuremath{#1{\mkern1mu}_{\scriptscriptstyle \pm #2}}} + + +% ──────────────────────────────────────────────────────────── +% TABLE OF CONTENTS (appendix only) +% ──────────────────────────────────────────────────────────── +\section*{Appendix Contents} +\addcontentsline{toc}{section}{Appendix Contents} + + +\begin{center} +\begin{tabular}{@{}p{0.82\textwidth}r@{}} +\textbf{A\quad Evaluation Contract Specifications} & \pageref{sec:app_contract} \\ +\quad A.1\enspace Matching Rule Definitions & \pageref{sec:app_contract_matching} \\ +\quad A.2\enspace Task-Form Contract Parameters & \pageref{sec:app_contract_params} \\ +\quad A.3\enspace Evaluation Scope Definitions & \pageref{sec:app_contract_scope} \\[4pt] +\textbf{B\quad Controlled Check Details} & \pageref{sec:app_checks} \\ +\quad B.1\enspace Fixed-Output Check: Full Sweep & \pageref{sec:app_checks_output} \\ +\quad B.2\enspace Fixed-Feature Check: Selection Summary & \pageref{sec:app_checks_feature} \\ +\quad B.3\enspace Selection Regret Under Matching Rules & \pageref{sec:app_checks_regret} \\ +\quad B.4\enspace Additional Value Tables & \pageref{sec:app_checks_values} \\[4pt] +\textbf{C\quad Comparator Eligibility Notes} & \pageref{sec:comparator_audit} \\[4pt] +\textbf{D\quad Seeded Audits} & \pageref{sec:app_seeded_audits} \\ +\quad D.1\enspace Seed Robustness Summary & \pageref{sec:app_seed_robustness} \\[4pt] +\textbf{E\quad Lightweight Head and Adaptation Details} & \pageref{sec:app_heads} \\[4pt] +\textbf{F\quad Limitations} & \pageref{sec:limitations} \\[4pt] +\textbf{G\quad Reproducibility and Evaluation Artifacts} & \pageref{sec:repro_compute_impact} \\ +\end{tabular} +\end{center} + +\noindent\textit{Retention rule.} +Appendix tables are retained when they add contract parameters, controlled-check arithmetic, +task-specific non-main metrics, seed summaries, eligibility checks, or protocol details. +Full task matrices and reference-summary tables that repeat the main result tables are not repeated here. + +\clearpage + +% ============================================================ +% A EVALUATION CONTRACT SPECIFICATIONS +% ============================================================ +\section{Evaluation Contract Specifications} +\label{sec:app_contract} + +% ──────────────────────────────────────────────────────────── +\subsection{Matching Rule Definitions} +\label{sec:app_contract_matching} + +The three matching rules used across occupancy task forms are defined as follows. + +\noindent\textbf{Exact matching.} +A predicted unit-time pair $(i,t) \in \widehat{P}_\tau$ is counted as a true positive if and only if the same pair appears in the observed fire set $P = \{(i,t): y_{i,t}=1\}$. +This is the strictest rule and yields the lowest $F_1$ for any fixed output. + +\noindent\textbf{Tolerated matching.} +A predicted pair $(i,t)$ is counted as correct if there exists an observed pair $(i',t') \in P$ such that $\|i - i'\|_\infty \le k$ and $|t - t'| \le \Delta t$, where $k$ is the spatial tolerance in grid cells and $\Delta t$ is the temporal tolerance in forecast steps. +Both parameters are fixed as part of the evaluation contract $\mathcal{C}$ before scoring. + +\noindent\textbf{Union matching.} +A predicted pair is counted as a true positive if it satisfies either exact or tolerated matching. +The resulting union-$F_1$ provides an upper bound on decision performance under the chosen tolerance. + +\noindent\textbf{Fixed parameter values.} +For occupancy, the spatial tolerance is $k=8$ grid cells. +The temporal tolerance is $\Delta t=3$ forecast steps for union matching and $\Delta t=0$ for spatial-only tolerance. +The threshold $\tau$ is selected on validation strict-$F_1$ before test scoring. +For fire spread, the spatial tolerance is $k=4$ grid cells, $\Delta t=0$, and the threshold is selected on validation spatial $F_1$. + +\noindent Table~\ref{tab:app_matching_rule_params} records the fixed matching-rule parameters. + +\begin{table}[h] +\centering +\small +\setlength{\tabcolsep}{10pt} +\renewcommand{\arraystretch}{1.2} +\caption{Matching-rule values used in the evaluation contracts.} +\label{tab:app_matching_rule_params} +\begin{tabular}{lll} +\toprule +\textbf{Parameter} & \textbf{Occupancy} & \textbf{Fire spread} \\ +\midrule +\(k\) & 8 cells & 4 cells \\ +\(\Delta t\) & 3 for union; 0 spatial-only & 0 \\ +\(\tau\) & val. strict \(F_1\) & val. spatial \(F_1\) \\ +\bottomrule +\end{tabular} +\end{table} + +% ──────────────────────────────────────────────────────────── +\subsection{Task-Form Contract Parameters} +\label{sec:app_contract_params} + +Table~\ref{tab:app_contract_params_full} lists fixed scoring values not shown in the main contract map. + +\begin{table}[h] +\centering +\scriptsize +\setlength{\tabcolsep}{3.5pt} +\renewcommand{\arraystretch}{1.2} +\caption{Fixed scoring values used by each task-form contract.} +\label{tab:app_contract_params_full} +\begin{adjustbox}{max width=\textwidth} +\begin{tabular}{llll} +\toprule +\textbf{\(\mathcal{T}\)} & \textbf{Scoring} & \textbf{Validation} & \textbf{\(\Omega\)} \\ +\midrule +Occupancy & \(k=8,\Delta t=3\); exact/tol./union \(F_1\) & val. strict \(F_1\) & global; top-5/10/20\% fire-prone \\ +Fire spread & \(k=4,\Delta t=0\); exact/spatial \(F_1\), AP & val. spatial \(F_1\) & spread-region cells \\ +Final burned area & log-RMSE, log-MAE, Spearman \(\rho\) & val. log-RMSE & test events \\ +Analog retrieval & nDCG@10; retrieved-event log error & val. nDCG@10 & test events \\ +Smoke PM\(_{2.5}\) & RMSE, MAE, Pearson \(r\); exceedance 35 & val. RMSE & test stations \\ +Extreme heat & RMSE-C, MAE-C, exceedance \(F_1\) & val. threshold 27/30/33\(^{\circ}\)C & heat-region stations \\ +\bottomrule +\end{tabular} +\end{adjustbox} +\end{table} + +% ──────────────────────────────────────────────────────────── +\subsection{Evaluation Scope Definitions} +\label{sec:app_contract_scope} + +\noindent\textbf{Global scope.} +Evaluation covers all spatial units in the domain, including fire-inactive regions. +This scope can mask model differences on fire-relevant locations because inactive cells inflate true-negative counts. + +\noindent\textbf{Fire-prone scope.} +Evaluation is restricted to grid cells in the top-$k$\% of historical fire activity. +We report results for top-5\%, top-10\%, and top-20\% cutoffs. +The cutoff thresholds are derived from the training period and held fixed at test time. + +\noindent\textbf{Spread region scope.} +For fire spread tasks, evaluation is restricted to the predicted and observed burned raster patches. +Only cells within the union of $\widehat{B}$ and $B$ contribute to metric computation. + +\noindent\textbf{Fixed scope sizes.} +The global scope contains 8,085,000 test cells. +The fire-prone top-5\%, top-10\%, and top-20\% scopes contain 404,280, 808,560, and 1,617,000 test cells, respectively. +The spread-region scope is event-specific and uses the union of $\widehat{B}$ and $B$. + +\begin{table}[h] +\centering +\small +\setlength{\tabcolsep}{8pt} +\renewcommand{\arraystretch}{1.2} +\caption{Scope values used in the evaluation contracts.} +\label{tab:app_scope_params} +\begin{tabular}{lcc} +\toprule +\textbf{\(\Omega\)} & \textbf{Definition} & \textbf{Units} \\ +\midrule +Global & full domain & 8,085,000 test cells \\ +Fire-prone top-5\% & top 5\% by training-period fire frequency & 404,280 test cells \\ +Fire-prone top-10\% & top 10\% by training-period fire frequency & 808,560 test cells \\ +Fire-prone top-20\% & top 20\% by training-period fire frequency & 1,617,000 test cells \\ +Spread region & union of \(\widehat{B}\) and \(B\) & event-specific cells \\ +\bottomrule +\end{tabular} +\end{table} + +\clearpage + +% ============================================================ +% B CONTROLLED CHECK DETAILS +% ============================================================ +\section{Controlled Check Details} +\label{sec:app_checks} + + +\begin{figure}[t] + \centering + \includegraphics[width=\textwidth]{figures/fig_fireprone_contract_progression_compact.pdf} + \caption{ +\textbf{Matching-rule sensitivity in fire-prone occupancy (RQ1).} +Each row holds the score field \(S\), label field \(Y\), threshold, and \(\Omega\) fixed, and changes only \(\Lambda\). +Legend: \textcolor[HTML]{404040}{$\blacksquare$} strict \(F_1\), +\textcolor[HTML]{808080}{$\blacksquare$} added \(F_1\) from spatial tolerance, +\textcolor[HTML]{BFBFBF}{$\blacksquare$} added \(F_1\) from union matching, +red outline \ourfm, and dashed line original weather FMs vs.\ added baselines. +The horizontal axis is \(F_1\) in percent. +} + \label{fig:fireprone_contract_progression} +\end{figure} + +% ──────────────────────────────────────────────────────────── +\subsection{Fixed-Output Check: Full Sweep} +\label{sec:app_checks_output} + +The fixed-output check holds the score field $S$ and label field $Y$ fixed and varies only $\Lambda$. +Table~\ref{tab:fireprone_contract_progression} reports the full global and fire-prone sweep for all retained backbones. +The same table is the numeric counterpart to Figure~\ref{fig:fireprone_contract_progression}. + +\begin{table*}[t] + \centering + \scriptsize + \setlength{\tabcolsep}{4pt} + \caption{Occupancy \(F_1\) scores across global and fire-prone scopes. Global uses the full validation/test domain; top-\(k\) rows use train-defined fire-prone masks from historical fire frequency. Values are percentages from the same validation-selected strict threshold. Tolerance is spatial-only; union adds temporal and spatial matching. \(\Delta\) is union minus strict. Cells report five-seed mean with std in small type.} + \label{tab:fireprone_contract_progression} + \begin{tabular}{@{}llcccc@{}} + \toprule + Backbone & \(\Omega\) & Strict \(F_1\uparrow\) & Tol.\ \(F_1\uparrow\) & Union \(F_1\uparrow\) & \(\Delta\) \(\uparrow\) \\ + \midrule + \ourfm & global & \ms{5.31}{1.24} & \ms{44.97}{2.15} & \ms{66.74}{7.08} & \ms{61.43}{8.13} \\ + & top 5\% & \ms{11.94}{2.66} & \ms{46.04}{5.20} & \ms{69.84}{7.52} & \ms{57.89}{8.62} \\ + & top 10\% & \ms{11.78}{2.49} & \ms{45.67}{4.82} & \ms{69.21}{7.44} & \ms{57.44}{8.65} \\ + & top 20\% & \ms{11.42}{2.26} & \ms{44.20}{4.18} & \ms{67.80}{7.64} & \ms{56.38}{8.87} \\ + \addlinespace[1pt] + Prithvi-WxC & global & \ms{0.0552}{0.0039} & \ms{7.1649}{0.6557} & \ms{20.1853}{1.8299} & \ms{20.1301}{1.8297} \\ + & top 5\% & \ms{1.4119}{1.1635} & \ms{19.2636}{4.5019} & \ms{42.5793}{4.5495} & \ms{41.1674}{3.4846} \\ + & top 10\% & \ms{1.2376}{1.3201} & \ms{14.8780}{8.4429} & \ms{32.6913}{13.2085} & \ms{31.4536}{11.9053} \\ + & top 20\% & \ms{1.1520}{1.3770} & \ms{13.1512}{9.4556} & \ms{28.1319}{15.2866} & \ms{26.9800}{13.9224} \\ + \addlinespace[1pt] + Aurora & global & \ms{0.0656}{0.0094} & \ms{8.5009}{1.9594} & \ms{23.1037}{4.9418} & \ms{23.0382}{4.9325} \\ + & top 5\% & \ms{0.9859}{0.9299} & \ms{15.1337}{6.0821} & \ms{35.4834}{11.0192} & \ms{34.4975}{10.3728} \\ + & top 10\% & \ms{0.7790}{1.0453} & \ms{12.7381}{6.5558} & \ms{30.5305}{10.8842} & \ms{29.7515}{9.8656} \\ + & top 20\% & \ms{0.6655}{1.1043} & \ms{10.5304}{7.4309} & \ms{24.9444}{12.5844} & \ms{24.2790}{11.4943} \\ + \addlinespace[1pt] + ClimaX & global & \ms{0.3480}{0.0754} & \ms{29.7535}{3.6073} & \ms{60.1506}{7.5865} & \ms{59.8026}{7.5454} \\ + & top 5\% & \ms{1.2937}{0.1086} & \ms{34.5791}{2.3772} & \ms{69.2186}{5.7215} & \ms{67.9249}{5.7263} \\ + & top 10\% & \ms{1.2522}{0.1602} & \ms{34.3341}{2.2852} & \ms{68.5713}{5.5377} & \ms{67.3191}{5.5538} \\ + & top 20\% & \ms{1.0287}{0.2686} & \ms{30.2140}{4.2857} & \ms{60.0650}{7.5674} & \ms{59.0363}{7.5891} \\ + \addlinespace[1pt] + StormCast & global & \ms{0.0626}{0.0119} & \ms{8.1951}{2.1895} & \ms{22.3817}{5.4294} & \ms{22.3191}{5.4178} \\ + & top 5\% & \ms{0.9573}{0.8011} & \ms{15.3219}{5.5337} & \ms{36.1857}{9.7331} & \ms{35.2284}{9.1816} \\ + & top 10\% & \ms{0.7284}{0.9280} & \ms{12.6669}{6.3290} & \ms{30.4748}{10.6527} & \ms{29.7464}{9.7494} \\ + & top 20\% & \ms{0.5795}{0.9104} & \ms{10.4157}{7.3437} & \ms{24.6598}{12.3973} & \ms{24.0803}{11.4988} \\ + \addlinespace[1pt] + DLWP & global & \ms{0.1693}{0.0419} & \ms{14.9148}{3.2446} & \ms{28.1901}{6.9658} & \ms{28.0208}{6.9257} \\ + & top 5\% & \ms{1.8054}{0.4835} & \ms{31.7231}{3.2923} & \ms{55.4596}{5.2920} & \ms{53.6542}{5.4752} \\ + & top 10\% & \ms{1.6110}{0.5999} & \ms{27.6581}{5.9216} & \ms{47.1269}{8.0111} & \ms{45.5158}{7.7927} \\ + & top 20\% & \ms{1.5248}{0.8987} & \ms{20.9403}{4.7971} & \ms{34.9301}{7.8471} & \ms{33.4054}{7.8760} \\ + \addlinespace[1pt] + FCN & global & \ms{0.2829}{0.0839} & \ms{19.5061}{3.3412} & \ms{40.0604}{9.3701} & \ms{39.7775}{9.3423} \\ + & top 5\% & \ms{1.6231}{0.5064} & \ms{29.3769}{2.7626} & \ms{54.3033}{7.4089} & \ms{52.6801}{7.4389} \\ + & top 10\% & \ms{1.1777}{0.5118} & \ms{22.4217}{3.9803} & \ms{43.4510}{9.2513} & \ms{42.2734}{9.0251} \\ + & top 20\% & \ms{0.9962}{0.4315} & \ms{16.9792}{3.9371} & \ms{34.0859}{8.2616} & \ms{33.0897}{7.9275} \\ + \addlinespace[1pt] + FengWu & global & \ms{0.2613}{0.0757} & \ms{12.0050}{6.0239} & \ms{24.1022}{13.6293} & \ms{23.8410}{13.5736} \\ + & top 5\% & \ms{1.5695}{0.3592} & \ms{16.2763}{3.7024} & \ms{30.1055}{5.0103} & \ms{28.5360}{4.7696} \\ + & top 10\% & \ms{1.2427}{0.5333} & \ms{12.9503}{5.6052} & \ms{24.1854}{8.6854} & \ms{22.9427}{8.1863} \\ + & top 20\% & \ms{1.1192}{0.5023} & \ms{11.9508}{5.0745} & \ms{22.7860}{7.9115} & \ms{21.6668}{7.4438} \\ + \addlinespace[1pt] + FuXi & global & \ms{0.3774}{0.1212} & \ms{21.0323}{4.8211} & \ms{37.2888}{9.4470} & \ms{36.9114}{9.4327} \\ + & top 5\% & \ms{2.0307}{0.6800} & \ms{31.8944}{4.7331} & \ms{53.9308}{8.3822} & \ms{51.9001}{8.6878} \\ + & top 10\% & \ms{1.6542}{0.7316} & \ms{24.0128}{5.7784} & \ms{40.2140}{9.9307} & \ms{38.5597}{9.7744} \\ + & top 20\% & \ms{1.3646}{0.6773} & \ms{21.9548}{5.8601} & \ms{36.7314}{10.0289} & \ms{35.3668}{9.9223} \\ + \addlinespace[1pt] + Pangu-Weather & global & \ms{0.2755}{0.1089} & \ms{17.0909}{4.0477} & \ms{35.6386}{9.0327} & \ms{35.3630}{9.0774} \\ + & top 5\% & \ms{1.3656}{0.3064} & \ms{22.2222}{6.8613} & \ms{43.4234}{13.2383} & \ms{42.0578}{13.0599} \\ + & top 10\% & \ms{1.0931}{0.3535} & \ms{18.9337}{5.9329} & \ms{38.5325}{11.7221} & \ms{37.4394}{11.5261} \\ + & top 20\% & \ms{0.8844}{0.3601} & \ms{17.0172}{5.4859} & \ms{34.5688}{10.2932} & \ms{33.6844}{10.1334} \\ + \addlinespace[1pt] + AlphaEarth & global & \ms{2.0606}{0.4404} & \ms{29.4476}{6.0064} & \ms{37.4286}{9.9458} & \ms{35.3679}{10.0271} \\ + & top 5\% & \ms{6.9133}{0.8450} & \ms{42.8790}{4.6087} & \ms{51.7449}{8.7321} & \ms{44.8315}{9.0763} \\ + & top 10\% & \ms{6.6366}{0.9901} & \ms{41.8981}{5.9454} & \ms{50.5712}{10.0057} & \ms{43.9346}{9.9156} \\ + & top 20\% & \ms{6.1908}{1.1330} & \ms{38.8325}{7.4966} & \ms{46.3833}{12.1697} & \ms{40.1925}{11.6788} \\ + \bottomrule + \end{tabular} +\end{table*} + +% ──────────────────────────────────────────────────────────── +\subsection{Fixed-Feature Check: Selection Summary} +\label{sec:app_checks_feature} + +The paper appendix keeps the fixed-feature result at the selection-summary level. +The full per-head rows are retained in the supplementary CSV files and are not repeated as a manuscript table because many degenerate heads produce identical zero decision scores. +The supplementary selection rows report the decision-score loss after changing only the head-selection metric. + +% ──────────────────────────────────────────────────────────── +\subsection{Selection Regret Under Matching Rules} +\label{sec:app_checks_regret} + +The fixed-feature check trains the same head family $\mathcal{H}$ on a fixed feature source and changes only the selection metric. +Table~\ref{tab:appendix_selection_regret_tolerance} reports the same selection comparison under exact, tolerated, and union matching. +Here, \(h_R\) is selected by PR-AUC and \(h_D\) is selected by the decision metric. +The reported regret is \(D(h_D)-D(h_R)\). +Exact zero entries mean the two selectors give the same decision score for all five seeds. + +\begin{table*}[!t] + \centering + \scriptsize + \setlength{\tabcolsep}{4pt} + \caption{Selection-regret values under exact, tolerated, and union matching. Values are percentage-point regret from selecting \(h_R\) by PR-AUC instead of \(h_D\) by the decision metric. Rows report mean with small std over five seeds; \(0.0000\) denotes exact zero regret.} + \label{tab:appendix_selection_regret_tolerance} + \begin{adjustbox}{max width=\textwidth} + \begin{tabular}{llccc} + \toprule + \textbf{Feature} & \textbf{\(\Omega\)} & \textbf{Exact regret} & \textbf{Tolerated regret} & \textbf{Union regret} \\ + \midrule + \ourfm & global & 0.0000 & \ms{8.7830}{9.6705} & \ms{8.7830}{9.6705} \\ + \ourfm & fire-prone & 0.0000 & \ms{3.4027}{3.2045} & \ms{3.4027}{3.2045} \\ + Prithvi-WxC & global & 0.0000 & 0.0000 & 0.0000 \\ + Prithvi-WxC & fire-prone & 0.0000 & 0.0000 & 0.0000 \\ + Aurora & global & \ms{0.0200}{0.0267} & \ms{9.8520}{12.9878} & \ms{9.8520}{12.9878} \\ + Aurora & fire-prone & \ms{0.8203}{1.8341} & \ms{14.3919}{32.1219} & \ms{14.3919}{32.1219} \\ + ClimaX & global & \ms{0.0003}{0.0004} & \ms{0.1296}{0.1775} & \ms{0.1296}{0.1775} \\ + ClimaX & fire-prone & 0.0000 & 0.0000 & 0.0000 \\ + StormCast & global & 0.0000 & 0.0000 & 0.0000 \\ + StormCast & fire-prone & 0.0000 & 0.0000 & 0.0000 \\ + DLWP & global & 0.0000 & 0.0000 & 0.0000 \\ + DLWP & fire-prone & \ms{0.0770}{0.1100} & \ms{4.3266}{4.3323} & \ms{4.3266}{4.3323} \\ + FCN & global & 0.0000 & 0.0000 & 0.0000 \\ + FCN & fire-prone & \ms{0.0006}{0.0013} & \ms{1.1680}{1.9872} & \ms{1.1680}{1.9872} \\ + FengWu & global & 0.0000 & 0.0000 & 0.0000 \\ + FengWu & fire-prone & \ms{0.0691}{0.1191} & \ms{0.5222}{0.6239} & \ms{0.5222}{0.6239} \\ + FuXi & global & 0.0000 & 0.0000 & 0.0000 \\ + FuXi & fire-prone & 0.0000 & \ms{0.1084}{0.1729} & \ms{0.1084}{0.1729} \\ + Pangu-Weather & global & 0.0000 & 0.0000 & 0.0000 \\ + Pangu-Weather & fire-prone & \ms{0.0728}{0.1179} & \ms{0.1849}{0.3263} & \ms{0.1849}{0.3263} \\ + AlphaEarth & global & 0.0000 & \ms{17.2217}{8.8492} & \ms{17.2217}{8.8492} \\ + AlphaEarth & fire-prone & 0.0000 & \ms{3.8804}{5.9483} & \ms{3.8804}{5.9483} \\ + \bottomrule + \end{tabular} + \end{adjustbox} +\end{table*} + + +% ──────────────────────────────────────────────────────────── +\subsection{Additional Value Tables} +\label{sec:app_checks_values} + +Table~\ref{tab:app_occupancy_ppr_scope} +reports the predicted-positive rate behind the occupancy \(F_1\) sweep. + +\begin{table*}[t] +\centering +\small +\setlength{\tabcolsep}{4pt} +\renewcommand{\arraystretch}{1.18} +\caption{For fixed occupancy \(\mathcal{T}\), this table reports predicted-positive rate. +Values are percentages under the same validation-selected strict threshold. +Scopes \(\Omega\) are fixed before test scoring; cells report five-seed mean with std in small type.} +\label{tab:app_occupancy_ppr_scope} +\begin{tabular}{lcccc} +\toprule +\textbf{Backbone} & \textbf{\(\Omega=\)global} & \textbf{\(\Omega=\)top 5\%} & \textbf{\(\Omega=\)top 10\%} & \textbf{\(\Omega=\)top 20\%} \\ +\midrule +\ourfm & \ms{1.6808}{0.3684} & \ms{3.0619}{1.0925} & \ms{1.5310}{0.5463} & \ms{0.7655}{0.2732} \\ +Prithvi-WxC & \ms{61.9711}{30.9101} & \ms{57.4117}{47.8987} & \ms{58.4565}{51.0897} & \ms{58.9788}{52.6991} \\ +Aurora & \ms{55.5849}{19.7524} & \ms{57.2238}{35.3400} & \ms{68.7942}{37.6958} & \ms{67.2891}{38.3991} \\ +ClimaX & \ms{5.6763}{3.9261} & \ms{24.0091}{9.2816} & \ms{11.8450}{4.5067} & \ms{5.7442}{4.1341} \\ +StormCast & \ms{60.6507}{17.4895} & \ms{57.6017}{35.2921} & \ms{68.0766}{37.3899} & \ms{67.8397}{39.2410} \\ +DLWP & \ms{4.3221}{1.5619} & \ms{9.4001}{5.0807} & \ms{4.9700}{3.6849} & \ms{1.9198}{1.4678} \\ +FCN & \ms{1.5202}{1.3446} & \ms{4.7856}{2.9409} & \ms{2.7257}{1.6353} & \ms{0.8368}{0.2358} \\ +FengWu & \ms{0.4277}{0.4830} & \ms{0.6004}{0.3041} & \ms{0.2609}{0.1935} & \ms{0.1501}{0.1206} \\ +FuXi & \ms{0.4505}{0.2773} & \ms{2.9315}{2.6392} & \ms{0.5197}{0.6074} & \ms{0.3621}{0.4346} \\ +Pangu-Weather & \ms{1.0801}{1.1308} & \ms{2.0549}{2.1893} & \ms{1.4029}{1.4739} & \ms{1.0103}{1.1084} \\ +AlphaEarth & \ms{0.0691}{0.0499} & \ms{0.2826}{0.1497} & \ms{0.1524}{0.0770} & \ms{0.0656}{0.0414} \\ +\bottomrule +\end{tabular} +\end{table*} + +Tables~\ref{tab:app_spread_ap_by_scope}--\ref{tab:app_heat_event_pr} +report additional values that are not repeated in the main tables. +Each table fixes the task \(\mathcal{T}\) and reports either a different \(\Omega\), metric, or event subset. + +\begin{table*}[t] +\centering +\scriptsize +\setlength{\tabcolsep}{3pt} +\caption{For fixed spread \(\mathcal{T}\) and strict \(\Lambda\), this table reports AP under three \(\Omega\) scopes: full test, top-5\% train-fire area, and top-10\% train-fire area. Values are percentages; cells report mean with small std.} +\label{tab:app_spread_ap_by_scope} +\begin{tabular}{lccc} +\toprule +Backbone & full \(\Omega\) AP & top-5\% \(\Omega\) AP & top-10\% \(\Omega\) AP \\ +\midrule +\ourfm & \ms{32.73}{0.60} & \ms{40.7452}{2.0542} & \ms{37.4096}{1.8731} \\ +Prithvi-WxC & \ms{4.8319}{0.1731} & \ms{12.6086}{0.4468} & \ms{8.7051}{0.1889} \\ +Aurora & \ms{17.7723}{0.4293} & \ms{30.3106}{0.9404} & \ms{26.4732}{0.6932} \\ +ClimaX & \ms{11.1726}{0.2337} & \ms{25.7871}{1.2896} & \ms{19.9977}{1.2217} \\ +StormCast & \ms{8.1147}{1.1569} & \ms{18.5461}{1.1727} & \ms{14.1286}{1.2956} \\ +DLWP & \ms{9.2142}{2.6587} & \ms{19.3346}{2.3922} & \ms{14.9788}{2.6696} \\ +FCN & \ms{6.6774}{1.3001} & \ms{16.7396}{3.2955} & \ms{11.9308}{2.3881} \\ +FengWu & \ms{11.0046}{2.7092} & \ms{21.1506}{1.2163} & \ms{17.0113}{1.5778} \\ +FuXi & \ms{13.5507}{0.3840} & \ms{22.5434}{0.4100} & \ms{19.1964}{0.3943} \\ +Pangu-Weather & \ms{10.6250}{1.4643} & \ms{19.8294}{1.3044} & \ms{15.8013}{1.1602} \\ +AlphaEarth & \ms{12.2847}{1.3562} & \ms{22.8692}{0.4915} & \ms{18.2992}{1.2110} \\ +\bottomrule +\end{tabular} +\end{table*} + +\begin{table*}[t] +\centering +\scriptsize +\setlength{\tabcolsep}{3pt} +\caption{For fixed final-area \(\mathcal{T}\) and \(\Omega\), this table reports median log error and acre-scale errors in addition to the main log-RMSE/log-MAE/Spearman metrics. Cells report mean with small std.} +\label{tab:app_burned_area_median_acre} +\begin{tabular}{lccc} +\toprule +Backbone & log median AE & acre median AE & acre MAPE \\ +\midrule +\ourfm & \ms{1.0235}{0.0982} & \ms{4504.0692}{459.0483} & \ms{1.4525}{0.0254} \\ +Prithvi-WxC & \ms{1.2184}{0.2107} & \ms{5375.8770}{788.7906} & \ms{1.9517}{0.2875} \\ +Aurora & \ms{1.4547}{0.0301} & \ms{9904.9483}{457.4260} & \ms{6.8728}{3.0026} \\ +ClimaX & \ms{1.6841}{0.1818} & \ms{18130.4820}{3248.3873} & \ms{8.2373}{2.8540} \\ +StormCast & \ms{1.4522}{0.1519} & \ms{11155.7881}{2020.8656} & \ms{4.6142}{1.1500} \\ +DLWP & \ms{1.0952}{0.1306} & \ms{4406.9315}{303.0944} & \ms{1.7357}{0.3625} \\ +FCN & \ms{1.1688}{0.1139} & \ms{5166.9993}{213.0333} & \ms{2.0800}{0.4004} \\ +FengWu & \ms{1.1589}{0.1772} & \ms{5137.2822}{628.7543} & \ms{2.0944}{0.4545} \\ +FuXi & \ms{1.1855}{0.0612} & \ms{5697.7117}{796.8785} & \ms{2.4411}{0.5567} \\ +Pangu-Weather & \ms{1.1221}{0.1470} & \ms{5092.3621}{483.8243} & \ms{1.9571}{0.3113} \\ +AlphaEarth & \ms{1.7459}{0.6057} & \ms{15110.7573}{7106.3417} & \ms{9.7398}{2.7425} \\ +\bottomrule +\end{tabular} +\end{table*} + +\begin{table*}[t] +\centering +\scriptsize +\setlength{\tabcolsep}{3pt} +\caption{For fixed retrieval \(\mathcal{T}\) and \(\Omega\), this table reports nDCG@5, best log gap, and rank \(\rho\) in addition to the main nDCG@10/log-error metrics. Cells report mean with small std.} +\label{tab:app_analog_rank_depth} +\begin{tabular}{lccc} +\toprule +Backbone & nDCG@5 & best log gap & rank $\rho$ \\ +\midrule +\ourfm & \ms{0.5175}{0.0445} & \ms{0.1868}{0.0285} & \ms{0.6019}{0.1460} \\ +Prithvi-WxC & \ms{0.3591}{0.0107} & \ms{0.2151}{0.0594} & \ms{0.1514}{0.1489} \\ +Aurora & \ms{0.4423}{0.0210} & \ms{0.1551}{0.0437} & \ms{0.2162}{0.1856} \\ +ClimaX & \ms{0.4151}{0.0293} & \ms{0.2129}{0.0653} & \ms{0.1587}{0.2831} \\ +StormCast & \ms{0.3960}{0.0240} & \ms{0.1714}{0.0310} & \ms{0.1258}{0.1625} \\ +DLWP & \ms{0.3795}{0.0274} & \ms{0.1944}{0.0807} & \ms{-0.3865}{0.2802} \\ +FCN & \ms{0.4250}{0.0112} & \ms{0.1856}{0.0846} & \ms{-0.1357}{0.2571} \\ +FengWu & \ms{0.4228}{0.0310} & \ms{0.1870}{0.0858} & \ms{-0.1926}{0.2194} \\ +FuXi & \ms{0.4544}{0.0356} & \ms{0.2171}{0.0806} & \ms{-0.1367}{0.2885} \\ +Pangu-Weather & \ms{0.3988}{0.0506} & \ms{0.1901}{0.0838} & \ms{-0.1970}{0.2216} \\ +AlphaEarth & \ms{0.5276}{0.0531} & \ms{0.1782}{0.0454} & \ms{0.4639}{0.2802} \\ +\bottomrule +\end{tabular} +\end{table*} + +\begin{table*}[t] +\centering +\scriptsize +\setlength{\tabcolsep}{3pt} +\caption{For fixed smoke \(\mathcal{T}\) and station \(\Omega\), this table reports RMSE, MAE, and 90th-percentile absolute error on test rows with observed PM$_{2.5}\ge35$; std uses a row bootstrap over those rows. Cells report mean with small std.} +\label{tab:app_smoke_high_event} +\begin{tabular}{lccc} +\toprule +Backbone & high-smoke RMSE & high-smoke MAE & high-smoke 90th AE \\ +\midrule +\ourfm & \ms{47.4870}{0.6346} & \ms{34.3954}{0.7654} & \ms{65.6213}{3.8778} \\ +Prithvi-WxC & \ms{57.2224}{1.7268} & \ms{47.3871}{0.3153} & \ms{74.9666}{3.2381} \\ +Aurora & \ms{57.2752}{1.7248} & \ms{47.4368}{0.3149} & \ms{75.0755}{3.1074} \\ +ClimaX & \ms{57.2828}{1.7239} & \ms{47.4407}{0.3140} & \ms{75.1012}{3.0777} \\ +StormCast & \ms{56.6512}{1.7517} & \ms{46.7914}{0.3281} & \ms{74.0794}{3.4707} \\ +DLWP & \ms{57.0075}{1.7359} & \ms{47.1971}{0.3198} & \ms{74.4936}{3.3826} \\ +FCN & \ms{57.0582}{1.7339} & \ms{47.2401}{0.3187} & \ms{74.6431}{3.1982} \\ +FengWu & \ms{57.0158}{1.7357} & \ms{47.1957}{0.3194} & \ms{74.5652}{3.2871} \\ +FuXi & \ms{56.9622}{1.7371} & \ms{47.1508}{0.3201} & \ms{74.3278}{3.4435} \\ +Pangu-Weather & \ms{57.1282}{1.7307} & \ms{47.3050}{0.3170} & \ms{74.6830}{3.2375} \\ +AlphaEarth & \ms{48.0665}{0.7904} & \ms{35.6088}{0.7341} & \ms{66.7613}{3.9235} \\ +\bottomrule +\end{tabular} +\end{table*} + +\begin{table*}[t] +\centering +\scriptsize +\setlength{\tabcolsep}{3pt} +\caption{For fixed heat \(\mathcal{T}\) and heat-region \(\Omega\), this table reports precision and recall for the exceedance label used by the main \(F_1\). Cells report mean with small std.} +\label{tab:app_heat_event_pr} +\begin{tabular}{lcc} +\toprule +Backbone & precision & recall \\ +\midrule +\ourfm & \ms{0.9767}{0.0117} & \ms{0.9330}{0.0299} \\ +Prithvi-WxC & \ms{0.8260}{0.0030} & \ms{0.9173}{0.0033} \\ +Aurora & \ms{0.5920}{0.0347} & \ms{0.0517}{0.0020} \\ +ClimaX & \ms{0.7397}{0.0099} & \ms{0.7994}{0.0051} \\ +StormCast & \ms{0.8840}{0.0237} & \ms{0.9320}{0.0165} \\ +DLWP & \ms{0.9429}{0.0085} & \ms{0.8899}{0.0167} \\ +FCN & \ms{0.9408}{0.0097} & \ms{0.9111}{0.0127} \\ +FengWu & \ms{0.3808}{0.2719} & \ms{0.0266}{0.0267} \\ +FuXi & \ms{0.3262}{0.1262} & \ms{0.1810}{0.0481} \\ +Pangu-Weather & \ms{0.1159}{0.0743} & \ms{0.0112}{0.0032} \\ +AlphaEarth & \ms{0.9824}{0.0040} & \ms{0.9278}{0.0178} \\ +\bottomrule +\end{tabular} +\end{table*} + +\clearpage + +% ============================================================ +% C COMPARATOR ELIGIBILITY NOTES +% ============================================================ +\section{Comparator Eligibility Notes} +\label{sec:comparator_audit} + +All numeric comparator rows in Tables~\ref{tab:primary_results} and~\ref{tab:supporting_results} +are included only after the task form, metric, matching rule, scope, and head family are fixed. +The appendix does not repeat those full matrices. +The key eligibility rule is simple: reported rows satisfy the same contract as the row block in which they appear, while excluded rows are excluded because their representation or output form does not satisfy that contract. + +\noindent\textbf{Reading rule.} +Exact-only, tolerated, union, ranking, retrieval, and regression scores answer different questions. +The fixed-contract reading is therefore to compare entries only within one row block and not to average across task forms. + +\clearpage + +% ============================================================ +% D SEEDED AUDITS +% ============================================================ +\section{Seeded Audits} +\label{sec:app_seeded_audits} + +\subsection{Seed Robustness Summary} +\label{sec:app_seed_robustness} + +Table~\ref{tab:app_seed_robustness} summarizes stochastic checks used to support the reported mean-with-std convention. +It is not a replacement for the main fixed-contract result tables. + +\begin{table}[h] +\centering +\small +\setlength{\tabcolsep}{5pt} +\renewcommand{\arraystretch}{1.2} +\caption{Seed summaries for stochastic checks. Values report mean with small std over completed seeds.} +\label{tab:app_seed_robustness} +\begin{adjustbox}{max width=\textwidth} +\begin{tabular}{p{0.28\textwidth}cllp{0.18\textwidth}} +\toprule +\textbf{\(\mathcal{T}\) check} & \textbf{Seeds} & \textbf{Primary value} & \textbf{Other value(s)} & \textbf{Reading} \\ +\midrule +Final burned area & +5 & log-RMSE \ms{1.1649}{0.0139} & +log-MAE \ms{1.0355}{0.0114}; Spear.\ \ms{0.5884}{0.0338} & +stable across seeds \\ +Smoke PM\(_{2.5}\) & +5 & RMSE \ms{4.4684}{0.0060} & +MAE \ms{2.4127}{0.0016}; \(r\) \ms{0.6360}{0.0013} & +stable at table precision \\ +Extreme heat & +5 & RMSE-C \ms{0.2179}{0.0048} & +MAE-C \ms{0.1787}{0.0020}; exceed.\ \(F_1\) \ms{0.9541}{0.0164} & +stable across seeds \\ +Fire spread & +5 & exact \(F_1\) \ms{37.98}{1.33} & +spatial \(F_1\) \ms{82.20}{0.32}; AP \ms{32.73}{0.60} & +stable across seeds \\ +Aurora paired-head check & +5 & fire-prone score diff.\ \ms{6.3500}{13.2800} & +PR-AUC and union choices differ in 2/5 seeds & +variable across seeds \\ +\bottomrule +\end{tabular} +\end{adjustbox} +\end{table} + +\clearpage + +% ============================================================ +% E LIGHTWEIGHT HEAD AND ADAPTATION DETAILS +% ============================================================ +\section{Lightweight Head and Adaptation Details} +\label{sec:app_heads} + +All frozen-transfer comparisons use the same five lightweight head architectures applied +on top of the frozen backbone representations. +Table~\ref{tab:app_head_architectures} summarises each head family, its architecture, +approximate parameter count, and the adaptation procedure used. + +\begin{table}[h] +\centering +\small +\setlength{\tabcolsep}{5pt} +\renewcommand{\arraystretch}{1.3} +\caption{Lightweight head architectures used in the fixed-contract transfer comparisons. +All heads are trained from random initialisation on the frozen backbone features. +Parameter counts are approximate and depend on the feature dimensionality of each backbone.} +\label{tab:app_head_architectures} +\begin{tabular}{p{0.15\textwidth}p{0.30\textwidth}p{0.12\textwidth}p{0.33\textwidth}} +\toprule +\textbf{$\mathcal{A}$ head} & \textbf{Architecture} & \textbf{Approx.\ params} & \textbf{Notes} \\ +\midrule +Constant prior & + Outputs a fixed bias vector, ignoring input features. & + Output dimension only & + Provides a degenerate baseline; selected when backbone features carry no useful signal. \\ +Linear probe & + Single linear layer mapping backbone features to output. No nonlinearity. & + $d\times c + c$ & + Standard frozen-representation baseline. \\ +Pixel MLP & + Two-layer MLP applied independently per spatial unit. & + $d\times h + h\times c$ & + Captures per-pixel nonlinearity; ignores spatial context. \\ +Shallow adapter & + Two-layer MLP with a spatial context window; uses $3\times3$ convolution before the linear output. & + $9dh + hc$ & + Balances local spatial context with parameter efficiency. \\ +Wide adapter & + Shallow adapter with wider hidden dimension. & + $9dH + Hc$ & + Higher capacity variant; can overfit on small fire-event sets. \\ +\bottomrule +\end{tabular} +\end{table} + +\noindent\textbf{Training protocol.} +Each occupancy head-control run uses seeds $\{1,7,42,99,123\}$, the five heads listed above, and the fixed variants identity, erode-r1, and close-r1. +The spread U-Net reference is trained for 4 epochs. +The threshold $\tau$ is selected on the validation split by maximising union-$F_1$ (for occupancy) or spatial $F_1$ (for spread) and held fixed at test time. +Morphology parameters (spatial tolerance $k$, temporal tolerance $\Delta t$) are fixed as part of the evaluation contract and are not tuned after validation. + +\noindent\textbf{Head selection procedure.} +For each (feature source, scope, seed) tuple, all five heads are trained independently. +The PR-AUC-based selector picks $h_R = \arg\max_{h \in \mathcal{H}} R(h)$ on the validation set; +the decision-based selector picks $h_D = \arg\max_{h \in \mathcal{H}} D(h)$ on the same set. +The selection regret $\delta = D(h_D) - D(h_R) \ge 0$ is computed on the held-out test set. + +\clearpage + +% ============================================================ +% F LIMITATIONS +% ============================================================ +\section{Limitations} +\label{sec:limitations} + +The conclusions apply to the task forms, scopes, evaluation rules, and comparator eligibility decisions used in this study. +The evaluation covers selected wildfire decision tasks and supporting retrieval and regression task forms. +Comparator eligibility is fixed before metric values are interpreted. +This eligibility rule keeps each comparison within one task-form contract. +It also leaves some model and task pairs outside the evaluated comparison set by design. + +The transfer comparison uses frozen backbones with lightweight heads. +The results therefore describe frozen-backbone transfer under the allowed head families in each contract. +Full fine-tuning, alternative adaptation procedures, and broader head families are outside the evaluated scope. +The task-specific reference baselines serve as empirical anchors for same-contract comparison. +\ourfm is a Lower-48 wildfire reference for the reported fixed-contract experiments. + + +The supporting retrieval and regression checks bound the primary spatial decision claim. +They provide task-form evidence rather than a single score across all wildfire-related prediction tasks. +The analysis focuses on the reported metric families, matching rules, and fixed comparison choices. +Operational response rules, intervention costs, and deployment policies are part of wildfire early-warning use contexts~\cite{goldammer1999early,pickell2017early,farahmand2020fdeo}. +They are outside the scope of this evaluation study and are not inferred from the reported scores. + +\clearpage + +% ============================================================ +% G REPRODUCIBILITY AND EVALUATION ARTIFACTS +% ============================================================ +\section{Reproducibility and Evaluation Artifacts} +\label{sec:repro_compute_impact} +\subsection{External Assets and Terms of Use} +\label{sec:external_assets_terms} + +We use external datasets and model assets only for research evaluation. +Access to each asset follows the original provider's portal, license, or terms of use; this submission does not imply that all assets are openly redistributable. +We do not redistribute raw external datasets, provider-hosted embeddings, or third-party model weights. +Table~\ref{tab:external_assets_licenses} records the source and terms-of-use status used to interpret reproducibility. + +\begin{table}[h] +\centering +\small +\setlength{\tabcolsep}{4pt} +\renewcommand{\arraystretch}{1.18} +\caption{External assets used by the study and their source or terms-of-use status.} +\label{tab:external_assets_licenses} +\begin{tabular}{p{0.25\textwidth}p{0.34\textwidth}p{0.34\textwidth}} +\toprule +\textbf{Asset family} & \textbf{Use in this study} & \textbf{Source and terms-of-use note} \\ +\midrule +NOAA HRRR fields~\cite{noaa_hrrr_ncei,noaa_hrrr_emc} +& Dynamic weather inputs for \ourfm and transfer tasks. +& NOAA provider terms and citation requirements apply. \\ +NASA FIRMS~\cite{nasa_firms} +& Active-fire occupancy supervision. +& NASA Earthdata/FIRMS access terms and citation requirements apply. \\ +LANDFIRE and WRC layers~\cite{landfire_fbfm40,landfire_canopy_cover,usfs_wrc_housing_density} +& Static fuel, canopy, and exposure context. +& Original geospatial-product provider terms and citations apply. \\ +LandScan~\cite{ornl_landscan_2024} +& Static population context. +& ORNL/LandScan source-specific access terms apply; raw data are not redistributed. \\ +WFIGS and MTBS~\cite{nifc_wfigs_perimeters,mtbs_usgs_2025} +& Event-level resources for burned-area and analog tasks. +& Original incident/perimeter-product provider terms and citations apply. \\ +External Earth-FM baselines~\cite{schmude2024prithviwxc,bodnar2025aurora,nguyen2023climax,pathak2024stormcast,weyn2020dlwp,pathak2022fourcastnet,chen2023fengwu,chen2023fuxi,bi2023panguweather,brown2025alphaearth} +& Frozen comparator representations or task-model baselines. +& Original model-provider licenses and access terms apply; third-party weights are not redistributed. \\ +\bottomrule +\end{tabular} +\end{table} + +This note supports the reproducibility statement and identifies the files that support the reported claims. +This file statement does not imply full raw-data release. +The main claims can be checked from the manuscript contracts, metric +definitions, and per-head result files, even if full raw-data release is +delayed or limited. Sections~3 and~4 specify the contract components used by +the main claims: task definition, split logic, label space, tolerance +parameters, scope definitions, threshold or operating-point rules, and +lightweight-head set. + +The manuscript source includes the \LaTeX{} result tables and figures for the expanded check and matching-rule support. +The retained result artifacts expose exact \(F_1\), +tolerated \(F_1\), union-\(F_1\), PR-AUC, per-head selection, +top-1 agreement, and selection-regret arithmetic. The manuscript also includes +full figure and table reproduction values in result tables and appendix tables. +These artifacts provide a check of the +selection-regret arithmetic and the table-construction logic from fixed +per-head rows. The seeded occupancy check uses seeds +$\{1,7,42,99,123\}$, and the spread task-specific U-Net check uses repeated seeds; reported error bars are standard deviations over the completed +seeded runs. Full raw wildfire inputs and large feature arrays are not +released at submission because redistribution and storage constraints require a +separate review. + +For stochastic results, the paper reports mean with standard deviation over repeated seeds. +For fixed-output or fixed-feature controls, the table uses one fixed output or feature set; the changed item is the matching rule or selection metric. + +The reported experiments use two resource classes on a shared Slurm-managed +cluster. Tabular retrieval/regression checks and same-feature head controls run +on CPU workers with 4 to 8 cores, 24 to 64~GB host memory, and 2 to 4~hour wall-clock +limits. Spread U-Net training and threshold calibration run on single-GPU jobs +with one B200 GPU, 8 CPU cores, 96~GB host memory, and a 4~hour wall-clock +limit. The seed/check waves reported in the appendix correspond to roughly +78 CPU job-hours and 12 GPU job-hours of scheduled wall-clock budget; +exploratory runs are not included in the reported compute accounting. + +The raw-data limitation is separate from the selection-regret files. +The supplementary source is sufficient to inspect the selection-regret arithmetic and reproduce the reported tables. +Full end-to-end recomputation from raw wildfire inputs is not included at submission because redistribution review is still required. +The broader impact is evaluation-facing rather than operational. +Better reading of wildfire transfer evidence can reduce overconfident benchmark claims, while misread transfer results could still encourage inappropriate reliance on models with low decision scores. +For that reason, the paper keeps its claims wildfire-centered, decision-task +specific, and explicitly separate from any predictive deployment +recommendation. diff --git a/paper/wildfire_fm_evaluation_contracts.pdf b/paper/wildfire_fm_evaluation_contracts.pdf deleted file mode 100644 index 98c5ab9e7d023669b5260ee0fdc8cef0a6d63892..0000000000000000000000000000000000000000 --- a/paper/wildfire_fm_evaluation_contracts.pdf +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:95596f05829639ecb3041721fe3c871caacc49f30c831d35f2c503b1073b498d -size 1994795 diff --git a/paper_outputs/figures/fig_fireprone_contract_progression_compact.pdf b/paper_outputs/figures/fig_fireprone_contract_progression_compact.pdf deleted file mode 100644 index 0e685407b7b36b947363ae871a81d496391c859c..0000000000000000000000000000000000000000 --- a/paper_outputs/figures/fig_fireprone_contract_progression_compact.pdf +++ /dev/null @@ -1,262 +0,0 @@ -%PDF-1.4 -%âãÏÓ -1 0 obj -<< /Type /Catalog /Pages 2 0 R >> -endobj -2 0 obj -<< /Type /Pages /Kids [3 0 R] /Count 1 >> -endobj -3 0 obj -<< /Type /Page /Parent 2 0 R /MediaBox [0 0 1320 470] /Resources << /Font << /F1 4 0 R /F2 5 0 R >> >> /Contents 6 0 R >> -endobj -4 0 obj -<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >> -endobj -5 0 obj -<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica-Bold >> -endobj -6 0 obj -<< /Length 19373 >> -stream -1.0000 1.0000 1.0000 rg 0.00 0.00 1320.00 470.00 re f -0.80 w 0.1500 0.1500 0.1500 RG 72.00 132.00 m 72.00 400.00 l S -0.80 w 0.1500 0.1500 0.1500 RG 72.00 132.00 m 1266.00 132.00 l S -0.45 w 0.8600 0.8600 0.8600 RG 68.00 132.00 m 1266.00 132.00 l S -BT /F1 7.00 Tf 0.2500 0.2500 0.2500 rg 1 0 0 1 60.36 129.00 Tm (0) Tj ET -0.45 w 0.8600 0.8600 0.8600 RG 68.00 199.00 m 1266.00 199.00 l S -BT /F1 7.00 Tf 0.2500 0.2500 0.2500 rg 1 0 0 1 56.72 196.00 Tm (20) Tj ET -0.45 w 0.8600 0.8600 0.8600 RG 68.00 266.00 m 1266.00 266.00 l S -BT /F1 7.00 Tf 0.2500 0.2500 0.2500 rg 1 0 0 1 56.72 263.00 Tm (40) Tj ET -0.45 w 0.8600 0.8600 0.8600 RG 68.00 333.00 m 1266.00 333.00 l S -BT /F1 7.00 Tf 0.2500 0.2500 0.2500 rg 1 0 0 1 56.72 330.00 Tm (60) Tj ET -0.45 w 0.8600 0.8600 0.8600 RG 68.00 400.00 m 1266.00 400.00 l S -BT /F1 7.00 Tf 0.2500 0.2500 0.2500 rg 1 0 0 1 56.72 397.00 Tm (80) Tj ET -BT /F2 8.00 Tf 0.1500 0.1500 0.1500 rg 1 0 0 1 34.00 408.00 Tm (F1 \(%\)) Tj ET -BT /F2 15.00 Tf 0.0000 0.0000 0.0000 rg 1 0 0 1 202.93 417.00 Tm (global) Tj ET -0.75 w 0.4200 0.4400 0.4600 RG 380.50 126.00 m 380.50 133.00 l S -0.75 w 0.4200 0.4400 0.4600 RG 380.50 138.00 m 380.50 145.00 l S -0.75 w 0.4200 0.4400 0.4600 RG 380.50 150.00 m 380.50 157.00 l S -0.75 w 0.4200 0.4400 0.4600 RG 380.50 162.00 m 380.50 169.00 l S -0.75 w 0.4200 0.4400 0.4600 RG 380.50 174.00 m 380.50 181.00 l S -0.75 w 0.4200 0.4400 0.4600 RG 380.50 186.00 m 380.50 193.00 l S -0.75 w 0.4200 0.4400 0.4600 RG 380.50 198.00 m 380.50 205.00 l S -0.75 w 0.4200 0.4400 0.4600 RG 380.50 210.00 m 380.50 217.00 l S -0.75 w 0.4200 0.4400 0.4600 RG 380.50 222.00 m 380.50 229.00 l S -0.75 w 0.4200 0.4400 0.4600 RG 380.50 234.00 m 380.50 241.00 l S -0.75 w 0.4200 0.4400 0.4600 RG 380.50 246.00 m 380.50 253.00 l S -0.75 w 0.4200 0.4400 0.4600 RG 380.50 258.00 m 380.50 265.00 l S -0.75 w 0.4200 0.4400 0.4600 RG 380.50 270.00 m 380.50 277.00 l S -0.75 w 0.4200 0.4400 0.4600 RG 380.50 282.00 m 380.50 289.00 l S -0.75 w 0.4200 0.4400 0.4600 RG 380.50 294.00 m 380.50 301.00 l S -0.75 w 0.4200 0.4400 0.4600 RG 380.50 306.00 m 380.50 313.00 l S -0.75 w 0.4200 0.4400 0.4600 RG 380.50 318.00 m 380.50 325.00 l S -0.75 w 0.4200 0.4400 0.4600 RG 380.50 330.00 m 380.50 337.00 l S -0.75 w 0.4200 0.4400 0.4600 RG 380.50 342.00 m 380.50 349.00 l S -0.75 w 0.4200 0.4400 0.4600 RG 380.50 354.00 m 380.50 361.00 l S -0.75 w 0.4200 0.4400 0.4600 RG 380.50 366.00 m 380.50 373.00 l S -0.75 w 0.4200 0.4400 0.4600 RG 380.50 378.00 m 380.50 385.00 l S -0.75 w 0.4200 0.4400 0.4600 RG 380.50 390.00 m 380.50 397.00 l S -0.75 w 0.4200 0.4400 0.4600 RG 380.50 402.00 m 380.50 409.00 l S -0.75 w 0.4200 0.4400 0.4600 RG 380.50 414.00 m 380.50 416.00 l S -BT /F2 15.00 Tf 0.0000 0.0000 0.0000 rg 1 0 0 1 515.07 417.00 Tm (top 5%) Tj ET -BT /F2 15.00 Tf 0.0000 0.0000 0.0000 rg 1 0 0 1 807.48 417.00 Tm (top 10%) Tj ET -BT /F2 15.00 Tf 0.0000 0.0000 0.0000 rg 1 0 0 1 1103.97 417.00 Tm (top 20%) Tj ET -0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 84.02 132.00 18.00 1.52 re B -0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 84.02 133.52 18.00 98.13 re B -0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 84.02 231.66 18.00 98.21 re B -BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 76.52 69.51 Tm (Ref.) Tj ET -0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 110.07 132.00 18.00 0.19 re B -0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 110.07 132.19 18.00 23.82 re B -0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 110.07 156.00 18.00 43.62 re B -BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 102.21 69.86 Tm (WxC) Tj ET -0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 136.11 132.00 18.00 0.22 re B -0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 136.11 132.22 18.00 28.26 re B -0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 136.11 160.48 18.00 48.92 re B -BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 119.34 78.77 Tm (Aurora) Tj ET -0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 162.16 132.00 18.00 1.17 re B -0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 162.16 133.17 18.00 98.51 re B -0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 162.16 231.67 18.00 101.83 re B -BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 146.38 77.78 Tm (ClimaX) Tj ET -0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 188.20 132.00 18.00 0.21 re B -0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 188.20 132.21 18.00 27.24 re B -0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 188.20 159.45 18.00 47.52 re B -BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 172.99 77.21 Tm (Storm) Tj ET -0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 214.25 132.00 18.00 0.57 re B -0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 214.25 132.57 18.00 49.40 re B -0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 214.25 181.96 18.00 44.47 re B -BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 201.30 74.95 Tm (DLWP) Tj ET -0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 240.30 132.00 18.00 0.95 re B -0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 240.30 132.95 18.00 64.40 re B -0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 240.30 197.35 18.00 68.86 re B -BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 233.14 69.15 Tm (FCN) Tj ET -0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 266.34 132.00 18.00 0.88 re B -0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 266.34 132.88 18.00 39.34 re B -0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 266.34 172.22 18.00 40.53 re B -BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 247.45 80.89 Tm (FengWu) Tj ET -0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 292.39 132.00 18.00 1.26 re B -0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 292.39 133.26 18.00 69.19 re B -0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 292.39 202.46 18.00 54.46 re B -BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 284.17 70.21 Tm (FuXi) Tj ET -0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 318.43 132.00 18.00 0.92 re B -0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 318.43 132.92 18.00 56.33 re B -0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 318.43 189.25 18.00 62.13 re B -BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 295.86 84.57 Tm (Pangu-W) Tj ET -0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 344.48 132.00 18.00 6.90 re B -0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 344.48 138.90 18.00 91.75 re B -0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 344.48 230.65 18.00 26.74 re B -BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 333.29 73.18 Tm (Alpha) Tj ET -0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 398.52 132.00 18.00 11.93 re B -0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 398.52 143.93 18.00 119.60 re B -0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 398.52 263.53 18.00 112.45 re B -BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 391.02 69.51 Tm (Ref.) Tj ET -0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 424.57 132.00 18.00 4.73 re B -0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 424.57 136.73 18.00 59.80 re B -0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 424.57 196.53 18.00 78.11 re B -BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 416.71 69.86 Tm (WxC) Tj ET -0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 450.61 132.00 18.00 3.30 re B -0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 450.61 135.30 18.00 47.40 re B -0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 450.61 182.70 18.00 68.17 re B -BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 433.84 78.77 Tm (Aurora) Tj ET -0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 476.66 132.00 18.00 4.33 re B -0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 476.66 136.33 18.00 111.51 re B -0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 476.66 247.84 18.00 116.04 re B -BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 460.88 77.78 Tm (ClimaX) Tj ET -0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 502.70 132.00 18.00 3.21 re B -0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 502.70 135.21 18.00 48.12 re B -0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 502.70 183.33 18.00 69.89 re B -BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 487.49 77.21 Tm (Storm) Tj ET -0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 528.75 132.00 18.00 6.05 re B -0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 528.75 138.05 18.00 100.22 re B -0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 528.75 238.27 18.00 79.52 re B -BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 515.80 74.95 Tm (DLWP) Tj ET -0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 554.80 132.00 18.00 5.44 re B -0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 554.80 137.44 18.00 92.98 re B -0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 554.80 230.41 18.00 83.50 re B -BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 547.64 69.15 Tm (FCN) Tj ET -0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 580.84 132.00 18.00 5.26 re B -0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 580.84 137.26 18.00 49.27 re B -0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 580.84 186.53 18.00 46.33 re B -BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 561.95 80.89 Tm (FengWu) Tj ET -0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 606.89 132.00 18.00 6.80 re B -0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 606.89 138.80 18.00 100.04 re B -0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 606.89 238.85 18.00 73.82 re B -BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 598.67 70.21 Tm (FuXi) Tj ET -0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 632.93 132.00 18.00 4.57 re B -0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 632.93 136.57 18.00 69.87 re B -0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 632.93 206.44 18.00 71.02 re B -BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 610.36 84.57 Tm (Pangu-W) Tj ET -0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 658.98 132.00 18.00 23.16 re B -0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 658.98 155.16 18.00 120.48 re B -0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 658.98 275.64 18.00 29.70 re B -BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 647.79 73.18 Tm (Alpha) Tj ET -0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 695.02 132.00 18.00 11.92 re B -0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 695.02 143.92 18.00 119.29 re B -0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 695.02 263.21 18.00 111.74 re B -BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 687.52 69.51 Tm (Ref.) Tj ET -0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 721.07 132.00 18.00 4.15 re B -0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 721.07 136.15 18.00 45.70 re B -0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 721.07 181.84 18.00 59.67 re B -BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 713.21 69.86 Tm (WxC) Tj ET -0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 747.11 132.00 18.00 2.61 re B -0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 747.11 134.61 18.00 40.06 re B -0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 747.11 174.67 18.00 59.60 re B -BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 730.34 78.77 Tm (Aurora) Tj ET -0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 773.16 132.00 18.00 4.19 re B -0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 773.16 136.19 18.00 110.82 re B -0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 773.16 247.02 18.00 114.69 re B -BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 757.38 77.78 Tm (ClimaX) Tj ET -0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 799.20 132.00 18.00 2.44 re B -0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 799.20 134.44 18.00 39.99 re B -0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 799.20 174.43 18.00 59.66 re B -BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 783.99 77.21 Tm (Storm) Tj ET -0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 825.25 132.00 18.00 5.40 re B -0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 825.25 137.40 18.00 87.26 re B -0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 825.25 224.65 18.00 65.22 re B -BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 812.30 74.95 Tm (DLWP) Tj ET -0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 851.30 132.00 18.00 3.95 re B -0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 851.30 135.95 18.00 71.17 re B -0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 851.30 207.11 18.00 70.45 re B -BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 844.14 69.15 Tm (FCN) Tj ET -0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 877.34 132.00 18.00 4.16 re B -0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 877.34 136.16 18.00 39.22 re B -0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 877.34 175.38 18.00 37.64 re B -BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 858.45 80.89 Tm (FengWu) Tj ET -0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 903.39 132.00 18.00 5.54 re B -0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 903.39 137.54 18.00 74.90 re B -0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 903.39 212.44 18.00 54.27 re B -BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 895.17 70.21 Tm (FuXi) Tj ET -0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 929.43 132.00 18.00 3.66 re B -0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 929.43 135.66 18.00 59.77 re B -0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 929.43 195.43 18.00 65.66 re B -BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 906.86 84.57 Tm (Pangu-W) Tj ET -0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 955.48 132.00 18.00 22.23 re B -0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 955.48 154.23 18.00 118.13 re B -0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 955.48 272.36 18.00 29.05 re B -BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 944.29 73.18 Tm (Alpha) Tj ET -0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 991.52 132.00 18.00 11.83 re B -0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 991.52 143.83 18.00 116.43 re B -0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 991.52 260.25 18.00 105.32 re B -BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 984.02 69.51 Tm (Ref.) Tj ET -0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 1017.57 132.00 18.00 3.86 re B -0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 1017.57 135.86 18.00 40.20 re B -0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 1017.57 176.06 18.00 50.19 re B -BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 1009.71 69.86 Tm (WxC) Tj ET -0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 1043.61 132.00 18.00 2.23 re B -0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 1043.61 134.23 18.00 33.05 re B -0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 1043.61 167.28 18.00 48.29 re B -BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 1026.84 78.77 Tm (Aurora) Tj ET -0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 1069.66 132.00 18.00 3.45 re B -0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 1069.66 135.45 18.00 97.77 re B -0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 1069.66 233.22 18.00 100.00 re B -BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 1053.88 77.78 Tm (ClimaX) Tj ET -0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 1095.70 132.00 18.00 1.94 re B -0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 1095.70 133.94 18.00 32.95 re B -0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 1095.70 166.89 18.00 47.72 re B -BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 1080.49 77.21 Tm (Storm) Tj ET -0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 1121.75 132.00 18.00 5.11 re B -0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 1121.75 137.11 18.00 65.04 re B -0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 1121.75 202.15 18.00 46.87 re B -BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 1108.80 74.95 Tm (DLWP) Tj ET -0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 1147.80 132.00 18.00 3.34 re B -0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 1147.80 135.34 18.00 53.54 re B -0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 1147.80 188.88 18.00 57.31 re B -BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 1140.64 69.15 Tm (FCN) Tj ET -0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 1173.84 132.00 18.00 3.75 re B -0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 1173.84 135.75 18.00 36.29 re B -0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 1173.84 172.04 18.00 36.30 re B -BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 1154.95 80.89 Tm (FengWu) Tj ET -0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 1199.89 132.00 18.00 4.57 re B -0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 1199.89 136.57 18.00 68.98 re B -0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 1199.89 205.55 18.00 49.50 re B -BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 1191.67 70.21 Tm (FuXi) Tj ET -0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 1225.93 132.00 18.00 2.96 re B -0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 1225.93 134.96 18.00 54.04 re B -0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 1225.93 189.01 18.00 58.80 re B -BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 1203.36 84.57 Tm (Pangu-W) Tj ET -0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 1251.98 132.00 18.00 20.74 re B -0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 1251.98 152.74 18.00 109.35 re B -0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 1251.98 262.09 18.00 25.30 re B -BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 1240.79 73.18 Tm (Alpha) Tj ET -0.45 w 0.9800 0.9800 0.9600 rg 0.7800 0.8000 0.7800 RG 77.00 362.00 304.00 23.00 re B -0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 90.00 371.00 24.00 9.00 re B -BT /F1 8.00 Tf 0.0000 0.0000 0.0000 rg 1 0 0 1 121.00 373.00 Tm (Strict) Tj ET -0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 188.00 371.00 24.00 9.00 re B -BT /F1 8.00 Tf 0.0000 0.0000 0.0000 rg 1 0 0 1 219.00 373.00 Tm (Tolerance) Tj ET -0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 286.00 371.00 24.00 9.00 re B -BT /F1 8.00 Tf 0.0000 0.0000 0.0000 rg 1 0 0 1 317.00 373.00 Tm (Union) Tj ET -endstream -endobj -xref -0 7 -0000000000 65535 f -0000000015 00000 n -0000000064 00000 n -0000000121 00000 n -0000000258 00000 n -0000000328 00000 n -0000000403 00000 n -trailer -<< /Size 7 /Root 1 0 R >> -startxref -19829 -%%EOF diff --git a/paper_outputs/figures/fig_primary_rank_change_map.pdf b/paper_outputs/figures/fig_primary_rank_change_map.pdf deleted file mode 100644 index 87993317c7a501f04d6924f004d5d4ed83af6a95..0000000000000000000000000000000000000000 --- a/paper_outputs/figures/fig_primary_rank_change_map.pdf +++ /dev/null @@ -1,133 +0,0 @@ -%PDF-1.4 -%âãÏÓ -1 0 obj -<< /Type /Catalog /Pages 2 0 R >> -endobj -2 0 obj -<< /Type /Pages /Kids [3 0 R] /Count 1 >> -endobj -3 0 obj -<< /Type /Page /Parent 2 0 R /MediaBox [0 0 205 150] /Resources << /Font << /F1 4 0 R /F2 5 0 R >> >> /Contents 6 0 R >> -endobj -4 0 obj -<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >> -endobj -5 0 obj -<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica-Bold >> -endobj -6 0 obj -<< /Length 7493 >> -stream -1.0000 1.0000 1.0000 rg 0.00 0.00 205.00 150.40 re f -BT /F2 5.45 Tf 0.0300 0.2000 0.3300 rg 1 0 0 1 22.18 113.30 Tm (WildFIRE-FM) Tj ET -0.9850 0.9860 0.9870 rg 0.00 99.60 205.00 10.40 re f -BT /F1 5.45 Tf 0.1200 0.1300 0.1400 rg 1 0 0 1 25.04 102.90 Tm (Prithvi-WxC) Tj ET -BT /F1 5.45 Tf 0.1200 0.1300 0.1400 rg 1 0 0 1 38.45 92.50 Tm (Aurora) Tj ET -0.9850 0.9860 0.9870 rg 0.00 78.80 205.00 10.40 re f -BT /F1 5.45 Tf 0.1200 0.1300 0.1400 rg 1 0 0 1 39.21 82.10 Tm (ClimaX) Tj ET -BT /F1 5.45 Tf 0.1200 0.1300 0.1400 rg 1 0 0 1 27.77 71.70 Tm (StormCast) Tj ET -0.9850 0.9860 0.9870 rg 0.00 58.00 205.00 10.40 re f -BT /F1 5.45 Tf 0.1200 0.1300 0.1400 rg 1 0 0 1 41.39 61.30 Tm (DLWP) Tj ET -BT /F1 5.45 Tf 0.1200 0.1300 0.1400 rg 1 0 0 1 45.86 50.90 Tm (FCN) Tj ET -0.9850 0.9860 0.9870 rg 0.00 37.20 205.00 10.40 re f -BT /F1 5.45 Tf 0.1200 0.1300 0.1400 rg 1 0 0 1 36.82 40.50 Tm (FengWu) Tj ET -BT /F1 5.45 Tf 0.1200 0.1300 0.1400 rg 1 0 0 1 45.05 30.10 Tm (FuXi) Tj ET -0.9850 0.9860 0.9870 rg 0.00 16.40 205.00 10.40 re f -BT /F1 5.45 Tf 0.1200 0.1300 0.1400 rg 1 0 0 1 16.98 19.70 Tm (Pangu-Weather) Tj ET -BT /F1 5.45 Tf 0.1200 0.1300 0.1400 rg 1 0 0 1 28.04 9.30 Tm (AlphaEarth) Tj ET -BT /F2 6.50 Tf 0.1200 0.1300 0.1400 rg 1 0 0 1 76.09 141.00 Tm (Occupancy top-5%) Tj ET -0.60 w 0.4200 0.4300 0.4400 RG 60.00 6.00 m 60.00 136.90 l S -0.60 w 0.4200 0.4300 0.4400 RG 148.00 6.00 m 148.00 136.90 l S -BT /F2 6.50 Tf 0.1200 0.1300 0.1400 rg 1 0 0 1 156.33 141.00 Tm (Spread region) Tj ET -0.60 w 0.4200 0.4300 0.4400 RG 156.00 6.00 m 156.00 136.90 l S -0.60 w 0.4200 0.4300 0.4400 RG 200.00 6.00 m 200.00 136.90 l S -BT /F2 5.00 Tf 0.1200 0.1300 0.1400 rg 1 0 0 1 72.63 128.90 Tm (Ex->Tol) Tj ET -0.45 w 0.9400 0.9500 0.9500 rg 1.0000 1.0000 1.0000 RG 60.00 110.00 44.00 10.40 re B -BT /F2 5.10 Tf 0.0800 0.0900 0.1000 rg 1 0 0 1 75.04 113.30 Tm (#2->2) Tj ET -0.45 w 0.9540 0.8570 0.8220 rg 1.0000 1.0000 1.0000 RG 60.00 99.60 44.00 10.40 re B -BT /F2 5.10 Tf 0.0800 0.0900 0.1000 rg 1 0 0 1 75.04 102.90 Tm (#7->8) Tj ET -0.45 w 0.9540 0.8570 0.8220 rg 1.0000 1.0000 1.0000 RG 60.00 89.20 44.00 10.40 re B -BT /F2 5.10 Tf 0.0800 0.0900 0.1000 rg 1 0 0 1 72.25 92.50 Tm (#10->11) Tj ET -0.45 w 0.4240 0.7000 0.6560 rg 1.0000 1.0000 1.0000 RG 60.00 78.80 44.00 10.40 re B -BT /F2 5.10 Tf 0.0800 0.0900 0.1000 rg 1 0 0 1 75.04 82.10 Tm (#9->3) Tj ET -0.45 w 0.8290 0.9250 0.9010 rg 1.0000 1.0000 1.0000 RG 60.00 68.40 44.00 10.40 re B -BT /F2 5.10 Tf 0.0800 0.0900 0.1000 rg 1 0 0 1 72.25 71.70 Tm (#11->10) Tj ET -0.45 w 0.9540 0.8570 0.8220 rg 1.0000 1.0000 1.0000 RG 60.00 58.00 44.00 10.40 re B -BT /F2 5.10 Tf 0.0800 0.0900 0.1000 rg 1 0 0 1 75.04 61.30 Tm (#4->5) Tj ET -0.45 w 0.9540 0.8570 0.8220 rg 1.0000 1.0000 1.0000 RG 60.00 47.60 44.00 10.40 re B -BT /F2 5.10 Tf 0.0800 0.0900 0.1000 rg 1 0 0 1 75.04 50.90 Tm (#5->6) Tj ET -0.45 w 0.9020 0.7110 0.6660 rg 1.0000 1.0000 1.0000 RG 60.00 37.20 44.00 10.40 re B -BT /F2 5.10 Tf 0.0800 0.0900 0.1000 rg 1 0 0 1 75.04 40.50 Tm (#6->9) Tj ET -0.45 w 0.9540 0.8570 0.8220 rg 1.0000 1.0000 1.0000 RG 60.00 26.80 44.00 10.40 re B -BT /F2 5.10 Tf 0.0800 0.0900 0.1000 rg 1 0 0 1 75.04 30.10 Tm (#3->4) Tj ET -0.45 w 0.8290 0.9250 0.9010 rg 1.0000 1.0000 1.0000 RG 60.00 16.40 44.00 10.40 re B -BT /F2 5.10 Tf 0.0800 0.0900 0.1000 rg 1 0 0 1 75.04 19.70 Tm (#8->7) Tj ET -0.45 w 0.9400 0.9500 0.9500 rg 1.0000 1.0000 1.0000 RG 60.00 6.00 44.00 10.40 re B -BT /F2 5.10 Tf 0.0800 0.0900 0.1000 rg 1 0 0 1 75.04 9.30 Tm (#1->1) Tj ET -0.25 w 0.8000 0.8100 0.8200 RG 60.00 6.00 m 60.00 120.40 l S -BT /F2 5.00 Tf 0.1200 0.1300 0.1400 rg 1 0 0 1 116.63 128.90 Tm (Tol->Un) Tj ET -0.45 w 0.8290 0.9250 0.9010 rg 1.0000 1.0000 1.0000 RG 104.00 110.00 44.00 10.40 re B -BT /F2 5.10 Tf 0.0800 0.0900 0.1000 rg 1 0 0 1 119.04 113.30 Tm (#2->1) Tj ET -0.45 w 0.9400 0.9500 0.9500 rg 1.0000 1.0000 1.0000 RG 104.00 99.60 44.00 10.40 re B -BT /F2 5.10 Tf 0.0800 0.0900 0.1000 rg 1 0 0 1 119.04 102.90 Tm (#8->8) Tj ET -0.45 w 0.8290 0.9250 0.9010 rg 1.0000 1.0000 1.0000 RG 104.00 89.20 44.00 10.40 re B -BT /F2 5.10 Tf 0.0800 0.0900 0.1000 rg 1 0 0 1 116.25 92.50 Tm (#11->10) Tj ET -0.45 w 0.8290 0.9250 0.9010 rg 1.0000 1.0000 1.0000 RG 104.00 78.80 44.00 10.40 re B -BT /F2 5.10 Tf 0.0800 0.0900 0.1000 rg 1 0 0 1 119.04 82.10 Tm (#3->2) Tj ET -0.45 w 0.8290 0.9250 0.9010 rg 1.0000 1.0000 1.0000 RG 104.00 68.40 44.00 10.40 re B -BT /F2 5.10 Tf 0.0800 0.0900 0.1000 rg 1 0 0 1 117.65 71.70 Tm (#10->9) Tj ET -0.45 w 0.7480 0.8800 0.8520 rg 1.0000 1.0000 1.0000 RG 104.00 58.00 44.00 10.40 re B -BT /F2 5.10 Tf 0.0800 0.0900 0.1000 rg 1 0 0 1 119.04 61.30 Tm (#5->3) Tj ET -0.45 w 0.7480 0.8800 0.8520 rg 1.0000 1.0000 1.0000 RG 104.00 47.60 44.00 10.40 re B -BT /F2 5.10 Tf 0.0800 0.0900 0.1000 rg 1 0 0 1 119.04 50.90 Tm (#6->4) Tj ET -0.45 w 0.9280 0.7840 0.7440 rg 1.0000 1.0000 1.0000 RG 104.00 37.20 44.00 10.40 re B -BT /F2 5.10 Tf 0.0800 0.0900 0.1000 rg 1 0 0 1 117.65 40.50 Tm (#9->11) Tj ET -0.45 w 0.9540 0.8570 0.8220 rg 1.0000 1.0000 1.0000 RG 104.00 26.80 44.00 10.40 re B -BT /F2 5.10 Tf 0.0800 0.0900 0.1000 rg 1 0 0 1 119.04 30.10 Tm (#4->5) Tj ET -0.45 w 0.9400 0.9500 0.9500 rg 1.0000 1.0000 1.0000 RG 104.00 16.40 44.00 10.40 re B -BT /F2 5.10 Tf 0.0800 0.0900 0.1000 rg 1 0 0 1 119.04 19.70 Tm (#7->7) Tj ET -0.45 w 0.8500 0.5650 0.5100 rg 1.0000 1.0000 1.0000 RG 104.00 6.00 44.00 10.40 re B -BT /F2 5.10 Tf 0.0800 0.0900 0.1000 rg 1 0 0 1 119.04 9.30 Tm (#1->6) Tj ET -0.25 w 0.8000 0.8100 0.8200 RG 104.00 6.00 m 104.00 120.40 l S -BT /F2 5.00 Tf 0.1200 0.1300 0.1400 rg 1 0 0 1 169.28 128.90 Tm (Ex->Sp) Tj ET -0.45 w 0.9400 0.9500 0.9500 rg 1.0000 1.0000 1.0000 RG 156.00 110.00 44.00 10.40 re B -BT /F2 5.10 Tf 0.0800 0.0900 0.1000 rg 1 0 0 1 171.04 113.30 Tm (#1->1) Tj ET -0.45 w 0.9400 0.9500 0.9500 rg 1.0000 1.0000 1.0000 RG 156.00 99.60 44.00 10.40 re B -BT /F2 5.10 Tf 0.0800 0.0900 0.1000 rg 1 0 0 1 171.04 102.90 Tm (#4->4) Tj ET -0.45 w 0.9400 0.9500 0.9500 rg 1.0000 1.0000 1.0000 RG 156.00 89.20 44.00 10.40 re B -BT /F2 5.10 Tf 0.0800 0.0900 0.1000 rg 1 0 0 1 171.04 92.50 Tm (#2->2) Tj ET -0.45 w 0.9400 0.9500 0.9500 rg 1.0000 1.0000 1.0000 RG 156.00 78.80 44.00 10.40 re B -BT /F2 5.10 Tf 0.0800 0.0900 0.1000 rg 1 0 0 1 171.04 82.10 Tm (#3->3) Tj ET -0.45 w 0.9400 0.9500 0.9500 rg 1.0000 1.0000 1.0000 RG 156.00 68.40 44.00 10.40 re B -BT /F2 5.10 Tf 0.0800 0.0900 0.1000 rg 1 0 0 1 171.04 71.70 Tm (#6->6) Tj ET -0.45 w 0.9400 0.9500 0.9500 rg 1.0000 1.0000 1.0000 RG 156.00 58.00 44.00 10.40 re B -BT /F2 5.10 Tf 0.0800 0.0900 0.1000 rg 1 0 0 1 171.04 61.30 Tm (#9->9) Tj ET -0.45 w 0.9400 0.9500 0.9500 rg 1.0000 1.0000 1.0000 RG 156.00 47.60 44.00 10.40 re B -BT /F2 5.10 Tf 0.0800 0.0900 0.1000 rg 1 0 0 1 168.25 50.90 Tm (#11->11) Tj ET -0.45 w 0.9400 0.9500 0.9500 rg 1.0000 1.0000 1.0000 RG 156.00 37.20 44.00 10.40 re B -BT /F2 5.10 Tf 0.0800 0.0900 0.1000 rg 1 0 0 1 168.25 40.50 Tm (#10->10) Tj ET -0.45 w 0.9400 0.9500 0.9500 rg 1.0000 1.0000 1.0000 RG 156.00 26.80 44.00 10.40 re B -BT /F2 5.10 Tf 0.0800 0.0900 0.1000 rg 1 0 0 1 171.04 30.10 Tm (#5->5) Tj ET -0.45 w 0.9540 0.8570 0.8220 rg 1.0000 1.0000 1.0000 RG 156.00 16.40 44.00 10.40 re B -BT /F2 5.10 Tf 0.0800 0.0900 0.1000 rg 1 0 0 1 171.04 19.70 Tm (#7->8) Tj ET -0.45 w 0.8290 0.9250 0.9010 rg 1.0000 1.0000 1.0000 RG 156.00 6.00 44.00 10.40 re B -BT /F2 5.10 Tf 0.0800 0.0900 0.1000 rg 1 0 0 1 171.04 9.30 Tm (#8->7) Tj ET -0.25 w 0.8000 0.8100 0.8200 RG 156.00 6.00 m 156.00 120.40 l S -0.55 w 0.4200 0.4300 0.4400 RG 0.00 120.40 m 205.00 120.40 l S -0.55 w 0.4200 0.4300 0.4400 RG 60.00 6.00 m 200.00 6.00 l S -endstream -endobj -xref -0 7 -0000000000 65535 f -0000000015 00000 n -0000000064 00000 n -0000000121 00000 n -0000000257 00000 n -0000000327 00000 n -0000000402 00000 n -trailer -<< /Size 7 /Root 1 0 R >> -startxref -7947 -%%EOF diff --git a/paper_outputs/figures/fig_rank_heatmap1.pdf b/paper_outputs/figures/fig_rank_heatmap1.pdf deleted file mode 100644 index 59b1ad77c4c2a2cda7a1d8922a775e02cb7671df..0000000000000000000000000000000000000000 Binary files a/paper_outputs/figures/fig_rank_heatmap1.pdf and /dev/null differ diff --git a/paper_outputs/figures/fig_task_rank_map.pdf b/paper_outputs/figures/fig_task_rank_map.pdf deleted file mode 100644 index 4ae1bcef71271945bba47c8c34a53751155e9db9..0000000000000000000000000000000000000000 --- a/paper_outputs/figures/fig_task_rank_map.pdf +++ /dev/null @@ -1,348 +0,0 @@ -%PDF-1.4 -%âãÏÓ -1 0 obj -<< /Type /Catalog /Pages 2 0 R >> -endobj -2 0 obj -<< /Type /Pages /Kids [3 0 R] /Count 1 >> -endobj -3 0 obj -<< /Type /Page /Parent 2 0 R /MediaBox [0 0 1120 430] /Resources << /Font << /F1 4 0 R /F2 5 0 R >> >> /Contents 6 0 R >> -endobj -4 0 obj -<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >> -endobj -5 0 obj -<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica-Bold >> -endobj -6 0 obj -<< /Length 22799 >> -stream -1.0000 1.0000 1.0000 rg 0.00 0.00 1120.00 430.00 re f -BT /F2 8.70 Tf 0.1200 0.1400 0.1600 rg 1 0 0 1 117.66 376.00 Tm (FireWx-FM ref.) Tj ET -BT /F2 8.70 Tf 0.1200 0.1400 0.1600 rg 1 0 0 1 211.06 376.00 Tm (Prithvi-WxC) Tj ET -BT /F2 8.70 Tf 0.1200 0.1400 0.1600 rg 1 0 0 1 308.29 376.00 Tm (Aurora) Tj ET -BT /F2 8.70 Tf 0.1200 0.1400 0.1600 rg 1 0 0 1 394.93 376.00 Tm (ClimaX) Tj ET -BT /F2 8.70 Tf 0.1200 0.1400 0.1600 rg 1 0 0 1 471.34 376.00 Tm (StormCast) Tj ET -BT /F2 8.70 Tf 0.1200 0.1400 0.1600 rg 1 0 0 1 568.76 376.00 Tm (DLWP) Tj ET -BT /F2 8.70 Tf 0.1200 0.1400 0.1600 rg 1 0 0 1 658.50 376.00 Tm (FCN) Tj ET -BT /F2 8.70 Tf 0.1200 0.1400 0.1600 rg 1 0 0 1 736.92 376.00 Tm (FengWu) Tj ET -BT /F2 8.70 Tf 0.1200 0.1400 0.1600 rg 1 0 0 1 829.82 376.00 Tm (FuXi) Tj ET -BT /F2 8.70 Tf 0.1200 0.1400 0.1600 rg 1 0 0 1 892.30 376.00 Tm (Pangu-Weather) Tj ET -BT /F2 8.70 Tf 0.1200 0.1400 0.1600 rg 1 0 0 1 987.57 376.00 Tm (AlphaEarth) Tj ET -BT /F2 7.70 Tf 0.1200 0.1400 0.1600 rg 1 0 0 1 12.00 325.00 Tm (Occupancy) Tj ET -BT /F2 7.10 Tf 0.1200 0.1400 0.1600 rg 1 0 0 1 12.00 314.00 Tm (Union F1 \(%\)) Tj ET -BT /F1 6.40 Tf 0.4200 0.4400 0.4600 rg 1 0 0 1 12.00 303.00 Tm (higher better) Tj ET -0.80 w 0.1500 0.4760 0.4860 rg 1.0000 1.0000 1.0000 RG 108.00 300.00 86.00 42.00 re B -BT /F2 11.20 Tf 1.0000 1.0000 1.0000 rg 1 0 0 1 144.88 324.00 Tm (#2) Tj ET -BT /F1 7.00 Tf 1.0000 1.0000 1.0000 rg 1 0 0 1 142.84 309.00 Tm (59.07) Tj ET -0.80 w 0.9300 0.9500 0.9400 rg 1.0000 1.0000 1.0000 RG 194.00 300.00 86.00 42.00 re B -BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 227.83 324.00 Tm (#11) Tj ET -BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 228.84 309.00 Tm (20.19) Tj ET -0.80 w 0.7780 0.8820 0.8640 rg 1.0000 1.0000 1.0000 RG 280.00 300.00 86.00 42.00 re B -BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 316.88 324.00 Tm (#9) Tj ET -BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 314.85 309.00 Tm (23.10) Tj ET -0.80 w 0.0500 0.4000 0.4200 rg 1.0000 1.0000 1.0000 RG 366.00 300.00 86.00 42.00 re B -BT /F2 11.20 Tf 1.0000 1.0000 1.0000 rg 1 0 0 1 402.88 324.00 Tm (#1) Tj ET -BT /F1 7.00 Tf 1.0000 1.0000 1.0000 rg 1 0 0 1 400.85 309.00 Tm (60.15) Tj ET -0.80 w 0.8540 0.9160 0.9020 rg 1.0000 1.0000 1.0000 RG 452.00 300.00 86.00 42.00 re B -BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 485.83 324.00 Tm (#10) Tj ET -BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 486.85 309.00 Tm (22.38) Tj ET -0.80 w 0.6260 0.8140 0.7880 rg 1.0000 1.0000 1.0000 RG 538.00 300.00 86.00 42.00 re B -BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 574.88 324.00 Tm (#7) Tj ET -BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 572.85 309.00 Tm (28.19) Tj ET -0.80 w 0.2500 0.5520 0.5520 rg 1.0000 1.0000 1.0000 RG 624.00 300.00 86.00 42.00 re B -BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 660.88 324.00 Tm (#3) Tj ET -BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 658.85 309.00 Tm (40.06) Tj ET -0.80 w 0.7020 0.8480 0.8260 rg 1.0000 1.0000 1.0000 RG 710.00 300.00 86.00 42.00 re B -BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 746.88 324.00 Tm (#8) Tj ET -BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 744.85 309.00 Tm (24.10) Tj ET -0.80 w 0.4500 0.7040 0.6840 rg 1.0000 1.0000 1.0000 RG 796.00 300.00 86.00 42.00 re B -BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 832.88 324.00 Tm (#5) Tj ET -BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 830.85 309.00 Tm (37.29) Tj ET -0.80 w 0.5500 0.7800 0.7500 rg 1.0000 1.0000 1.0000 RG 882.00 300.00 86.00 42.00 re B -BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 918.88 324.00 Tm (#6) Tj ET -BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 916.85 309.00 Tm (35.64) Tj ET -0.80 w 0.3500 0.6280 0.6180 rg 1.0000 1.0000 1.0000 RG 968.00 300.00 86.00 42.00 re B -BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 1004.88 324.00 Tm (#4) Tj ET -BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 1002.85 309.00 Tm (37.43) Tj ET -BT /F2 7.70 Tf 0.1200 0.1400 0.1600 rg 1 0 0 1 12.00 283.00 Tm (Fire spread) Tj ET -BT /F2 7.10 Tf 0.1200 0.1400 0.1600 rg 1 0 0 1 12.00 272.00 Tm (AP \(%\)) Tj ET -BT /F1 6.40 Tf 0.4200 0.4400 0.4600 rg 1 0 0 1 12.00 261.00 Tm (higher better) Tj ET -0.80 w 0.0500 0.4000 0.4200 rg 1.0000 1.0000 1.0000 RG 108.00 258.00 86.00 42.00 re B -BT /F2 11.20 Tf 1.0000 1.0000 1.0000 rg 1 0 0 1 144.88 282.00 Tm (#1) Tj ET -BT /F1 7.00 Tf 1.0000 1.0000 1.0000 rg 1 0 0 1 142.84 267.00 Tm (30.09) Tj ET -0.80 w 0.7780 0.8820 0.8640 rg 1.0000 1.0000 1.0000 RG 194.00 258.00 86.00 42.00 re B -BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 230.88 282.00 Tm (#9) Tj ET -BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 230.66 267.00 Tm (5.00) Tj ET -0.80 w 0.1500 0.4760 0.4860 rg 1.0000 1.0000 1.0000 RG 280.00 258.00 86.00 42.00 re B -BT /F2 11.20 Tf 1.0000 1.0000 1.0000 rg 1 0 0 1 316.88 282.00 Tm (#2) Tj ET -BT /F1 7.00 Tf 1.0000 1.0000 1.0000 rg 1 0 0 1 314.85 267.00 Tm (16.62) Tj ET -0.80 w 0.6260 0.8140 0.7880 rg 1.0000 1.0000 1.0000 RG 366.00 258.00 86.00 42.00 re B -BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 402.88 282.00 Tm (#7) Tj ET -BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 400.85 267.00 Tm (11.17) Tj ET -0.80 w 0.8540 0.9160 0.9020 rg 1.0000 1.0000 1.0000 RG 452.00 258.00 86.00 42.00 re B -BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 485.83 282.00 Tm (#10) Tj ET -BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 488.67 267.00 Tm (2.81) Tj ET -0.80 w 0.7020 0.8480 0.8260 rg 1.0000 1.0000 1.0000 RG 538.00 258.00 86.00 42.00 re B -BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 574.88 282.00 Tm (#8) Tj ET -BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 574.66 267.00 Tm (5.94) Tj ET -0.80 w 0.9300 0.9500 0.9400 rg 1.0000 1.0000 1.0000 RG 624.00 258.00 86.00 42.00 re B -BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 657.83 282.00 Tm (#11) Tj ET -BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 660.66 267.00 Tm (2.39) Tj ET -0.80 w 0.3500 0.6280 0.6180 rg 1.0000 1.0000 1.0000 RG 710.00 258.00 86.00 42.00 re B -BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 746.88 282.00 Tm (#4) Tj ET -BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 744.85 267.00 Tm (13.17) Tj ET -0.80 w 0.2500 0.5520 0.5520 rg 1.0000 1.0000 1.0000 RG 796.00 258.00 86.00 42.00 re B -BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 832.88 282.00 Tm (#3) Tj ET -BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 830.85 267.00 Tm (14.35) Tj ET -0.80 w 0.4500 0.7040 0.6840 rg 1.0000 1.0000 1.0000 RG 882.00 258.00 86.00 42.00 re B -BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 918.88 282.00 Tm (#5) Tj ET -BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 916.85 267.00 Tm (12.69) Tj ET -0.80 w 0.5500 0.7800 0.7500 rg 1.0000 1.0000 1.0000 RG 968.00 258.00 86.00 42.00 re B -BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 1004.88 282.00 Tm (#6) Tj ET -BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 1002.85 267.00 Tm (11.83) Tj ET -BT /F2 7.70 Tf 0.1200 0.1400 0.1600 rg 1 0 0 1 12.00 241.00 Tm (Burned area) Tj ET -BT /F2 7.10 Tf 0.1200 0.1400 0.1600 rg 1 0 0 1 12.00 230.00 Tm (log-RMSE) Tj ET -BT /F1 6.40 Tf 0.4200 0.4400 0.4600 rg 1 0 0 1 12.00 219.00 Tm (lower better) Tj ET -0.80 w 0.0500 0.4000 0.4200 rg 1.0000 1.0000 1.0000 RG 108.00 216.00 86.00 42.00 re B -BT /F2 11.20 Tf 1.0000 1.0000 1.0000 rg 1 0 0 1 144.88 240.00 Tm (#1) Tj ET -BT /F1 7.00 Tf 1.0000 1.0000 1.0000 rg 1 0 0 1 144.66 225.00 Tm (1.17) Tj ET -0.80 w 0.3500 0.6280 0.6180 rg 1.0000 1.0000 1.0000 RG 194.00 216.00 86.00 42.00 re B -BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 230.88 240.00 Tm (#4) Tj ET -BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 230.66 225.00 Tm (1.36) Tj ET -0.80 w 0.7780 0.8820 0.8640 rg 1.0000 1.0000 1.0000 RG 280.00 216.00 86.00 42.00 re B -BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 316.88 240.00 Tm (#9) Tj ET -BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 316.67 225.00 Tm (1.87) Tj ET -0.80 w 0.8540 0.9160 0.9020 rg 1.0000 1.0000 1.0000 RG 366.00 216.00 86.00 42.00 re B -BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 399.83 240.00 Tm (#10) Tj ET -BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 402.67 225.00 Tm (2.03) Tj ET -0.80 w 0.7020 0.8480 0.8260 rg 1.0000 1.0000 1.0000 RG 452.00 216.00 86.00 42.00 re B -BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 488.88 240.00 Tm (#8) Tj ET -BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 488.67 225.00 Tm (1.67) Tj ET -0.80 w 0.1500 0.4760 0.4860 rg 1.0000 1.0000 1.0000 RG 538.00 216.00 86.00 42.00 re B -BT /F2 11.20 Tf 1.0000 1.0000 1.0000 rg 1 0 0 1 574.88 240.00 Tm (#2) Tj ET -BT /F1 7.00 Tf 1.0000 1.0000 1.0000 rg 1 0 0 1 574.66 225.00 Tm (1.31) Tj ET -0.80 w 0.4500 0.7040 0.6840 rg 1.0000 1.0000 1.0000 RG 624.00 216.00 86.00 42.00 re B -BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 660.88 240.00 Tm (#5) Tj ET -BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 660.66 225.00 Tm (1.37) Tj ET -0.80 w 0.5500 0.7800 0.7500 rg 1.0000 1.0000 1.0000 RG 710.00 216.00 86.00 42.00 re B -BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 746.88 240.00 Tm (#6) Tj ET -BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 746.66 225.00 Tm (1.37) Tj ET -0.80 w 0.6260 0.8140 0.7880 rg 1.0000 1.0000 1.0000 RG 796.00 216.00 86.00 42.00 re B -BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 832.88 240.00 Tm (#7) Tj ET -BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 832.66 225.00 Tm (1.41) Tj ET -0.80 w 0.2500 0.5520 0.5520 rg 1.0000 1.0000 1.0000 RG 882.00 216.00 86.00 42.00 re B -BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 918.88 240.00 Tm (#3) Tj ET -BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 918.66 225.00 Tm (1.33) Tj ET -0.80 w 0.9300 0.9500 0.9400 rg 1.0000 1.0000 1.0000 RG 968.00 216.00 86.00 42.00 re B -BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 1001.83 240.00 Tm (#11) Tj ET -BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 1004.66 225.00 Tm (2.41) Tj ET -BT /F2 7.70 Tf 0.1200 0.1400 0.1600 rg 1 0 0 1 12.00 199.00 Tm (Analog retrieval) Tj ET -BT /F2 7.10 Tf 0.1200 0.1400 0.1600 rg 1 0 0 1 12.00 188.00 Tm (nDCG@10) Tj ET -BT /F1 6.40 Tf 0.4200 0.4400 0.4600 rg 1 0 0 1 12.00 177.00 Tm (higher better) Tj ET -0.80 w 0.0500 0.4000 0.4200 rg 1.0000 1.0000 1.0000 RG 108.00 174.00 86.00 42.00 re B -BT /F2 11.20 Tf 1.0000 1.0000 1.0000 rg 1 0 0 1 144.88 198.00 Tm (#1) Tj ET -BT /F1 7.00 Tf 1.0000 1.0000 1.0000 rg 1 0 0 1 142.84 183.00 Tm (0.510) Tj ET -0.80 w 0.9300 0.9500 0.9400 rg 1.0000 1.0000 1.0000 RG 194.00 174.00 86.00 42.00 re B -BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 227.83 198.00 Tm (#11) Tj ET -BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 228.84 183.00 Tm (0.386) Tj ET -0.80 w 0.7020 0.8480 0.8260 rg 1.0000 1.0000 1.0000 RG 280.00 174.00 86.00 42.00 re B -BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 316.88 198.00 Tm (#8) Tj ET -BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 314.85 183.00 Tm (0.405) Tj ET -0.80 w 0.5500 0.7800 0.7500 rg 1.0000 1.0000 1.0000 RG 366.00 174.00 86.00 42.00 re B -BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 402.88 198.00 Tm (#6) Tj ET -BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 400.85 183.00 Tm (0.414) Tj ET -0.80 w 0.6260 0.8140 0.7880 rg 1.0000 1.0000 1.0000 RG 452.00 174.00 86.00 42.00 re B -BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 488.88 198.00 Tm (#7) Tj ET -BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 486.85 183.00 Tm (0.408) Tj ET -0.80 w 0.8540 0.9160 0.9020 rg 1.0000 1.0000 1.0000 RG 538.00 174.00 86.00 42.00 re B -BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 571.83 198.00 Tm (#10) Tj ET -BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 572.85 183.00 Tm (0.397) Tj ET -0.80 w 0.2500 0.5520 0.5520 rg 1.0000 1.0000 1.0000 RG 624.00 174.00 86.00 42.00 re B -BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 660.88 198.00 Tm (#3) Tj ET -BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 658.85 183.00 Tm (0.432) Tj ET -0.80 w 0.4500 0.7040 0.6840 rg 1.0000 1.0000 1.0000 RG 710.00 174.00 86.00 42.00 re B -BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 746.88 198.00 Tm (#5) Tj ET -BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 744.85 183.00 Tm (0.425) Tj ET -0.80 w 0.3500 0.6280 0.6180 rg 1.0000 1.0000 1.0000 RG 796.00 174.00 86.00 42.00 re B -BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 832.88 198.00 Tm (#4) Tj ET -BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 830.85 183.00 Tm (0.428) Tj ET -0.80 w 0.7780 0.8820 0.8640 rg 1.0000 1.0000 1.0000 RG 882.00 174.00 86.00 42.00 re B -BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 918.88 198.00 Tm (#9) Tj ET -BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 916.85 183.00 Tm (0.402) Tj ET -0.80 w 0.1500 0.4760 0.4860 rg 1.0000 1.0000 1.0000 RG 968.00 174.00 86.00 42.00 re B -BT /F2 11.20 Tf 1.0000 1.0000 1.0000 rg 1 0 0 1 1004.88 198.00 Tm (#2) Tj ET -BT /F1 7.00 Tf 1.0000 1.0000 1.0000 rg 1 0 0 1 1002.85 183.00 Tm (0.509) Tj ET -BT /F2 7.70 Tf 0.1200 0.1400 0.1600 rg 1 0 0 1 12.00 157.00 Tm (Smoke PM2.5) Tj ET -BT /F2 7.10 Tf 0.1200 0.1400 0.1600 rg 1 0 0 1 12.00 146.00 Tm (RMSE) Tj ET -BT /F1 6.40 Tf 0.4200 0.4400 0.4600 rg 1 0 0 1 12.00 135.00 Tm (lower better) Tj ET -0.80 w 0.1500 0.4760 0.4860 rg 1.0000 1.0000 1.0000 RG 108.00 132.00 86.00 42.00 re B -BT /F2 11.20 Tf 1.0000 1.0000 1.0000 rg 1 0 0 1 144.88 156.00 Tm (#2) Tj ET -BT /F1 7.00 Tf 1.0000 1.0000 1.0000 rg 1 0 0 1 144.66 141.00 Tm (4.46) Tj ET -0.80 w 0.7020 0.8480 0.8260 rg 1.0000 1.0000 1.0000 RG 194.00 132.00 86.00 42.00 re B -BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 230.88 156.00 Tm (#8) Tj ET -BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 230.66 141.00 Tm (6.04) Tj ET -0.80 w 0.7780 0.8820 0.8640 rg 1.0000 1.0000 1.0000 RG 280.00 132.00 86.00 42.00 re B -BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 316.88 156.00 Tm (#9) Tj ET -BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 316.67 141.00 Tm (6.04) Tj ET -0.80 w 0.8540 0.9160 0.9020 rg 1.0000 1.0000 1.0000 RG 366.00 132.00 86.00 42.00 re B -BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 399.83 156.00 Tm (#10) Tj ET -BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 402.67 141.00 Tm (6.04) Tj ET -0.80 w 0.9300 0.9500 0.9400 rg 1.0000 1.0000 1.0000 RG 452.00 132.00 86.00 42.00 re B -BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 485.83 156.00 Tm (#11) Tj ET -BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 488.67 141.00 Tm (6.12) Tj ET -0.80 w 0.4500 0.7040 0.6840 rg 1.0000 1.0000 1.0000 RG 538.00 132.00 86.00 42.00 re B -BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 574.88 156.00 Tm (#5) Tj ET -BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 574.66 141.00 Tm (5.93) Tj ET -0.80 w 0.3500 0.6280 0.6180 rg 1.0000 1.0000 1.0000 RG 624.00 132.00 86.00 42.00 re B -BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 660.88 156.00 Tm (#4) Tj ET -BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 660.66 141.00 Tm (5.93) Tj ET -0.80 w 0.5500 0.7800 0.7500 rg 1.0000 1.0000 1.0000 RG 710.00 132.00 86.00 42.00 re B -BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 746.88 156.00 Tm (#6) Tj ET -BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 746.66 141.00 Tm (5.93) Tj ET -0.80 w 0.6260 0.8140 0.7880 rg 1.0000 1.0000 1.0000 RG 796.00 132.00 86.00 42.00 re B -BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 832.88 156.00 Tm (#7) Tj ET -BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 832.66 141.00 Tm (5.93) Tj ET -0.80 w 0.2500 0.5520 0.5520 rg 1.0000 1.0000 1.0000 RG 882.00 132.00 86.00 42.00 re B -BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 918.88 156.00 Tm (#3) Tj ET -BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 918.66 141.00 Tm (5.93) Tj ET -0.80 w 0.0500 0.4000 0.4200 rg 1.0000 1.0000 1.0000 RG 968.00 132.00 86.00 42.00 re B -BT /F2 11.20 Tf 1.0000 1.0000 1.0000 rg 1 0 0 1 1004.88 156.00 Tm (#1) Tj ET -BT /F1 7.00 Tf 1.0000 1.0000 1.0000 rg 1 0 0 1 1004.66 141.00 Tm (4.44) Tj ET -BT /F2 7.70 Tf 0.1200 0.1400 0.1600 rg 1 0 0 1 12.00 115.00 Tm (Extreme heat) Tj ET -BT /F2 7.10 Tf 0.1200 0.1400 0.1600 rg 1 0 0 1 12.00 104.00 Tm (RMSE-C) Tj ET -BT /F1 6.40 Tf 0.4200 0.4400 0.4600 rg 1 0 0 1 12.00 93.00 Tm (lower better) Tj ET -0.80 w 0.0500 0.4000 0.4200 rg 1.0000 1.0000 1.0000 RG 108.00 90.00 86.00 42.00 re B -BT /F2 11.20 Tf 1.0000 1.0000 1.0000 rg 1 0 0 1 144.88 114.00 Tm (#1) Tj ET -BT /F1 7.00 Tf 1.0000 1.0000 1.0000 rg 1 0 0 1 142.84 99.00 Tm (0.218) Tj ET -0.80 w 0.7780 0.8820 0.8640 rg 1.0000 1.0000 1.0000 RG 194.00 90.00 86.00 42.00 re B -BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 230.88 114.00 Tm (#9) Tj ET -BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 230.66 99.00 Tm (4.62) Tj ET -0.80 w 0.9300 0.9500 0.9400 rg 1.0000 1.0000 1.0000 RG 280.00 90.00 86.00 42.00 re B -BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 313.83 114.00 Tm (#11) Tj ET -BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 314.85 99.00 Tm (18.05) Tj ET -0.80 w 0.8540 0.9160 0.9020 rg 1.0000 1.0000 1.0000 RG 366.00 90.00 86.00 42.00 re B -BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 399.83 114.00 Tm (#10) Tj ET -BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 400.85 99.00 Tm (17.65) Tj ET -0.80 w 0.2500 0.5520 0.5520 rg 1.0000 1.0000 1.0000 RG 452.00 90.00 86.00 42.00 re B -BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 488.88 114.00 Tm (#3) Tj ET -BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 488.67 99.00 Tm (1.77) Tj ET -0.80 w 0.7020 0.8480 0.8260 rg 1.0000 1.0000 1.0000 RG 538.00 90.00 86.00 42.00 re B -BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 574.88 114.00 Tm (#8) Tj ET -BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 574.66 99.00 Tm (2.27) Tj ET -0.80 w 0.5500 0.7800 0.7500 rg 1.0000 1.0000 1.0000 RG 624.00 90.00 86.00 42.00 re B -BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 660.88 114.00 Tm (#6) Tj ET -BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 660.66 99.00 Tm (2.17) Tj ET -0.80 w 0.3500 0.6280 0.6180 rg 1.0000 1.0000 1.0000 RG 710.00 90.00 86.00 42.00 re B -BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 746.88 114.00 Tm (#4) Tj ET -BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 746.66 99.00 Tm (2.13) Tj ET -0.80 w 0.4500 0.7040 0.6840 rg 1.0000 1.0000 1.0000 RG 796.00 90.00 86.00 42.00 re B -BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 832.88 114.00 Tm (#5) Tj ET -BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 832.66 99.00 Tm (2.13) Tj ET -0.80 w 0.6260 0.8140 0.7880 rg 1.0000 1.0000 1.0000 RG 882.00 90.00 86.00 42.00 re B -BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 918.88 114.00 Tm (#7) Tj ET -BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 918.66 99.00 Tm (2.20) Tj ET -0.80 w 0.1500 0.4760 0.4860 rg 1.0000 1.0000 1.0000 RG 968.00 90.00 86.00 42.00 re B -BT /F2 11.20 Tf 1.0000 1.0000 1.0000 rg 1 0 0 1 1004.88 114.00 Tm (#2) Tj ET -BT /F1 7.00 Tf 1.0000 1.0000 1.0000 rg 1 0 0 1 1002.85 99.00 Tm (0.219) Tj ET -0.80 w 0.2000 0.2200 0.2400 RG 108.00 90.00 946.00 252.00 re S -BT /F2 9.00 Tf 0.2400 0.2500 0.2600 rg 1 0 0 1 908.00 63.00 Tm (within-row rank) Tj ET -0.9300 0.9500 0.9400 rg 834.00 46.00 2.60 10.00 re f -0.9300 0.9500 0.9400 rg 836.50 46.00 2.60 10.00 re f -0.9300 0.9500 0.9400 rg 839.00 46.00 2.60 10.00 re f -0.9300 0.9500 0.9400 rg 841.50 46.00 2.60 10.00 re f -0.9300 0.9500 0.9400 rg 844.00 46.00 2.60 10.00 re f -0.9300 0.9500 0.9400 rg 846.50 46.00 2.60 10.00 re f -0.9300 0.9500 0.9400 rg 849.00 46.00 2.60 10.00 re f -0.9300 0.9500 0.9400 rg 851.50 46.00 2.60 10.00 re f -0.8540 0.9160 0.9020 rg 854.00 46.00 2.60 10.00 re f -0.8540 0.9160 0.9020 rg 856.50 46.00 2.60 10.00 re f -0.8540 0.9160 0.9020 rg 859.00 46.00 2.60 10.00 re f -0.8540 0.9160 0.9020 rg 861.50 46.00 2.60 10.00 re f -0.8540 0.9160 0.9020 rg 864.00 46.00 2.60 10.00 re f -0.8540 0.9160 0.9020 rg 866.50 46.00 2.60 10.00 re f -0.8540 0.9160 0.9020 rg 869.00 46.00 2.60 10.00 re f -0.8540 0.9160 0.9020 rg 871.50 46.00 2.60 10.00 re f -0.7780 0.8820 0.8640 rg 874.00 46.00 2.60 10.00 re f -0.7780 0.8820 0.8640 rg 876.50 46.00 2.60 10.00 re f -0.7780 0.8820 0.8640 rg 879.00 46.00 2.60 10.00 re f -0.7780 0.8820 0.8640 rg 881.50 46.00 2.60 10.00 re f -0.7780 0.8820 0.8640 rg 884.00 46.00 2.60 10.00 re f -0.7780 0.8820 0.8640 rg 886.50 46.00 2.60 10.00 re f -0.7780 0.8820 0.8640 rg 889.00 46.00 2.60 10.00 re f -0.7780 0.8820 0.8640 rg 891.50 46.00 2.60 10.00 re f -0.7020 0.8480 0.8260 rg 894.00 46.00 2.60 10.00 re f -0.7020 0.8480 0.8260 rg 896.50 46.00 2.60 10.00 re f -0.7020 0.8480 0.8260 rg 899.00 46.00 2.60 10.00 re f -0.7020 0.8480 0.8260 rg 901.50 46.00 2.60 10.00 re f -0.7020 0.8480 0.8260 rg 904.00 46.00 2.60 10.00 re f -0.7020 0.8480 0.8260 rg 906.50 46.00 2.60 10.00 re f -0.7020 0.8480 0.8260 rg 909.00 46.00 2.60 10.00 re f -0.7020 0.8480 0.8260 rg 911.50 46.00 2.60 10.00 re f -0.6260 0.8140 0.7880 rg 914.00 46.00 2.60 10.00 re f -0.6260 0.8140 0.7880 rg 916.50 46.00 2.60 10.00 re f -0.6260 0.8140 0.7880 rg 919.00 46.00 2.60 10.00 re f -0.6260 0.8140 0.7880 rg 921.50 46.00 2.60 10.00 re f -0.6260 0.8140 0.7880 rg 924.00 46.00 2.60 10.00 re f -0.6260 0.8140 0.7880 rg 926.50 46.00 2.60 10.00 re f -0.6260 0.8140 0.7880 rg 929.00 46.00 2.60 10.00 re f -0.6260 0.8140 0.7880 rg 931.50 46.00 2.60 10.00 re f -0.5500 0.7800 0.7500 rg 934.00 46.00 2.60 10.00 re f -0.5500 0.7800 0.7500 rg 936.50 46.00 2.60 10.00 re f -0.5500 0.7800 0.7500 rg 939.00 46.00 2.60 10.00 re f -0.5500 0.7800 0.7500 rg 941.50 46.00 2.60 10.00 re f -0.5500 0.7800 0.7500 rg 944.00 46.00 2.60 10.00 re f -0.5500 0.7800 0.7500 rg 946.50 46.00 2.60 10.00 re f -0.5500 0.7800 0.7500 rg 949.00 46.00 2.60 10.00 re f -0.5500 0.7800 0.7500 rg 951.50 46.00 2.60 10.00 re f -0.4500 0.7040 0.6840 rg 954.00 46.00 2.60 10.00 re f -0.4500 0.7040 0.6840 rg 956.50 46.00 2.60 10.00 re f -0.4500 0.7040 0.6840 rg 959.00 46.00 2.60 10.00 re f -0.4500 0.7040 0.6840 rg 961.50 46.00 2.60 10.00 re f -0.4500 0.7040 0.6840 rg 964.00 46.00 2.60 10.00 re f -0.4500 0.7040 0.6840 rg 966.50 46.00 2.60 10.00 re f -0.4500 0.7040 0.6840 rg 969.00 46.00 2.60 10.00 re f -0.4500 0.7040 0.6840 rg 971.50 46.00 2.60 10.00 re f -0.3500 0.6280 0.6180 rg 974.00 46.00 2.60 10.00 re f -0.3500 0.6280 0.6180 rg 976.50 46.00 2.60 10.00 re f -0.3500 0.6280 0.6180 rg 979.00 46.00 2.60 10.00 re f -0.3500 0.6280 0.6180 rg 981.50 46.00 2.60 10.00 re f -0.3500 0.6280 0.6180 rg 984.00 46.00 2.60 10.00 re f -0.3500 0.6280 0.6180 rg 986.50 46.00 2.60 10.00 re f -0.3500 0.6280 0.6180 rg 989.00 46.00 2.60 10.00 re f -0.3500 0.6280 0.6180 rg 991.50 46.00 2.60 10.00 re f -0.2500 0.5520 0.5520 rg 994.00 46.00 2.60 10.00 re f -0.2500 0.5520 0.5520 rg 996.50 46.00 2.60 10.00 re f -0.2500 0.5520 0.5520 rg 999.00 46.00 2.60 10.00 re f -0.2500 0.5520 0.5520 rg 1001.50 46.00 2.60 10.00 re f -0.2500 0.5520 0.5520 rg 1004.00 46.00 2.60 10.00 re f -0.2500 0.5520 0.5520 rg 1006.50 46.00 2.60 10.00 re f -0.2500 0.5520 0.5520 rg 1009.00 46.00 2.60 10.00 re f -0.2500 0.5520 0.5520 rg 1011.50 46.00 2.60 10.00 re f -0.1500 0.4760 0.4860 rg 1014.00 46.00 2.60 10.00 re f -0.1500 0.4760 0.4860 rg 1016.50 46.00 2.60 10.00 re f -0.1500 0.4760 0.4860 rg 1019.00 46.00 2.60 10.00 re f -0.1500 0.4760 0.4860 rg 1021.50 46.00 2.60 10.00 re f -0.1500 0.4760 0.4860 rg 1024.00 46.00 2.60 10.00 re f -0.1500 0.4760 0.4860 rg 1026.50 46.00 2.60 10.00 re f -0.1500 0.4760 0.4860 rg 1029.00 46.00 2.60 10.00 re f -0.1500 0.4760 0.4860 rg 1031.50 46.00 2.60 10.00 re f -BT /F1 7.00 Tf 0.2500 0.2600 0.2700 rg 1 0 0 1 834.00 34.00 Tm (rank 11) Tj ET -BT /F1 7.00 Tf 0.2500 0.2600 0.2700 rg 1 0 0 1 1013.84 34.00 Tm (rank 1) Tj ET -endstream -endobj -xref -0 7 -0000000000 65535 f -0000000015 00000 n -0000000064 00000 n -0000000121 00000 n -0000000258 00000 n -0000000328 00000 n -0000000403 00000 n -trailer -<< /Size 7 /Root 1 0 R >> -startxref -23255 -%%EOF diff --git a/release.sha256 b/release.sha256 new file mode 100644 index 0000000000000000000000000000000000000000..73be4ff1a31d9decbed1ff8a7c9f0a6496de5471 --- /dev/null +++ b/release.sha256 @@ -0,0 +1,97 @@ +4e38d2809b88b28a66393b719bf6d7abb83f7601a68676077e4432e2b81192a2 ./.gitattributes +0c1cb3e3f8aab33317290c59d9c22f6a95232fce42696f2cbd1ae8c2e18ce918 ./CITATION.cff +e454135f11698320be1308c4f02c0056695f4be5fe711e585989dd8f3acd6387 ./LICENSE +02c1d42e7f0e07812e468ad065a1e0cbac341ec6865bd8636dc0bc7477203272 ./README.md +0c15a975d94eb79b22fae5d7fcc73e98d2252f71dd8a8e4a9148e1d1addacc3f ./REPRODUCIBILITY.md +6b1486e2f9b9e8d9438aeaa5a0b4c0fe8acca4bf44082e7da423e2c86e36f453 ./artifacts/results/downstream_summary.md +ca93c917b149fa5a14bdf1d9e9e5faf3f352bd68f3cf422dad5d8c5df9e39c89 ./artifacts/results/pixel_context_tune3_summary.md +ad536fce668df5df6dd4d2c842b07da72d343df77f03ec01c7ff5f2313fbdf06 ./artifacts/results/supporting_table4_event_repaired_compsci_20260902_summary_clean.md +7d5bd785dd82b33c80ccdd44dc36f8202438fd01ac6f0a95abbe66867f249605 ./artifacts/results/supporting_table4_smoke_heat_compsci_20260902_summary_clean.md +e75b2535fb54af31be9fd6716cae217c66e71ffe43cc4a22557f992b1e8e026f ./artifacts/results/supporting_table4_smoke_heat_fill_compsci_20260902_summary_clean.md +8a3ac1e56d716917064f7f9520f75beda47389b7cb92ccabb0d2221b5fd93c67 ./artifacts/results/tune4_repaired_eventfm_summary.md +5322effe71b334b6f63b29b53cf409602b6201621f41cde180293d1005137eb3 ./artifacts/results/tune4_throttled_summary.md +bb0a15455e118c73345ba1bcc415b20b2db0fbe1e4e3c0ff8c9c4aee321829de ./data_sources/DATA_SOURCES.md +5b82dd6c279bc8915113b967924732662ac8cdf84309dba37d17493e01a456b5 ./docs/inference_contract.md +f2e356c8c4bdf5d95e7dd772302c3dcf6f3d51bedb332ed101fcaeb3261c3c95 ./docs/model_card.md +bdf86e6c11209d57944ce0e65cb4213dfde63681bca3ab896c01246d8f0c1920 ./docs/reproducibility_manifest.md +507ef63385767cdb275c7f6ace193bb7c4d5743082dc0c5d87f81f36ff5f3d10 ./docs/serving_notes.md +1fd1fbd55ed0f9d3f016f97b50103a82e29da55424b66f64c57b1d7470177bd7 ./docs/split_design.md +bab34810ba7b50e4325170320d2fa1dcf4555d9a8194c65ec06fa31155b278e9 ./examples/final_prediction/README.md +16437e7574c25b895e33a24d4c7514d4a6612e7c4f357ebf060e8ccbb0ae3803 ./examples/final_prediction/firewxfm_conus_20260706_t12_usgs_calibrated_heatmap.png +e97eeef3c58b8ea967677ceda5c4c256c7cacb4581c736debf0ca448854c2e0b ./examples/final_prediction/firewxfm_conus_20260706_t12_usgs_calibrated_heatmap_rgb.tif +9a4a132b00c7e0ae3a15dcf8674b0f734a3669ad89d32d2413053fbb450e47f2 ./examples/final_prediction/firewxfm_conus_20260706_t12_usgs_calibrated_probability_5km_lower48.tif +ada4f8ddc4e9e9c132f7911fe9387da6f77458ae25f1f9b7f6bcaa69957dbb75 ./examples/final_prediction/firewxfm_conus_20260706_t12_usgs_calibrated_summary.json +b1d887fc877e8274c85d78c61f227400ab2fd4cad0ae140a280f8c536cb43ff5 ./examples/final_prediction/firewxfm_conus_20260706_t12_usgs_like_visual_rgba_1000m.tif +4db5e50324d6df89229c3a6c6591a694e677883449f320d4175db3fd4aa5879c ./firewxfm/__init__.py +0e3abf729faf0d679b8204892445a6fe2fac12cf8da3e6a501fdfbc938ca08cf ./firewxfm/modeling_unet.py +661603ea0df9de2bb2840d1527d5dbb633120d8682c436489430f34726599815 ./firewxfm/serve_conus.py +8e2a98e4c3765aa9383df41baeacf0dc4b0837e5621f9f06cfbe73353839790a ./firewxfm/tiled_inference.py +94fbdc49b2d1391bc5cdbe5616a0782c74cd03e6b34b21495f9a8c9526708a14 ./models/README.md +01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b ./models/checkpoints/.gitkeep +8a0818aae5534e3c33fa4100bebbc525b387d3a8e197caa9ce7bf998ef937947 ./models/checkpoints/firewxfm_2024_staticfix_region_balanced_bce_seed42.pt +7a69145f8a7aa411f4a86d4b556a3af52c5fde35cc7437caf5e3d9ee522ea0a5 ./models/checkpoints/firewxfm_conus_lowposw_noexposure_seed42.pt +d08253fe8e7743282af093171eff817bdb5bfe6bc2ef5c370efbf6aac1472aad ./models/metadata/final_model_manifest.json +8a10961abcf91afb452219edb95151f7e25e3a8fac6c6b61c868aabed3c79c01 ./models/metadata/input_channels.json +d7ed23ef515f41b069884a4001b25295db7159e4edef3a89191906d3e6ed31ca ./models/metadata/input_normalization_stats.json +64626bbc36baec1d59e4d36c04be2a6416302219281b7d05697bb4ecdb96f743 ./models/metadata/run_summary.json +ef87d379cd19cd1811a304ceebc2ff5d97189945054de496b85c295569b9aaee ./models/metadata/tile_summary.json +4b5777c523412787b9b498870e54111976870b31b1a8e8c5a5c22a5ce76e502f ./paper/Does_Your_Wildfire_Prediction_Model_ICLR_revision.pdf +f5c980badb6bf21bf4bbe79efaf5de69a755d843d47578f85696d701515ec8dd ./paper/iclr_source/.latexmkrc +4cb969d8304013ed1f84261ae1b4adf78f2ae193a2e3297a5533dd29c96fb78b ./paper/iclr_source/figures/fig_fireprone_contract_progression_compact.pdf +e69261ed4a79dc443705f9b6667c207ef7d6e461669c5628d0014099f9a1376b ./paper/iclr_source/figures/fig_primary_rank_change_map.pdf +865b23a7147ae3c4394ac87ef8f26db2318e765f5e3bddd34bbf356eee69adc7 ./paper/iclr_source/figures/fig_rank_heatmap1.pdf +fabb8b55aac901199cc03773741a26685becffd074f52568c93bee517c2c42c0 ./paper/iclr_source/figures/fig_selection_regret_scatter.pdf +bc4d35ad9cb4c1f9ba8f31c7c340d9684c9dd2d55f5a2e60604a2b58b90cbe40 ./paper/iclr_source/figures/fig_task_contract_tiles.pdf +015ab951b0af5c130e4894092a5dd0bb0fd62e710467163a9df8246d8cf369f4 ./paper/iclr_source/figures/matching.pdf +7dca6ae4a9b179693802f47d24dd66734c0f332b372a7976832a0d429333b755 ./paper/iclr_source/figures/overview_wildfire.pdf +1a1bcaeee67560deac2fcef23720ef7903ad51feb740bf12eee4fc410a8d5053 ./paper/iclr_source/main.tex +d0ed13d09875bccc74b47b228926cdf1e53e8b0115967cf7c7de4a768665bde2 ./paper/iclr_source/neurips_2026.sty +b6dc132b080896ef7d15f326378fb9961ec5871eb87bd7f698cf654bbcf09afa ./paper/iclr_source/references.bib +71c35ae829cc93e37ca51221d68d58482fa542cba0c3926be6b02054284aefee ./paper/iclr_source/sections/0_abstract.tex +386556c64fe55529438b35417d9e5439e398ba0589ebcb377a7b0e528386f94a ./paper/iclr_source/sections/1_intro.tex +cb10a000c24f7fc5dfd292e4fd14d4aaaf454f79136d9519994c8b764d2f3ad4 ./paper/iclr_source/sections/2_backbone.tex +e586df715599ae93ec0d55bf0e783305f4d1a08ca2c7cdb669746c90ca009928 ./paper/iclr_source/sections/3_prelim.tex +8e0b9b40bcfaa1bd12df07175ee10d9f591841410423f42c05abc2f918a03032 ./paper/iclr_source/sections/4_experiments.tex +e55d0a72848ad08ca6190ed856317bb24263492b8b0f8f662a2bfe1e93e0f51e ./paper/iclr_source/sections/5_conclusion.tex +ee38fdf42ea7e55abec54f9316a12186ca87ff4428d91f5a254de65ea2f63cc4 ./paper/iclr_source/sections/appendix.tex +8b7394ad5e5c41318ad6822c497fed5e4544f760a866c085e912a78a1918812e ./requirements.txt +b58db89a3ff76181fba4068c643dc5fc966137225ec5a371a3c43598ae449b84 ./scripts/apply_prediction_mask.py +ddf0044775d15d678935876bff4ac5d358edb186bfa1a9a19a26aa038703e8d0 ./scripts/audit_firewxfm_longitude_seams.py +058838b94b3685269fc3e473f4ec777112100a77bc630cdaf529a628e0c46969 ./scripts/audit_firewxfm_phase0.py +a1c1a282afa419674038e51f625bd5ac01001c4f82164086694b33909ff5817b ./scripts/build_5km_input_stack.py +c4bf68941fad434a2dda8baee7aebd4fb767fcfa7c5e044e6b30a171bca99bf0 ./scripts/check_release_files.py +9b9a2aa1f0402870f6bafb726704197b44e33acad47f5119da0a7684e491a5c3 ./scripts/hrrr_downloader.py +9dcafbbf89863e75b874e52a0afab8d2f16c4899bbfd9e65776f5b30997ed12f ./scripts/make_firewxfm_current_static_corrected_stack.py +cdb0a58098b37a598358d79d6011b820528204bb4d1f47fb69904e8505707ac4 ./scripts/prepare_firms_target_hour_csvs.py +ade6ebccad0b8d729159dfc9d80bc0e3884175936a45287167edc408bdbd74a3 ./scripts/run_firewxfm_baselines.py +05e254c4c706022b6e8a3b263d05a2e9757191d31bab0817d5bc891cbf182990 ./scripts/smoke_inference.py +44249fdf2c47abbf5c9da76b0fa4da0241623090b18c12b787e9b980bcb9e191 ./scripts/write_prediction_tif.py +981ea0db8bab3fc83578a0421abb21b61118d3deb804b014539963c6a7a4193f ./spatial_serving/__init__.py +9b12046bc0c6a44b767404e890e4c2f273c8f5b32d17beeafbdb5c226eac2465 ./spatial_serving/grid_to_polygons.py +e317ae7c928965513130c2e8cbe7196fe446a67811bbc430b8dc11992566d53f ./technical_report/FireWx-FM_technical_report.pdf +af83ff302b763ecd9988f9da17281e61933b9a9b039e584439518bd2c6dd8f31 ./technical_report/source/figure_sources/fig_contract_src.tex +797c6d2e10108b5905261a305e307473c4a46788011f44786e5291719fa06525 ./technical_report/source/figure_sources/fig_matching_src.tex +61e293727166368cd52f491a4e728ece1e6afd978232f0e475d5b6839ecb6d2f ./technical_report/source/figure_sources/fig_overview_src.tex +4b03575224629202b3ae45cf9f59dffc3fef46ca16e6cb8957e3211e54fc5376 ./technical_report/source/figure_sources/make_topk_diagnostics_fig.py +2dc2ada020bbbe21788f4596fdcf3cd8ad189bddca295de17e53249b5729dd7e ./technical_report/source/figures/fig_contract.pdf +5281b4fee6ba7e63072da45bde1a9205f78e54d7d4ebd025641eb734fd96044f ./technical_report/source/figures/fig_matching.pdf +b87736560ee695b41b83e3788ed3f24efa3750e9dfc941503f8bfae963487f9c ./technical_report/source/figures/fig_overview.pdf +706d00338fe3f1c40788e62a6edca9e73152ba6959012e6f5ef702105232bde3 ./technical_report/source/figures/fig_topk_diagnostics.pdf +16437e7574c25b895e33a24d4c7514d4a6612e7c4f357ebf060e8ccbb0ae3803 ./technical_report/source/figures/firewxfm_conus_20260706_t12_usgs_calibrated_heatmap.png +a492762600dd4baec7903739cab715c1ffb66856f0616e4e67d30f79fbdddb40 ./technical_report/source/figures/firewxfm_staticfix_conus_release_heatmap.png +7dca6ae4a9b179693802f47d24dd66734c0f332b372a7976832a0d429333b755 ./technical_report/source/figures/overview_wildfire.pdf +2d5b40b9ce88d6e194e58d8a54f607f1a195c9d7bbe9cec1947f041cf5aa2399 ./technical_report/source/main.tex +11b33d7a4f346e7e824b5c196b22bfe858202f0c06a58e8b17061c33239075bd ./technical_report/source/reference_tech.bib +19b64d206124248b911828b3bb766580a09d3e6fb330a091747f46fe2064b6bf ./training/README.md +3db9344c83af8d810a3de405f22bbe672d65bda22395f650fccf3d3bc408ebab ./training/build_phase1_cache_regional_hrrr.py +33cd0d9f36ad3281a2c8b165c65e2eeecb5864a6d2a31b8e7b3c70bbf22cfea6 ./training/cache_helpers.py +b8f315616031e97ba0a8d9a899ef736cb22d1f19a774d5b072c506fd4193e4eb ./training/configs/baseline_suite_template.json +508c278cdbc7f701e9ae3aa7607e7bcfedacfc38e6e87649f5c329c62d83a43e ./training/configs/eval_region_disjoint_template.json +a5aadd5d2cf357cd86076b1dd7b462376ae32a41782f4578292fda801f8a5586 ./training/configs/stage1_cache_conus_hrrr_us_5km_l12_template.json +abf9c24f1fd2c845dc05acf72e42c9283e355173459baf039266dfacfc1b4d43 ./training/configs/stage1_cache_conus_multiyear_template.json +2b9f3b3fb96df1acf0a5cf2694240dffbfdf0f49c02228808a142479821032cb ./training/configs/train_firewxfm_conus_multiyear_template.json +ef499acc3dad266a4802c2c116a62d86cbfd7bcea5078a5efa6c2522728ca1d8 ./training/configs/train_firewxfm_conus_noexposure_template.json +eae0ff6aa6ef7a977da0fbc51ff40435f3eca26b45b38d362f11a27f8033d876 ./training/configs/train_firewxfm_multitask_pretrain_template.json +095e7fd941b1a2e50c0702b0615101c61f0330f133a2a5cc466ad170cc548a4e ./training/eval_metrics.py +e23bb63d431cbb9ae9d547d1c47b1f6c20efd7228748d5633859f759cc0aa852 ./training/train_cold_tiled_mainline.py +2fa225980cfa34b22bc7467417e71c52ced7b6264c330cb6efb4d1c1bb34d0b7 ./training/train_utils.py diff --git a/requirements.txt b/requirements.txt index 10b87dfa80e125c916f4d87c85d69bebdd1eab03..72836ce2b62180f492ec5b0180e73dbdc7ad793e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,7 @@ -# No runtime dependency is required for paper-output reproduction. +numpy +torch +rasterio +geopandas +pyyaml +pillow +scipy diff --git a/scripts/apply_prediction_mask.py b/scripts/apply_prediction_mask.py new file mode 100644 index 0000000000000000000000000000000000000000..07336d406ee6b075fb432bdcbd0fad50a8fe61b1 --- /dev/null +++ b/scripts/apply_prediction_mask.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +"""Apply a binary mask to a FireWx-FM probability raster stored as NPY.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +import numpy as np + + +def summarize(values: np.ndarray) -> dict[str, object]: + vals = np.asarray(values, dtype=np.float32) + vals = vals[np.isfinite(vals)] + if vals.size == 0: + return {"cells": 0} + return { + "cells": int(vals.size), + "mean": float(vals.mean()), + "q50": float(np.quantile(vals, 0.50)), + "q90": float(np.quantile(vals, 0.90)), + "q99": float(np.quantile(vals, 0.99)), + "max": float(vals.max()), + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--prediction-npy", type=Path, required=True) + parser.add_argument("--mask-npy", type=Path, required=True) + parser.add_argument("--output-npy", type=Path, required=True) + parser.add_argument("--summary-json", type=Path, required=True) + parser.add_argument("--outside-value", type=float, default=0.0) + args = parser.parse_args() + + pred = np.load(args.prediction_npy).astype(np.float32) + mask = np.load(args.mask_npy).astype(bool) + if pred.shape != mask.shape: + raise ValueError(f"prediction {pred.shape} does not match mask {mask.shape}") + + out = pred.copy() + out[~mask] = np.float32(args.outside_value) + args.output_npy.parent.mkdir(parents=True, exist_ok=True) + np.save(args.output_npy, out) + + summary = { + "prediction_npy": str(args.prediction_npy), + "mask_npy": str(args.mask_npy), + "output_npy": str(args.output_npy), + "outside_value": float(args.outside_value), + "prediction_before": summarize(pred), + "prediction_after": summarize(out), + "mask_cells": int(mask.sum()), + "mask_fraction": float(mask.mean()), + } + args.summary_json.parent.mkdir(parents=True, exist_ok=True) + args.summary_json.write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps(summary, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/audit_firewxfm_longitude_seams.py b/scripts/audit_firewxfm_longitude_seams.py new file mode 100644 index 0000000000000000000000000000000000000000..13a0bb2a7797351167bbc6aa6cab45cbb8a7c00c --- /dev/null +++ b/scripts/audit_firewxfm_longitude_seams.py @@ -0,0 +1,256 @@ +#!/usr/bin/env python3 +"""Audit longitude-band discontinuities in a FireWx-FM probability GeoTIFF. + +The audit is intentionally source-product focused. It reads the probability +GeoTIFF before tiling and reports broad longitude-band summaries, adjacent-band +jumps, and optional input-stack channel summaries. This catches central-US seams +that would otherwise be hidden by frontend smoothing. +""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import Any + +import numpy as np +import rasterio +from rasterio.warp import transform as transform_coords + + +CHANNEL_NAMES = [ + "t2m", + "d2m", + "u10", + "v10", + "cape", + "sp", + "blh", + "vis", + "prate", + "tp", + "dynamic_valid", + "static_valid", + "fuel_fbfm40", + "canopy_cover", + "housing_density", + "population", +] + +DEFAULT_BANDS = [ + (-125.0, -115.0), + (-115.0, -110.0), + (-110.0, -105.0), + (-105.0, -100.0), + (-100.0, -95.0), + (-95.0, -90.0), + (-90.0, -85.0), + (-85.0, -75.0), + (-75.0, -66.0), +] + +DEFAULT_CHANNELS = [4, 7, 10, 11, 12, 13] +PROBABILITY_THRESHOLDS = (0.01, 0.03, 0.05, 0.10, 0.15, 0.20) + + +def parse_bands(text: str | None) -> list[tuple[float, float]]: + if not text: + return DEFAULT_BANDS + bands: list[tuple[float, float]] = [] + for raw in text.split(","): + item = raw.strip() + if not item: + continue + lo, hi = item.split(":") + bands.append((float(lo), float(hi))) + return bands + + +def parse_channels(text: str | None) -> list[int]: + if not text: + return DEFAULT_CHANNELS + return [int(v) for v in text.split(",") if v.strip()] + + +def band_label(band: tuple[float, float]) -> str: + lo, hi = band + return f"lon_{lo:g}_to_{hi:g}" + + +def summarize(values: np.ndarray, *, probability: bool = False) -> dict[str, Any]: + vals = np.asarray(values, dtype=np.float32) + vals = vals[np.isfinite(vals)] + if vals.size == 0: + return {"cells": 0} + out: dict[str, Any] = { + "cells": int(vals.size), + "mean": float(vals.mean()), + "std": float(vals.std()), + "min": float(vals.min()), + "q10": float(np.quantile(vals, 0.10)), + "q50": float(np.quantile(vals, 0.50)), + "q90": float(np.quantile(vals, 0.90)), + "q95": float(np.quantile(vals, 0.95)), + "q99": float(np.quantile(vals, 0.99)), + "max": float(vals.max()), + } + if probability: + for threshold in PROBABILITY_THRESHOLDS: + out[f"frac_ge_{threshold:.2f}"] = float((vals >= threshold).mean()) + else: + out["frac_zero"] = float((vals == 0.0).mean()) + out["frac_le_0p25"] = float((vals <= 0.25).mean()) + return out + + +def load_lon_lat(path: Path) -> tuple[np.ndarray, np.ndarray, str]: + with rasterio.open(path) as src: + rows = np.arange(src.height) + cols = np.arange(src.width) + xs = src.transform.c + (cols + 0.5) * src.transform.a + ys = src.transform.f + (rows + 0.5) * src.transform.e + xx, yy = np.meshgrid(xs, ys) + lon_values, lat_values = transform_coords( + src.crs, + "EPSG:4326", + xx.ravel().tolist(), + yy.ravel().tolist(), + ) + lon = np.asarray(lon_values, dtype=np.float32).reshape(src.height, src.width) + lat = np.asarray(lat_values, dtype=np.float32).reshape(src.height, src.width) + return lon, lat, str(src.crs) + + +def read_probability(path: Path) -> np.ndarray: + with rasterio.open(path) as src: + return src.read(1).astype(np.float32) + + +def read_stack(path: Path) -> np.ndarray: + if path.suffix.lower() == ".npy": + stack = np.load(path, mmap_mode="r") + if stack.ndim != 3: + raise ValueError(f"Expected [C,H,W] stack in {path}, got {stack.shape}") + return stack + with rasterio.open(path) as src: + return src.read().astype(np.float32) + + +def adjacent_band_jumps(band_summaries: dict[str, Any]) -> list[dict[str, Any]]: + labels = list(band_summaries) + jumps: list[dict[str, Any]] = [] + for left, right in zip(labels[:-1], labels[1:]): + a = band_summaries[left].get("probability", {}) + b = band_summaries[right].get("probability", {}) + if not a.get("cells") or not b.get("cells"): + continue + jumps.append( + { + "left_band": left, + "right_band": right, + "mean_delta_right_minus_left": float(b["mean"] - a["mean"]), + "q90_delta_right_minus_left": float(b["q90"] - a["q90"]), + "q95_delta_right_minus_left": float(b["q95"] - a["q95"]), + "frac_ge_0p10_delta_right_minus_left": float(b["frac_ge_0.10"] - a["frac_ge_0.10"]), + "mean_ratio_right_over_left": float(b["mean"] / max(a["mean"], 1e-8)), + "q90_ratio_right_over_left": float(b["q90"] / max(a["q90"], 1e-8)), + } + ) + return jumps + + +def evaluate_central_seam(jumps: list[dict[str, Any]], max_abs_mean_delta: float, min_mean_ratio: float) -> dict[str, Any]: + central_pairs = { + ("lon_-110_to_-105", "lon_-105_to_-100"), + ("lon_-105_to_-100", "lon_-100_to_-95"), + } + flagged = [] + for jump in jumps: + pair = (jump["left_band"], jump["right_band"]) + if pair not in central_pairs: + continue + mean_delta = abs(float(jump["mean_delta_right_minus_left"])) + ratio = float(jump["mean_ratio_right_over_left"]) + inverse_ratio = 1.0 / max(ratio, 1e-8) + if mean_delta > max_abs_mean_delta or min(ratio, inverse_ratio) < min_mean_ratio: + flagged.append(jump) + return { + "central_pairs_checked": [list(v) for v in sorted(central_pairs)], + "max_abs_mean_delta": float(max_abs_mean_delta), + "min_mean_ratio": float(min_mean_ratio), + "flagged_pairs": flagged, + "pass": len(flagged) == 0, + "note": "Pass means no large adjacent-band probability jump in the two Rockies/Great-Plains seam pairs.", + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--probability-tif", type=Path, required=True) + parser.add_argument("--lower48-mask-npy", type=Path, required=True) + parser.add_argument("--input-stack", type=Path, help="Optional [16,H,W] NPY or 16-band GeoTIFF input stack.") + parser.add_argument("--bands", help="Comma list of lon_min:lon_max bands. Defaults to CONUS bands.") + parser.add_argument("--channels", help="Comma list of stack channel indices to summarize.") + parser.add_argument("--max-abs-central-mean-delta", type=float, default=0.018) + parser.add_argument("--min-central-mean-ratio", type=float, default=0.55) + parser.add_argument("--output-json", type=Path, required=True) + args = parser.parse_args() + + probability = read_probability(args.probability_tif) + lower48 = np.load(args.lower48_mask_npy).astype(bool) + if probability.shape != lower48.shape: + raise ValueError(f"probability {probability.shape} != mask {lower48.shape}") + lon, lat, crs = load_lon_lat(args.probability_tif) + if lon.shape != probability.shape: + raise ValueError(f"lon/lat grid {lon.shape} != probability {probability.shape}") + + stack = read_stack(args.input_stack) if args.input_stack else None + if stack is not None and tuple(stack.shape[1:]) != tuple(probability.shape): + raise ValueError(f"input stack grid {stack.shape[1:]} != probability {probability.shape}") + channels = parse_channels(args.channels) + bands = parse_bands(args.bands) + + band_summaries: dict[str, Any] = {} + for band in bands: + lo, hi = band + mask = lower48 & (lon >= lo) & (lon < hi) + label = band_label(band) + entry: dict[str, Any] = { + "longitude_range": [float(lo), float(hi)], + "cells": int(mask.sum()), + "probability": summarize(probability[mask], probability=True), + } + if stack is not None: + entry["input_channels"] = {} + for channel in channels: + if not 0 <= channel < stack.shape[0]: + raise ValueError(f"channel {channel} outside stack channel count {stack.shape[0]}") + name = CHANNEL_NAMES[channel] if channel < len(CHANNEL_NAMES) else f"channel_{channel}" + entry["input_channels"][name] = summarize(np.asarray(stack[channel][mask], dtype=np.float32)) + band_summaries[label] = entry + + jumps = adjacent_band_jumps(band_summaries) + result = { + "probability_tif": str(args.probability_tif), + "lower48_mask_npy": str(args.lower48_mask_npy), + "input_stack": str(args.input_stack) if args.input_stack else None, + "crs": crs, + "shape": [int(v) for v in probability.shape], + "lower48_cells": int(lower48.sum()), + "bands": band_summaries, + "adjacent_band_jumps": jumps, + "central_seam_check": evaluate_central_seam( + jumps, + max_abs_mean_delta=float(args.max_abs_central_mean_delta), + min_mean_ratio=float(args.min_central_mean_ratio), + ), + } + args.output_json.parent.mkdir(parents=True, exist_ok=True) + args.output_json.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps({"wrote": str(args.output_json), "central_seam_pass": result["central_seam_check"]["pass"]}, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/audit_firewxfm_phase0.py b/scripts/audit_firewxfm_phase0.py new file mode 100644 index 0000000000000000000000000000000000000000..e6067721bf5632d4fc8ea9b3eae156c38937db2b --- /dev/null +++ b/scripts/audit_firewxfm_phase0.py @@ -0,0 +1,226 @@ +#!/usr/bin/env python3 +"""Phase-0 data and split audit for FireWx-FM experiments. + +The audit is intentionally lightweight: it does not open GRIB, GeoTIFF, or NumPy +payloads. It checks path availability, expected issue/target timestamps, split +assignment, static-layer path configuration, and existing cache manifest +summaries. This makes it safe to run before launching expensive cache builds or +training jobs. +""" + +from __future__ import annotations + +import argparse +import csv +import json +import re +from collections import Counter, defaultdict +from datetime import datetime, timedelta +from pathlib import Path +from typing import Any, Dict, Iterable, List + + +HRRR_RE = re.compile(r"hrrr\.(\d{8})\.t(\d{2})z\.wrfsfcf00\.grib2$") + + +def load_json(path: Path) -> Dict[str, Any]: + return json.loads(path.read_text(encoding="utf-8")) + + +def write_json(path: Path, data: Dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(data, indent=2, sort_keys=True), encoding="utf-8") + + +def parse_day(value: str) -> datetime: + return datetime.strptime(value, "%Y-%m-%d") + + +def iter_days(start: str, end: str) -> Iterable[datetime]: + current = parse_day(start) + last = parse_day(end) + while current <= last: + yield current + current += timedelta(days=1) + + +def list_hrrr_issue_times(roots: Iterable[str]) -> Dict[str, str]: + out: Dict[str, str] = {} + for root_value in roots: + root = Path(root_value) + if not root.exists(): + continue + for path in root.rglob("*.grib2"): + match = HRRR_RE.match(path.name) + if not match: + continue + date_part, hour_part = match.groups() + issue = datetime.strptime(f"{date_part}{hour_part}", "%Y%m%d%H") + out.setdefault(issue.isoformat(), str(path)) + return dict(sorted(out.items())) + + +def split_for_timestamp(ts: datetime, config: Dict[str, Any]) -> str: + if "split_years" in config: + for split, years in config["split_years"].items(): + if ts.year in {int(v) for v in years}: + return str(split) + return "unassigned" + if "split_months" in config: + for split, months in config["split_months"].items(): + if ts.month in {int(v) for v in months}: + return str(split) + return "unassigned" + return "unassigned" + + +def summarize_counter(counter: Counter) -> Dict[str, int]: + return {str(k): int(v) for k, v in sorted(counter.items(), key=lambda item: str(item[0]))} + + +def expected_records(config: Dict[str, Any]) -> List[Dict[str, Any]]: + issue_hours = [int(v) for v in config.get("issue_hours", [0, 6, 12, 18])] + lead_hours = int(config.get("target_offset_hours", 12)) + rows: List[Dict[str, Any]] = [] + for item in config.get("input_ranges", []): + for day in iter_days(str(item["start"]), str(item["end"])): + for hour in issue_hours: + issue = day.replace(hour=hour) + target = issue + timedelta(hours=lead_hours) + rows.append( + { + "issue_timestamp": issue.isoformat(), + "target_timestamp": target.isoformat(), + "issue_year": issue.year, + "issue_month": issue.month, + "issue_hour": issue.hour, + "split": split_for_timestamp(issue, config), + "firms_name": f"{target.strftime('%Y%m%d_%H')}.csv", + } + ) + return rows + + +def audit_expected_availability(config: Dict[str, Any]) -> Dict[str, Any]: + hrrr_index = list_hrrr_issue_times(config.get("hrrr_roots", [])) + firms_dir = Path(str(config.get("firms_dir", ""))) + rows = expected_records(config) + by_split = defaultdict(lambda: Counter()) + by_year = defaultdict(lambda: Counter()) + by_month = defaultdict(lambda: Counter()) + missing_hrrr_examples: List[str] = [] + missing_firms_examples: List[str] = [] + + for row in rows: + issue_key = str(row["issue_timestamp"]) + label_path = firms_dir / str(row["firms_name"]) + hrrr_ok = issue_key in hrrr_index + firms_ok = label_path.exists() + matched = hrrr_ok and firms_ok + for bucket in [by_split[str(row["split"])], by_year[str(row["issue_year"])], by_month[f"{row['issue_year']:04d}-{row['issue_month']:02d}"]]: + bucket["expected"] += 1 + bucket["hrrr_available"] += int(hrrr_ok) + bucket["firms_available"] += int(firms_ok) + bucket["matched"] += int(matched) + if not hrrr_ok and len(missing_hrrr_examples) < 20: + missing_hrrr_examples.append(issue_key) + if not firms_ok and len(missing_firms_examples) < 20: + missing_firms_examples.append(str(label_path)) + + return { + "expected_issue_times": int(len(rows)), + "indexed_hrrr_issue_times": int(len(hrrr_index)), + "firms_dir_exists": firms_dir.exists(), + "by_split": {k: summarize_counter(v) for k, v in sorted(by_split.items())}, + "by_year": {k: summarize_counter(v) for k, v in sorted(by_year.items())}, + "by_month": {k: summarize_counter(v) for k, v in sorted(by_month.items())}, + "missing_hrrr_examples": missing_hrrr_examples, + "missing_firms_examples": missing_firms_examples, + } + + +def audit_static_layers(config: Dict[str, Any]) -> Dict[str, Any]: + out: Dict[str, Any] = {} + for name, spec in config.get("static_layers", {}).items(): + path = Path(str(spec.get("path", ""))) + out[str(name)] = { + "path": str(path), + "exists": path.exists(), + "resampling": str(spec.get("resampling", "")), + } + return out + + +def read_csv_rows(path: Path) -> List[Dict[str, str]]: + with path.open("r", encoding="utf-8", newline="") as handle: + return list(csv.DictReader(handle)) + + +def audit_cache_manifest(config: Dict[str, Any], manifest_arg: str | None) -> Dict[str, Any]: + manifest_path = Path(manifest_arg) if manifest_arg else Path(str(config["cache_root"])) / "manifests" / "phase1_manifest.csv" + if not manifest_path.exists(): + return {"path": str(manifest_path), "exists": False} + rows = read_csv_rows(manifest_path) + split_counts = Counter(row.get("split", "") for row in rows) + positive_cells = 0 + fire_points = 0.0 + samples_missing = 0 + static_missing = 0 + input_times = [row.get("input_timestamp", "") for row in rows if row.get("input_timestamp")] + for row in rows: + try: + positive_cells += int(float(row.get("pos_cells", 0) or 0)) + except ValueError: + pass + try: + fire_points += float(row.get("fire_points", 0.0) or 0.0) + except ValueError: + pass + if row.get("sample_path") and not Path(row["sample_path"]).exists(): + samples_missing += 1 + if row.get("static_path") and not Path(row["static_path"]).exists(): + static_missing += 1 + return { + "path": str(manifest_path), + "exists": True, + "rows": int(len(rows)), + "split_counts": summarize_counter(split_counts), + "first_input_timestamp": min(input_times) if input_times else "", + "last_input_timestamp": max(input_times) if input_times else "", + "total_positive_cells": int(positive_cells), + "total_fire_points": float(fire_points), + "sample_paths_missing": int(samples_missing), + "static_paths_missing": int(static_missing), + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", type=Path, required=True, help="Stage-1 cache config JSON.") + parser.add_argument("--manifest", type=str, default="", help="Optional existing phase1_manifest.csv path.") + parser.add_argument("--output", type=Path, default=Path("artifacts/manifests/phase0_data_split_audit.json")) + args = parser.parse_args() + + config = load_json(args.config) + audit = { + "config_path": str(args.config), + "cache_root": str(config.get("cache_root", "")), + "target_crs": str(config.get("target_crs", "")), + "target_resolution_m": int(config.get("target_resolution_m", 0) or 0), + "target_offset_hours": int(config.get("target_offset_hours", 0) or 0), + "input_ranges": config.get("input_ranges", []), + "issue_hours": [int(v) for v in config.get("issue_hours", [])], + "split_years": config.get("split_years", {}), + "split_months": config.get("split_months", {}), + "hrrr_roots": [{"path": str(path), "exists": Path(str(path)).exists()} for path in config.get("hrrr_roots", [])], + "availability": audit_expected_availability(config), + "static_layers": audit_static_layers(config), + "cache_manifest": audit_cache_manifest(config, args.manifest or None), + } + write_json(args.output, audit) + print(json.dumps({"wrote": str(args.output), "expected_issue_times": audit["availability"]["expected_issue_times"]}, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/audit_release.py b/scripts/audit_release.py deleted file mode 100644 index f3677dcf8fc83c8525f56ad47bc73bd74e5ffe5d..0000000000000000000000000000000000000000 --- a/scripts/audit_release.py +++ /dev/null @@ -1,146 +0,0 @@ -#!/usr/bin/env python3 -"""Audit the public WildFIRE-FM release before upload.""" - -from __future__ import annotations - -import json -import re -from pathlib import Path - - -ROOT = Path(__file__).resolve().parents[1] - -REQUIRED = [ - "README.md", - "LICENSE", - "requirements.txt", - "data_sources/DATA_SOURCES.md", - "models/wildfire_fm/README.md", - "models/wildfire_fm/modeling_unet.py", - "models/wildfire_fm/checkpoint_manifest.json", - "paper/wildfire_fm_evaluation_contracts.pdf", - "paper_outputs/figures/overview_wildfire.pdf", - "paper_outputs/figures/matching.pdf", - "paper_outputs/figures/fig_task_contract_tiles.pdf", - "paper_outputs/figures/fig_primary_rank_change_map.pdf", - "paper_outputs/figures/fig_selection_regret_scatter.pdf", - "paper_outputs/figures/fig_rank_heatmap1.pdf", - "assets/wildfire_fm_model_card.svg", - "assets/release_contents.svg", - "assets/selection_regret_final.png", - "assets/supporting_rank_map_final.png", - "assets/primary_rank_change_final.png", - "artifacts/manifests/paper_outputs.sha256", - "scripts/check_paper_output_hashes.py", -] - -FORBIDDEN_FILE_SUFFIXES = {".tex", ".bib", ".tikz"} -FORBIDDEN_FILE_NAMES = {"manuscript_final.pdf"} - -FORBIDDEN_TEXT = [ - "/home/yx21e", - "/blue/", - "/orange/", - "fsu-compsci", - "TBD", - "N/A", - "Pangu24", -] - -TEXT_SUFFIXES = {".md", ".py", ".sh", ".tex", ".csv", ".json", ".yml", ".yaml", ".txt"} -SKIP_FOR_FORBIDDEN = {"audit_release.py", "build_selection_regret_rq2_figure.py"} - - -def iter_text_files() -> list[Path]: - out: list[Path] = [] - for path in ROOT.rglob("*"): - if ".git" in path.parts or "__pycache__" in path.parts: - continue - if path.name in SKIP_FOR_FORBIDDEN: - continue - if path.is_file() and path.suffix in TEXT_SUFFIXES: - out.append(path) - return sorted(out) - - -def main() -> None: - issues: list[str] = [] - - for rel in REQUIRED: - if not (ROOT / rel).exists(): - issues.append(f"missing required file: {rel}") - for path in ROOT.rglob("*"): - if ".git" in path.parts or "__pycache__" in path.parts: - continue - if path.is_file() and (path.suffix in FORBIDDEN_FILE_SUFFIXES or path.name in FORBIDDEN_FILE_NAMES): - issues.append(f"forbidden manuscript/source artifact present: {path.relative_to(ROOT)}") - - for path in iter_text_files(): - text = path.read_text(errors="ignore") - for token in FORBIDDEN_TEXT: - if token in text: - issues.append(f"{path.relative_to(ROOT)} contains forbidden token {token!r}") - - readme = (ROOT / "README.md").read_text(errors="ignore") - for phrase in ["WildFIRE-FM", "Quick Start", "Data Sources", "Task-Contract Snapshot"]: - if phrase not in readme: - issues.append(f"README missing expected model-card phrase: {phrase}") - - manifest_path = ROOT / "models/wildfire_fm/checkpoint_manifest.json" - if manifest_path.exists(): - data = json.loads(manifest_path.read_text()) - checkpoints = data.get("checkpoints", []) - if len(checkpoints) != 5: - issues.append("checkpoint manifest should list five seeded checkpoints") - for item in checkpoints: - rel = item.get("filename", "") - if not rel.startswith("models/wildfire_fm/checkpoints/seed_"): - issues.append(f"unexpected checkpoint filename in manifest: {rel}") - if "source_path" in item: - issues.append("checkpoint manifest exposes source_path") - if not re.fullmatch(r"[0-9a-f]{64}", str(item.get("sha256", ""))): - issues.append(f"bad sha256 in checkpoint manifest: {item}") - - for path in (ROOT / "paper_outputs/tables").glob("*.tex"): - text = path.read_text(errors="ignore") - if re.search(r"\\ms\{[^}]*\}\{0\.0000\}", text): - issues.append(f"{path.relative_to(ROOT)} displays zero std in an \\ms cell") - - checksum_manifest = ROOT / "artifacts/manifests/paper_outputs.sha256" - if checksum_manifest.exists(): - listed: list[str] = [] - for line in checksum_manifest.read_text(errors="ignore").splitlines(): - if not line.strip(): - continue - parts = line.split(None, 1) - if len(parts) != 2: - issues.append(f"bad checksum manifest line: {line!r}") - continue - rel = parts[1].strip() - listed.append(rel) - if not (ROOT / rel).exists(): - issues.append(f"checksum manifest lists missing output: {rel}") - expected_paths = [] - for rel_root in ["paper", "paper_outputs", "assets"]: - root_dir = ROOT / rel_root - if root_dir.exists(): - expected_paths.extend(str(p.relative_to(ROOT)) for p in root_dir.rglob("*") if p.is_file()) - expected = sorted(set(expected_paths)) - if sorted(listed) != expected: - missing = sorted(set(expected) - set(listed)) - extra = sorted(set(listed) - set(expected)) - if missing: - issues.append(f"checksum manifest missing outputs: {missing}") - if extra: - issues.append(f"checksum manifest has extra outputs: {extra}") - - if issues: - print("Release audit failed:") - for issue in issues: - print(f"- {issue}") - raise SystemExit(1) - print("Release audit passed.") - - -if __name__ == "__main__": - main() diff --git a/scripts/build_5km_input_stack.py b/scripts/build_5km_input_stack.py new file mode 100644 index 0000000000000000000000000000000000000000..109022aded41759f543b7486ac782f365871d95c --- /dev/null +++ b/scripts/build_5km_input_stack.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +"""Build or repair the live 5 km FireWx-FM input stack. + +This is the production-facing entry point for the 16-channel FireWx-FM stack. +It applies the corrected static-layer path used by the staticfix CONUS model: + +1. start from a live 16-channel stack produced by the daily HRRR pipeline, +2. replace channels 11-15 from an audited 5 km static cache or raw static + rasters on the same EPSG:5070 grid, and +3. write a corrected NumPy/TIFF stack plus a JSON audit summary. + +The implementation lives in ``make_firewxfm_current_static_corrected_stack`` so +older experiment commands remain valid. This wrapper gives production a stable +script name that can replace the older ``build_5km_input_stack.py``. +""" + +from __future__ import annotations + +from make_firewxfm_current_static_corrected_stack import main + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/build_fireprone_contract_progression_figure.py b/scripts/build_fireprone_contract_progression_figure.py deleted file mode 100644 index 864e0c13e9c9376d4d333bf51b9a857f7d11ec86..0000000000000000000000000000000000000000 --- a/scripts/build_fireprone_contract_progression_figure.py +++ /dev/null @@ -1,111 +0,0 @@ -#!/usr/bin/env python3 -"""Build the RQ1 fire-prone contract-progression figure from summary JSON.""" - -from __future__ import annotations - -import json -from pathlib import Path - -from simple_pdf import PdfCanvas, draw_axes - - -ROOT = Path(__file__).resolve().parents[1] -IN_JSON = ROOT / "artifacts" / "results" / "fireprone_contract_progression_summary.json" -OUT = ROOT / "paper_outputs" / "figures" / "fig_fireprone_contract_progression_compact.pdf" - -MODEL_ORDER = [ - ("reference", "FireWx-FM ref.", "Ref."), - ("prithvi_wxc", "Prithvi-WxC", "WxC"), - ("aurora", "Aurora", "Aurora"), - ("climax", "ClimaX", "ClimaX"), - ("stormcast", "StormCast", "Storm"), - ("dlwp", "DLWP", "DLWP"), - ("fcn", "FCN", "FCN"), - ("fengwu", "FengWu", "FengWu"), - ("fuxi", "FuXi", "FuXi"), - ("pangu6", "Pangu-Weather", "Pangu-W"), - ("alphaearth", "AlphaEarth", "Alpha"), -] - -SCOPE_ORDER = [ - ("full_domain", "global"), - ("train_fire_top05pct", "top 5%"), - ("train_fire_top10pct", "top 10%"), - ("train_fire_top20pct", "top 20%"), -] - - -def dashed_vline(c: PdfCanvas, x: float, y_start: float, y_end: float) -> None: - dash, gap = 7.0, 5.0 - y = y_start - while y < y_end: - c.line([(x, y), (x, min(y + dash, y_end))], color=(0.42, 0.44, 0.46), lw=0.75) - y += dash + gap - - -def main() -> None: - data = json.loads(IN_JSON.read_text(encoding="utf-8")) - by_key = {(row["model_tag"], row["scope"]): row for row in data["summary"]} - - c = PdfCanvas(width=1320, height=470) - c.rect(0, 0, c.width, c.height, fill=(1, 1, 1)) - - x0, y0, plot_w, plot_h, ymax = 72, 132, 1194, 268, 80.0 - draw_axes(c, x0, y0, plot_w, plot_h, ymax, [0, 20, 40, 60, 80]) - c.text(x0 - 38, y0 + plot_h + 8, "F1 (%)", size=8, color=(0.15, 0.15, 0.15), bold=True) - - colors = { - "strict": (0.09, 0.22, 0.37), - "tolerance": (0.31, 0.55, 0.80), - "union": (0.75, 0.84, 0.94), - } - block_gap = 10.0 - fire_gap = 28.0 - block_w = (plot_w - fire_gap - 2 * block_gap) / len(SCOPE_ORDER) - bar_step = block_w / len(MODEL_ORDER) - bar_w = min(18.0, bar_step * 0.80) - - scope_lefts: dict[str, float] = {} - current_x = x0 + 8.0 - for scope_idx, (scope, scope_label) in enumerate(SCOPE_ORDER): - if scope_idx == 1: - dashed_vline(c, current_x - fire_gap / 2.0, y0 - 6, y0 + plot_h + 16) - scope_lefts[scope] = current_x - c.text(current_x + block_w / 2.0, y0 + plot_h + 17, scope_label, size=15.0, align="center", bold=True) - if scope_idx < len(SCOPE_ORDER) - 1: - current_x += block_w + (fire_gap if scope_idx == 0 else block_gap) - - for scope, _scope_label in SCOPE_ORDER: - block_x = scope_lefts[scope] - for idx, (model_tag, _label, short) in enumerate(MODEL_ORDER): - row = by_key[(model_tag, scope)] - strict = row["strict_f1"]["mean"] * 100.0 - tolerance = row["tolerance_f1"]["mean"] * 100.0 - union = row["union_f1"]["mean"] * 100.0 - bx = block_x + idx * bar_step + (bar_step - bar_w) / 2.0 - base = y0 - for segment, value in [ - ("strict", max(0.0, strict)), - ("tolerance", max(0.0, tolerance - strict)), - ("union", max(0.0, union - tolerance)), - ]: - height = plot_h * value / ymax - if height <= 0: - continue - c.rect(bx, base, bar_w, height, fill=colors[segment], stroke=(1, 1, 1), lw=0.35) - base += height - c.text_rotated(bx + bar_w / 2.0 - 3.0, y0 - 76, short, angle_deg=-45.0, size=10.0, align="right") - - legend_x, legend_y = x0 + 18, y0 + plot_h - 26 - c.rect(legend_x - 13, legend_y - 12, 304, 23, fill=(0.98, 0.98, 0.96), stroke=(0.78, 0.80, 0.78), lw=0.45) - for idx, (label, color) in enumerate([("Strict", colors["strict"]), ("Tolerance", colors["tolerance"]), ("Union", colors["union"])]): - x = legend_x + idx * 98 - c.rect(x, legend_y - 3, 24, 9, fill=color, stroke=(1, 1, 1), lw=0.35) - c.text(x + 31, legend_y - 1, label, size=8.0) - - OUT.parent.mkdir(parents=True, exist_ok=True) - c.save(OUT) - - -if __name__ == "__main__": - main() diff --git a/scripts/build_selection_regret_rq2_figure.py b/scripts/build_selection_regret_rq2_figure.py deleted file mode 100644 index 523e505a463505e1aeeec5a45b960fbaecb6f755..0000000000000000000000000000000000000000 --- a/scripts/build_selection_regret_rq2_figure.py +++ /dev/null @@ -1,187 +0,0 @@ -#!/usr/bin/env python3 -"""Build the compact RQ2 selection-regret interval figure. - -The output is TikZ rather than a raster/PDF plot so the manuscript does not -depend on a local matplotlib installation. -""" - -from __future__ import annotations - -import csv -import math -from pathlib import Path - - -ROOT = Path(__file__).resolve().parents[1] -IN_CSV = ROOT / "artifacts" / "results" / "selection_regret_scope_sweep_20260505.csv" -OUT_TIKZ = ROOT / "paper_outputs" / "figures" / "fig_selection_regret_rq2.tikz" -OUT_VALUES = ROOT / "artifacts" / "results" / "selection_regret_rq2_figure_values.csv" - -ROW_ORDER = [ - "FireWx-FM ref.", - "Prithvi-WxC", - "Aurora", - "ClimaX", - "StormCast", - "DLWP", - "FCN", - "FengWu", - "FuXi", - "Pangu-Weather", - "AlphaEarth", -] - -SCOPES = { - "global": "global", - "fire_prone_top20": "top20", -} - - -def read_rows() -> dict[tuple[str, str], dict[str, str]]: - with IN_CSV.open("r", encoding="utf-8", newline="") as fh: - rows = list(csv.DictReader(fh)) - by_key = {(row["label"], row["scope"]): row for row in rows} - missing = [ - (label, scope) - for label in ROW_ORDER - for scope in SCOPES.values() - if (label, scope) not in by_key - ] - if missing: - raise SystemExit(f"Missing selection-regret rows: {missing}") - bad_labels = sorted({row["label"] for row in rows if "Pangu24" in row["label"]}) - if bad_labels: - raise SystemExit(f"Stale Pangu24 labels found in final CSV: {bad_labels}") - return by_key - - -def pct(row: dict[str, str], field: str) -> float: - return float(row[field]) * 100.0 - - -def escape_tex(text: str) -> str: - return text.replace("&", r"\&").replace("%", r"\%") - - -def nice_floor(value: float, step: float) -> float: - return math.floor(value / step) * step - - -def nice_ceil(value: float, step: float) -> float: - return math.ceil(value / step) * step - - -def write_tikz(records: list[dict[str, float | str]], x_min: float, x_max: float) -> None: - width = 5.55 - left = 2.45 - y_step = 0.41 - top = 4.35 - bottom = -0.35 - offset = 0.13 - x_span = x_max - x_min - - def x(value: float) -> float: - return left + (value - x_min) / x_span * width - - lines: list[str] = [ - r"% Auto-generated by scripts/build_selection_regret_rq2_figure.py.", - r"\begin{tikzpicture}[x=1cm,y=1cm]", - r"\footnotesize", - ] - - ticks = [tick for tick in range(int(x_min), int(x_max) + 1, 10) if tick % 10 == 0] - if 0 not in ticks: - ticks.append(0) - ticks = sorted(set(ticks)) - - for tick in ticks: - xt = x(float(tick)) - color = "wfgray" if tick == 0 else "black!12" - lw = "0.55pt" if tick == 0 else "0.35pt" - lines.append( - rf"\draw[{color}, line width={lw}] ({xt:.3f},{bottom:.3f}) -- ({xt:.3f},{top + 0.18:.3f});" - ) - lines.append(rf"\node[anchor=north, font=\scriptsize, text=black!70] at ({xt:.3f},{bottom - 0.06:.3f}) {{{tick}}};") - - axis_y = bottom - lines.append( - rf"\draw[black!45, line width=0.4pt] ({x(x_min):.3f},{axis_y:.3f}) -- ({x(x_max):.3f},{axis_y:.3f});" - ) - - for idx, record in enumerate(records): - y_base = top - idx * y_step - label = str(record["feature_source"]) - label_tex = escape_tex(label) - if label == "FireWx-FM ref.": - label_tex = rf"\textcolor{{wfblue}}{{\textbf{{{label_tex}}}}}" - lines.append(rf"\node[anchor=east, font=\scriptsize, text=black!82] at ({left - 0.13:.3f},{y_base:.3f}) {{{label_tex}}};") - - for scope_key, color, y_offset, marker in [ - ("global", "wfslate", -offset, "circle"), - ("top20", "wforange", offset, "square"), - ]: - mean = float(record[f"{scope_key}_mean_pp"]) - std = float(record[f"{scope_key}_std_pp"]) - y_val = y_base + y_offset - x_lo = x(mean - std) - x_hi = x(mean + std) - x_mid = x(mean) - lines.append(rf"\draw[{color}, line width=0.72pt] ({x_lo:.3f},{y_val:.3f}) -- ({x_hi:.3f},{y_val:.3f});") - lines.append(rf"\draw[{color}, line width=0.72pt] ({x_lo:.3f},{y_val - 0.035:.3f}) -- ({x_lo:.3f},{y_val + 0.035:.3f});") - lines.append(rf"\draw[{color}, line width=0.72pt] ({x_hi:.3f},{y_val - 0.035:.3f}) -- ({x_hi:.3f},{y_val + 0.035:.3f});") - if marker == "circle": - lines.append(rf"\filldraw[{color}] ({x_mid:.3f},{y_val:.3f}) circle[radius=0.045];") - else: - lines.append(rf"\filldraw[{color}] ({x_mid - 0.045:.3f},{y_val - 0.045:.3f}) rectangle ({x_mid + 0.045:.3f},{y_val + 0.045:.3f});") - - lines.append(r"\end{tikzpicture}") - OUT_TIKZ.parent.mkdir(parents=True, exist_ok=True) - OUT_TIKZ.write_text("\n".join(lines) + "\n", encoding="utf-8") - - -def main() -> None: - by_key = read_rows() - records: list[dict[str, float | str]] = [] - interval_edges: list[float] = [] - - for label in ROW_ORDER: - g = by_key[(label, SCOPES["global"])] - f = by_key[(label, SCOPES["fire_prone_top20"])] - g_mean = pct(g, "union_regret_mean") - g_std = pct(g, "union_regret_std") - f_mean = pct(f, "union_regret_mean") - f_std = pct(f, "union_regret_std") - interval_edges.extend([g_mean - g_std, g_mean + g_std, f_mean - f_std, f_mean + f_std]) - records.append( - { - "feature_source": label, - "global_mean_pp": g_mean, - "global_std_pp": g_std, - "top20_mean_pp": f_mean, - "top20_std_pp": f_std, - } - ) - - x_min = min(-5.0, nice_floor(min(interval_edges), 10.0)) - x_max = max(30.0, nice_ceil(max(interval_edges), 10.0)) - write_tikz(records, x_min, x_max) - - OUT_VALUES.parent.mkdir(parents=True, exist_ok=True) - with OUT_VALUES.open("w", encoding="utf-8", newline="") as fh: - fieldnames = ["feature_source", "global_mean_pp", "global_std_pp", "top20_mean_pp", "top20_std_pp"] - writer = csv.DictWriter(fh, fieldnames=fieldnames) - writer.writeheader() - for record in records: - writer.writerow( - { - "feature_source": record["feature_source"], - "global_mean_pp": f"{float(record['global_mean_pp']):.4f}", - "global_std_pp": f"{float(record['global_std_pp']):.4f}", - "top20_mean_pp": f"{float(record['top20_mean_pp']):.4f}", - "top20_std_pp": f"{float(record['top20_std_pp']):.4f}", - } - ) - - -if __name__ == "__main__": - main() diff --git a/scripts/build_task_rank_map.py b/scripts/build_task_rank_map.py deleted file mode 100644 index 65d65118cdb04485e0242539bb0790de9baa6bb7..0000000000000000000000000000000000000000 --- a/scripts/build_task_rank_map.py +++ /dev/null @@ -1,141 +0,0 @@ -#!/usr/bin/env python3 -"""Build the RQ4 task-rank map from released paper table TeX. - -This uses only the Python standard library plus the small PDF helper bundled in -this repository. It does not require raw data, feature caches, or matplotlib. -""" - -from __future__ import annotations - -import re -from pathlib import Path - -from simple_pdf import PdfCanvas, mix - - -ROOT = Path(__file__).resolve().parents[1] -TABLE_DIR = ROOT / "paper_outputs" / "tables" -OUT = ROOT / "paper_outputs" / "figures" / "fig_task_rank_map.pdf" - -MODELS = [ - "FireWx-FM ref.", - "Prithvi-WxC", - "Aurora", - "ClimaX", - "StormCast", - "DLWP", - "FCN", - "FengWu", - "FuXi", - "Pangu-Weather", - "AlphaEarth", -] - -DISPLAY_MODELS = [ - "FireWx-FM ref.", - "Prithvi-WxC", - "Aurora", - "ClimaX", - "StormCast", - "DLWP", - "FCN", - "FengWu", - "FuXi", - "Pangu-Weather", - "AlphaEarth", -] - - -def means_from_row(block: str, label: str, occurrence: int = 0) -> list[float]: - starts = [m.start() for m in re.finditer(re.escape(label), block)] - if len(starts) <= occurrence: - raise ValueError(f"Missing row {label!r} occurrence {occurrence}") - start = starts[occurrence] - end = block.find(r"\\", start) - if end < 0: - raise ValueError(f"Missing row terminator after {label!r}") - row = block[start:end] - values = [float(x) for x in re.findall(r"\\ms\{(-?\d+(?:\.\d+)?)\}\{", row)] - if not values: - raise ValueError(f"No values found for row {label!r}") - return values - - -def rank_values(values: list[float], higher_is_better: bool) -> list[int]: - order = sorted(range(len(values)), key=lambda i: values[i], reverse=higher_is_better) - ranks = [0] * len(values) - for rank, idx in enumerate(order, start=1): - ranks[idx] = rank - return ranks - - -def rank_color(rank: int, n_cols: int) -> tuple[float, float, float]: - t = (n_cols - rank) / max(1, n_cols - 1) - if t <= 0.5: - return mix((0.93, 0.95, 0.94), (0.55, 0.78, 0.75), t / 0.5) - return mix((0.55, 0.78, 0.75), (0.05, 0.40, 0.42), (t - 0.5) / 0.5) - - -def fmt_value(value: float) -> str: - return f"{value:.2f}" if abs(value) >= 1 else f"{value:.3f}" - - -def main() -> None: - primary = (TABLE_DIR / "tab_primary_results.tex").read_text() - supporting = (TABLE_DIR / "tab_supporting_results.tex").read_text() - - primary_rows = {label: means_from_row(primary, label) for label in MODELS} - supporting_top = {label: means_from_row(supporting, label, occurrence=0) for label in MODELS} - supporting_bottom = {label: means_from_row(supporting, label, occurrence=1) for label in MODELS} - - tasks = [ - ("Occupancy", "Union F1 (%)", "higher better", [primary_rows[m][2] for m in MODELS], True), - ("Fire spread", "AP (%)", "higher better", [primary_rows[m][5] for m in MODELS], True), - ("Burned area", "log-RMSE", "lower better", [supporting_top[m][0] for m in MODELS], False), - ("Analog retrieval", "nDCG@10", "higher better", [supporting_top[m][3] for m in MODELS], True), - ("Smoke PM2.5", "RMSE", "lower better", [supporting_bottom[m][0] for m in MODELS], False), - ("Extreme heat", "RMSE-C", "lower better", [supporting_bottom[m][3] for m in MODELS], False), - ] - - n_rows = len(tasks) - n_cols = len(MODELS) - c = PdfCanvas(width=1120, height=430) - c.rect(0, 0, c.width, c.height, fill=(1, 1, 1)) - - x0, y0 = 108, 90 - cell_w, cell_h = 86, 42 - grid_top = y0 + n_rows * cell_h - - for j, model in enumerate(DISPLAY_MODELS): - c.text(x0 + j * cell_w + cell_w / 2, grid_top + 34, model, size=8.7, align="center", bold=True, color=(0.12, 0.14, 0.16)) - - for i, (task, metric, direction, values, higher) in enumerate(tasks): - y = grid_top - (i + 1) * cell_h - c.text(12, y + 25, task, size=7.7, bold=True, color=(0.12, 0.14, 0.16)) - c.text(12, y + 14, metric, size=7.1, bold=True, color=(0.12, 0.14, 0.16)) - c.text(12, y + 3, direction, size=6.4, color=(0.42, 0.44, 0.46)) - ranks = rank_values(values, higher) - for j, (rank, value) in enumerate(zip(ranks, values)): - x = x0 + j * cell_w - color = rank_color(rank, n_cols) - text_color = (1, 1, 1) if rank <= 2 else (0.07, 0.09, 0.11) - c.rect(x, y, cell_w, cell_h, fill=color, stroke=(1, 1, 1), lw=0.8) - c.text(x + cell_w / 2, y + 24, f"#{rank}", size=11.2, align="center", bold=True, color=text_color) - c.text(x + cell_w / 2, y + 9, fmt_value(value), size=7.0, align="center", color=text_color) - - c.rect(x0, y0, cell_w * n_cols, cell_h * n_rows, stroke=(0.20, 0.22, 0.24), lw=0.8) - - key_x, key_y = x0 + cell_w * n_cols - 220, 38 - c.text(key_x + 110, key_y + 25, "within-row rank", size=9.0, bold=True, align="center", color=(0.24, 0.25, 0.26)) - for i in range(80): - color = rank_color(n_cols - int(i / 80 * (n_cols - 1)), n_cols) - c.rect(key_x + i * 2.5, key_y + 8, 2.6, 10, fill=color) - c.text(key_x, key_y - 4, f"rank {n_cols}", size=7.0, color=(0.25, 0.26, 0.27)) - c.text(key_x + 200, key_y - 4, "rank 1", size=7.0, align="right", color=(0.25, 0.26, 0.27)) - - OUT.parent.mkdir(parents=True, exist_ok=True) - c.save(OUT) - - -if __name__ == "__main__": - main() diff --git a/scripts/check_paper_output_hashes.py b/scripts/check_paper_output_hashes.py deleted file mode 100644 index 3014c0a41b5a6b07932708edb043b405ad11bd24..0000000000000000000000000000000000000000 --- a/scripts/check_paper_output_hashes.py +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env python3 -"""Verify final paper-output checksums.""" - -from __future__ import annotations - -import hashlib -from pathlib import Path - - -ROOT = Path(__file__).resolve().parents[1] -MANIFEST = ROOT / "artifacts" / "manifests" / "paper_outputs.sha256" - - -def sha256(path: Path) -> str: - digest = hashlib.sha256() - with path.open("rb") as fh: - for chunk in iter(lambda: fh.read(1024 * 1024), b""): - digest.update(chunk) - return digest.hexdigest() - - -def main() -> None: - issues: list[str] = [] - for line_no, line in enumerate(MANIFEST.read_text(encoding="utf-8").splitlines(), start=1): - if not line.strip(): - continue - try: - expected, rel = line.split(None, 1) - except ValueError: - issues.append(f"bad checksum line {line_no}: {line!r}") - continue - path = ROOT / rel.strip() - if not path.exists(): - issues.append(f"missing output listed in checksum manifest: {rel}") - continue - actual = sha256(path) - if actual != expected: - issues.append(f"checksum mismatch for {rel}: expected {expected}, got {actual}") - - if issues: - print("Paper-output checksum check failed:") - for issue in issues: - print(f"- {issue}") - raise SystemExit(1) - print("Paper-output checksum check passed.") - - -if __name__ == "__main__": - main() diff --git a/scripts/check_release_files.py b/scripts/check_release_files.py new file mode 100644 index 0000000000000000000000000000000000000000..66e922db3d68ea101f0c9d515a444b4676696e46 --- /dev/null +++ b/scripts/check_release_files.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +"""Check that the FireWx-FM release artifacts are present.""" + +from __future__ import annotations + +from pathlib import Path + + +REQUIRED = [ + "firewxfm/modeling_unet.py", + "firewxfm/tiled_inference.py", + "firewxfm/serve_conus.py", + "models/metadata/input_channels.json", + "models/metadata/final_model_manifest.json", + "models/metadata/input_normalization_stats.json", + "models/checkpoints/firewxfm_conus_lowposw_noexposure_seed42.pt", + "models/checkpoints/firewxfm_2024_staticfix_region_balanced_bce_seed42.pt", + "examples/final_prediction/firewxfm_conus_20260706_t12_usgs_calibrated_probability_5km_lower48.tif", + "examples/final_prediction/firewxfm_conus_20260706_t12_usgs_calibrated_heatmap.png", + "examples/final_prediction/firewxfm_conus_20260706_t12_usgs_calibrated_heatmap_rgb.tif", + "examples/final_prediction/firewxfm_conus_20260706_t12_usgs_calibrated_summary.json", +] + + +OPTIONAL = [ + "models/metadata/run_summary.json", + "models/metadata/tile_summary.json", + "docs/split_design.md", + "docs/reproducibility_manifest.md", + "docs/inference_contract.md", + "docs/serving_notes.md", + "docs/model_card.md", + "REPRODUCIBILITY.md", + "paper/Does_Your_Wildfire_Prediction_Model_ICLR_revision.pdf", + "paper/iclr_source/main.tex", + "technical_report/FireWx-FM_technical_report.pdf", + "technical_report/source/main.tex", + "scripts/audit_firewxfm_phase0.py", + "scripts/prepare_firms_target_hour_csvs.py", + "scripts/smoke_inference.py", + "training/configs/stage1_cache_conus_multiyear_template.json", + "training/configs/train_firewxfm_conus_multiyear_template.json", + "training/configs/train_firewxfm_multitask_pretrain_template.json", + "training/configs/eval_region_disjoint_template.json", + "training/configs/baseline_suite_template.json", + "scripts/build_5km_input_stack.py", + "scripts/audit_firewxfm_longitude_seams.py", + "scripts/make_firewxfm_current_static_corrected_stack.py", + "scripts/apply_prediction_mask.py", + "scripts/write_prediction_tif.py", +] + + +def describe(path: Path) -> str: + if not path.exists(): + return "missing" + if path.is_file(): + size_mb = path.stat().st_size / (1024 * 1024) + return f"ok ({size_mb:.2f} MB)" + return "ok" + + +def main() -> int: + root = Path(__file__).resolve().parents[1] + missing = [] + print("Required artifacts") + for rel in REQUIRED: + status = describe(root / rel) + print(f" {rel}: {status}") + if status == "missing": + missing.append(rel) + print("\nOptional artifacts") + for rel in OPTIONAL: + print(f" {rel}: {describe(root / rel)}") + if missing: + print("\nRelease check failed. Missing required files:") + for rel in missing: + print(f" - {rel}") + return 1 + print("\nRelease check passed.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/data_downloader/hrrr_downloader.py b/scripts/hrrr_downloader.py similarity index 100% rename from data_downloader/hrrr_downloader.py rename to scripts/hrrr_downloader.py diff --git a/scripts/make_firewxfm_current_static_corrected_stack.py b/scripts/make_firewxfm_current_static_corrected_stack.py new file mode 100644 index 0000000000000000000000000000000000000000..8b060b516c95ca7887729896c25d136d9bd188a5 --- /dev/null +++ b/scripts/make_firewxfm_current_static_corrected_stack.py @@ -0,0 +1,285 @@ +#!/usr/bin/env python3 +"""Replace static channels in a current FireWx-FM stack. + +The preferred path uses an already-audited 5 km static cache from the corrected +training cache. A raw-raster fallback is available for rebuilding that cache. +""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import Any + +import numpy as np +import rasterio +from affine import Affine +from rasterio.enums import Resampling +from rasterio.vrt import WarpedVRT +from rasterio.warp import reproject + + +STATIC_LAYERS = [ + ("fuel_fbfm40", 12, "LANDFIRE_FUELS_2024/FBFM40/LF2024_FBFM40_CONUS/Tif/LF2024_FBFM40_CONUS.tif", Resampling.nearest), + ("canopy_cover", 13, "LANDFIRE_FUELS_2024/CC/LF2024_CC_CONUS/Tif/LF2024_CC_CONUS.tif", Resampling.nearest), + ("housing_density", 14, "WRC_Housing_Density/extracted/HUDen_CONUS.tif", Resampling.bilinear), + ("population", 15, "LandScan_Global_2024/extracted/landscan-global-2024.tif", Resampling.bilinear), +] + +STATIC_CACHE_NAME_TO_CHANNEL = { + "fuel_fbfm40": 12, + "canopy_cover": 13, + "housing_density": 14, + "population": 15, +} + + +def summarize(arr: np.ndarray, mask: np.ndarray | None = None) -> dict[str, Any]: + vals = arr[mask] if mask is not None else arr.ravel() + vals = vals[np.isfinite(vals)] + if vals.size == 0: + return {"cells": 0} + return { + "cells": int(vals.size), + "mean": float(vals.mean()), + "q50": float(np.quantile(vals, 0.50)), + "q90": float(np.quantile(vals, 0.90)), + "q99": float(np.quantile(vals, 0.99)), + "max": float(vals.max()), + "fraction_zero": float((vals == 0.0).mean()), + "fraction_le_0p25": float((vals <= 0.25).mean()), + } + + +def load_lower48_mask( + reference_tif: Path, + lower48_mask_npy: Path | None = None, +) -> tuple[np.ndarray, str]: + """Load the Lower-48 mask or derive it from the reference stack.""" + if lower48_mask_npy is not None: + lower48 = np.load(lower48_mask_npy).astype(bool) + return lower48, str(lower48_mask_npy) + + with rasterio.open(reference_tif) as ref: + # Prefer the dataset mask because the live stack is expected to carry the + # CONUS support footprint directly on the reference grid. + lower48 = ref.dataset_mask() > 0 + if not np.any(lower48): + raise ValueError( + "Could not derive a Lower-48 mask from the reference stack; " + "supply --lower48-mask-npy explicitly." + ) + return lower48.astype(bool), f"{reference_tif}::dataset_mask()" + + +def read_on_reference_grid(path: Path, ref: rasterio.DatasetReader, resampling: Resampling) -> tuple[np.ndarray, np.ndarray]: + """Read a source raster directly on the reference 5 km grid. + + The raw LANDFIRE/WRC layers are 30 m CONUS rasters. Reading the whole + overlapping source window before reprojection can require many GB of RAM. + WarpedVRT lets GDAL sample only the requested 5 km output grid. + """ + with rasterio.open(path) as src: + vrt_kwargs: dict[str, Any] = { + "crs": ref.crs, + "transform": ref.transform, + "width": ref.width, + "height": ref.height, + "resampling": resampling, + "dtype": "float32", + } + if src.nodata is not None: + vrt_kwargs["src_nodata"] = src.nodata + vrt_kwargs["nodata"] = np.nan + with WarpedVRT(src, **vrt_kwargs) as vrt: + masked = vrt.read(1, masked=True) + + data = np.asarray(masked.filled(np.nan), dtype=np.float32) + valid = (~np.ma.getmaskarray(masked)) & np.isfinite(data) & (data > -9000.0) + clean = np.where(valid, data, 0.0).astype(np.float32) + return clean, valid.astype(np.float32) + + +def transform_from_centers(y: np.ndarray, x: np.ndarray) -> Affine: + if y.ndim != 1 or x.ndim != 1 or len(y) < 2 or len(x) < 2: + raise ValueError("Static cache lat/lon coordinate arrays must be 1-D with at least two cells.") + dx = float(np.median(np.diff(x))) + dy = float(np.median(np.abs(np.diff(y)))) + if dx <= 0 or dy <= 0: + raise ValueError(f"Unexpected static cache grid spacing dx={dx}, dy={dy}") + left = float(x[0]) - 0.5 * dx + if y[0] > y[-1]: + top = float(y[0]) + 0.5 * dy + return Affine(dx, 0.0, left, 0.0, -dy, top) + bottom = float(y[0]) - 0.5 * dy + return Affine(dx, 0.0, left, 0.0, dy, bottom) + + +def replace_from_static_cache(stack: np.ndarray, static_cache_npz: Path, reference_tif: Path, lower48: np.ndarray) -> dict[str, Any]: + print(f"[static] using corrected static cache: {static_cache_npz}", flush=True) + with np.load(static_cache_npz, allow_pickle=True) as data: + static = data["static"].astype(np.float32) + static_valid = data["static_valid"].astype(np.float32) + y_src = data["lat"].astype(np.float32) + x_src = data["lon"].astype(np.float32) + coord_crs = str(data["coord_crs"][0]) if "coord_crs" in data else "EPSG:5070" + names = [str(v) for v in data["static_names"].tolist()] + + source_transform = transform_from_centers(y_src, x_src) + static_aligned = np.zeros((static.shape[0], stack.shape[1], stack.shape[2]), dtype=np.float32) + static_valid_aligned = np.zeros((1, stack.shape[1], stack.shape[2]), dtype=np.float32) + with rasterio.open(reference_tif) as ref: + dst_transform = ref.transform + dst_crs = ref.crs + print( + f"[static] reproject cache grid {static.shape[1:]} -> reference grid {stack.shape[1:]}", + flush=True, + ) + reproject( + source=static_valid[0], + destination=static_valid_aligned[0], + src_transform=source_transform, + src_crs=coord_crs, + dst_transform=dst_transform, + dst_crs=dst_crs, + src_nodata=None, + dst_nodata=0.0, + resampling=Resampling.nearest, + ) + for i, name in enumerate(names): + print(f"[static] align cache layer {name}", flush=True) + reproject( + source=static[i], + destination=static_aligned[i], + src_transform=source_transform, + src_crs=coord_crs, + dst_transform=dst_transform, + dst_crs=dst_crs, + src_nodata=None, + dst_nodata=0.0, + resampling=Resampling.nearest, + ) + + layer_summaries: dict[str, Any] = {} + stack[11] = static_valid_aligned[0] + for i, name in enumerate(names): + channel = STATIC_CACHE_NAME_TO_CHANNEL.get(name) + if channel is None: + continue + stack[channel] = static_aligned[i] + layer_summaries[name] = { + "channel": int(channel), + "source_index": int(i), + "path": str(static_cache_npz), + "resampling": "cache_nearest_reproject_to_reference", + "values_all": summarize(static_aligned[i]), + "values_lower48": summarize(static_aligned[i], lower48), + } + return { + "source": "corrected_static_cache", + "static_cache_npz": str(static_cache_npz), + "coord_crs": coord_crs, + "source_shape": [int(v) for v in static.shape], + "aligned_shape": [int(v) for v in static_aligned.shape], + "source_names": names, + "layer_summaries": layer_summaries, + } + + +def write_stack_tif(path: Path, stack: np.ndarray, reference_tif: Path) -> None: + with rasterio.open(reference_tif) as ref: + profile = ref.profile.copy() + profile.update(count=stack.shape[0], dtype="float32", nodata=None, compress="deflate") + path.parent.mkdir(parents=True, exist_ok=True) + with rasterio.open(path, "w", **profile) as dst: + dst.write(stack.astype(np.float32)) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--input-stack-npy", type=Path, required=True) + parser.add_argument("--reference-stack-tif", type=Path, required=True) + parser.add_argument("--raw-root", type=Path) + parser.add_argument("--static-cache-npz", type=Path) + parser.add_argument("--lower48-mask-npy", type=Path) + parser.add_argument("--write-lower48-mask-npy", type=Path) + parser.add_argument("--output-stack-npy", type=Path, required=True) + parser.add_argument("--output-stack-tif", type=Path) + parser.add_argument("--summary-json", type=Path, required=True) + args = parser.parse_args() + if args.static_cache_npz is None and args.raw_root is None: + raise ValueError("Provide --static-cache-npz for the fast path or --raw-root for raw raster rebuild.") + + stack = np.load(args.input_stack_npy).astype(np.float32) + if stack.ndim != 3 or stack.shape[0] != 16: + raise ValueError(f"Expected [16,H,W] input stack, got {stack.shape}") + lower48, lower48_source = load_lower48_mask(args.reference_stack_tif, args.lower48_mask_npy) + if lower48.shape != stack.shape[1:]: + raise ValueError(f"lower48 mask {lower48.shape} does not match stack {stack.shape[1:]}") + if args.write_lower48_mask_npy is not None: + args.write_lower48_mask_npy.parent.mkdir(parents=True, exist_ok=True) + np.save(args.write_lower48_mask_npy, lower48.astype(bool)) + + old_static = stack[11:16].copy() + with rasterio.open(args.reference_stack_tif) as ref: + if (ref.height, ref.width) != stack.shape[1:]: + raise ValueError(f"reference grid {(ref.height, ref.width)} does not match stack {stack.shape[1:]}") + + if args.static_cache_npz is not None: + replacement = replace_from_static_cache(stack, args.static_cache_npz, args.reference_stack_tif, lower48) + layer_summaries = replacement["layer_summaries"] + else: + print("[static] rebuilding from raw rasters", flush=True) + static_valids = [] + layer_summaries: dict[str, Any] = {} + with rasterio.open(args.reference_stack_tif) as ref: + for name, channel, rel_path, resampling in STATIC_LAYERS: + print(f"[static] read {name}: {args.raw_root / rel_path}", flush=True) + arr, valid = read_on_reference_grid(args.raw_root / rel_path, ref, resampling) + stack[channel] = arr + static_valids.append(valid) + layer_summaries[name] = { + "channel": int(channel), + "path": str(args.raw_root / rel_path), + "resampling": str(resampling.name), + "valid_fraction_all": float((valid > 0.0).mean()), + "valid_fraction_lower48": float((valid[lower48] > 0.0).mean()), + "values_all": summarize(arr), + "values_lower48": summarize(arr, lower48), + } + static_valid = np.stack(static_valids, axis=0).mean(axis=0).astype(np.float32) + stack[11] = static_valid + replacement = {"source": "raw_rasters", "raw_root": str(args.raw_root)} + + args.output_stack_npy.parent.mkdir(parents=True, exist_ok=True) + np.save(args.output_stack_npy, stack.astype(np.float32)) + if args.output_stack_tif is not None: + write_stack_tif(args.output_stack_tif, stack, args.reference_stack_tif) + + summary = { + "input_stack_npy": str(args.input_stack_npy), + "reference_stack_tif": str(args.reference_stack_tif), + "raw_root": str(args.raw_root) if args.raw_root is not None else None, + "static_cache_npz": str(args.static_cache_npz) if args.static_cache_npz is not None else None, + "lower48_mask_npy": str(args.lower48_mask_npy) if args.lower48_mask_npy is not None else None, + "lower48_mask_source": lower48_source, + "lower48_mask_written": str(args.write_lower48_mask_npy) if args.write_lower48_mask_npy is not None else None, + "output_stack_npy": str(args.output_stack_npy), + "output_stack_tif": str(args.output_stack_tif) if args.output_stack_tif is not None else None, + "replacement": replacement, + "shape": [int(v) for v in stack.shape], + "old_static_valid_lower48": summarize(old_static[0], lower48), + "new_static_valid_lower48": summarize(stack[11], lower48), + "old_fuel_lower48": summarize(old_static[1], lower48), + "new_fuel_lower48": summarize(stack[12], lower48), + "layers": layer_summaries, + } + args.summary_json.parent.mkdir(parents=True, exist_ok=True) + args.summary_json.write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps(summary, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/prepare_firms_target_hour_csvs.py b/scripts/prepare_firms_target_hour_csvs.py new file mode 100644 index 0000000000000000000000000000000000000000..471744dc27e3c95baa89a98c40aa99cf64b601ad --- /dev/null +++ b/scripts/prepare_firms_target_hour_csvs.py @@ -0,0 +1,257 @@ +#!/usr/bin/env python3 +"""Convert daily FIRMS CSV files into target-hour label CSVs. + +The Phase-1 cache builder expects one FIRMS CSV per target timestamp, named +``YYYYMMDD_HH.csv``. Public FIRMS exports are often daily CSVs with +``acq_date`` and ``acq_time`` columns. This utility assigns each detection to +the nearest expected target timestamp, usually the 00/06/12/18 UTC grid used by +the HRRR cache, and writes empty target files when no detections are present. +""" + +from __future__ import annotations + +import argparse +import json +import re +from datetime import datetime, timedelta +from pathlib import Path +from typing import Any, Dict, Iterable, List, Sequence, Tuple + +import numpy as np +import pandas as pd + + +DEFAULT_FIRMS_COLUMNS = [ + "latitude", + "longitude", + "bright_ti4", + "scan", + "track", + "acq_date", + "acq_time", + "satellite", + "instrument", + "confidence", + "version", + "bright_ti5", + "frp", + "daynight", + "type", +] + + +def load_json(path: Path) -> Dict[str, Any]: + return json.loads(path.read_text(encoding="utf-8")) + + +def parse_day(value: str) -> datetime: + return datetime.strptime(value, "%Y-%m-%d") + + +def iter_days(start: datetime, end: datetime) -> Iterable[datetime]: + current = start + while current <= end: + yield current + current += timedelta(days=1) + + +def expected_targets_from_config(config: Dict[str, Any]) -> List[datetime]: + issue_hours = [int(v) for v in config.get("issue_hours", [0, 6, 12, 18])] + lead_hours = int(config.get("target_offset_hours", 12)) + targets: List[datetime] = [] + for item in config.get("input_ranges", []): + for day in iter_days(parse_day(str(item["start"])), parse_day(str(item["end"]))): + for hour in issue_hours: + targets.append(day.replace(hour=hour) + timedelta(hours=lead_hours)) + return sorted(set(targets)) + + +def expected_targets_from_args(args: argparse.Namespace) -> List[datetime]: + if not args.start_date or not args.end_date: + raise SystemExit("--start-date and --end-date are required when --config is not provided.") + hours = [int(v) for v in args.target_hours.split(",") if v.strip()] + targets: List[datetime] = [] + for day in iter_days(parse_day(args.start_date), parse_day(args.end_date)): + for hour in hours: + targets.append(day.replace(hour=hour)) + return sorted(set(targets)) + + +def normalize_acq_time(value: object) -> str: + if pd.isna(value): + return "" + text = str(value).strip() + if text.endswith(".0"): + text = text[:-2] + digits = re.sub(r"\D", "", text) + if not digits: + return "" + return digits.zfill(4)[-4:] + + +def parse_firms_timestamps(df: pd.DataFrame, date_column: str, time_column: str) -> pd.Series: + if date_column not in df.columns or time_column not in df.columns: + raise KeyError(f"FIRMS CSV must contain '{date_column}' and '{time_column}' columns.") + dates = df[date_column].astype(str).str.strip() + times = df[time_column].map(normalize_acq_time) + return pd.to_datetime(dates + times, format="%Y-%m-%d%H%M", errors="coerce") + + +def nearest_expected_targets( + timestamps: pd.Series, + expected_targets: Sequence[datetime], + max_delta: timedelta, +) -> Tuple[pd.Series, pd.Series]: + expected_ns = np.asarray([pd.Timestamp(ts).value for ts in expected_targets], dtype=np.int64) + values = timestamps.astype("datetime64[ns]").astype("int64").to_numpy() + invalid_nat = values == np.iinfo(np.int64).min + + insert_pos = np.searchsorted(expected_ns, values, side="left") + right_pos = np.clip(insert_pos, 0, len(expected_ns) - 1) + left_pos = np.clip(insert_pos - 1, 0, len(expected_ns) - 1) + left_delta = np.abs(values - expected_ns[left_pos]) + right_delta = np.abs(values - expected_ns[right_pos]) + choose_right = right_delta < left_delta + nearest_pos = np.where(choose_right, right_pos, left_pos) + nearest_delta = np.minimum(left_delta, right_delta) + + max_delta_ns = int(pd.Timedelta(max_delta).value) + ok = (~invalid_nat) & (nearest_delta <= max_delta_ns) + assigned_values = np.where(ok, expected_ns[nearest_pos], np.iinfo(np.int64).min) + assigned = pd.to_datetime(assigned_values) + return pd.Series(assigned, index=timestamps.index), pd.Series(ok, index=timestamps.index) + + +def read_header_from_daily_dir(daily_dir: Path) -> List[str]: + for path in sorted(daily_dir.glob("*.csv")): + try: + return list(pd.read_csv(path, nrows=0).columns) + except pd.errors.EmptyDataError: + continue + return list(DEFAULT_FIRMS_COLUMNS) + + +def source_dates_for_targets(targets: Sequence[datetime], half_window: timedelta) -> List[datetime]: + first = min(targets) - half_window + last = max(targets) + half_window + return list(iter_days(first.replace(hour=0, minute=0, second=0, microsecond=0), last.replace(hour=0, minute=0, second=0, microsecond=0))) + + +def write_empty_csv(path: Path, columns: Sequence[str]) -> None: + pd.DataFrame(columns=list(columns)).to_csv(path, index=False) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--daily-dir", type=Path, required=True, help="Directory containing daily FIRMS YYYY-MM-DD.csv files.") + parser.add_argument("--output-dir", type=Path, required=True, help="Directory for YYYYMMDD_HH.csv target files.") + parser.add_argument("--config", type=Path, default=None, help="Optional Stage-1 cache config used to derive target timestamps.") + parser.add_argument("--start-date", type=str, default="", help="Target start date if --config is not provided.") + parser.add_argument("--end-date", type=str, default="", help="Target end date if --config is not provided.") + parser.add_argument("--target-hours", type=str, default="0,6,12,18", help="Comma-separated target hours if --config is not provided.") + parser.add_argument("--bin-hours", type=float, default=6.0, help="Centered assignment window. Default assigns to nearest 6-hour target.") + parser.add_argument("--date-column", type=str, default="acq_date") + parser.add_argument("--time-column", type=str, default="acq_time") + parser.add_argument("--fire-type", type=str, default="0", help="Keep this FIRMS type value when a type column exists. Use 'all' to disable.") + parser.add_argument("--no-bbox-filter", action="store_true", help="Do not filter detections to the config lat/lon bounds.") + parser.add_argument("--summary", type=Path, default=None, help="Optional JSON summary path.") + parser.add_argument("--dry-run", action="store_true", help="Report work without writing CSV files.") + args = parser.parse_args() + + config: Dict[str, Any] = {} + if args.config: + config = load_json(args.config) + expected_targets = expected_targets_from_config(config) + else: + expected_targets = expected_targets_from_args(args) + if not expected_targets: + raise SystemExit("No expected target timestamps were found.") + + half_window = timedelta(hours=float(args.bin_hours) / 2.0) + output_columns = read_header_from_daily_dir(args.daily_dir) + assigned_frames: Dict[datetime, List[pd.DataFrame]] = {ts: [] for ts in expected_targets} + stats: Dict[str, Any] = { + "daily_dir": str(args.daily_dir), + "output_dir": str(args.output_dir), + "config": str(args.config) if args.config else "", + "expected_targets": len(expected_targets), + "first_target": min(expected_targets).isoformat(), + "last_target": max(expected_targets).isoformat(), + "bin_hours": float(args.bin_hours), + "source_files_found": 0, + "source_files_missing": 0, + "source_rows": 0, + "assigned_rows": 0, + "dropped_missing_timestamp": 0, + "dropped_outside_window": 0, + "dropped_bbox": 0, + "dropped_type": 0, + } + + for day in source_dates_for_targets(expected_targets, half_window): + path = args.daily_dir / f"{day.strftime('%Y-%m-%d')}.csv" + if not path.exists(): + stats["source_files_missing"] += 1 + continue + stats["source_files_found"] += 1 + df = pd.read_csv(path) + stats["source_rows"] += int(len(df)) + if df.empty: + continue + if not args.no_bbox_filter and config: + before = len(df) + df = df[ + df["latitude"].astype(float).between(float(config["lat_min"]), float(config["lat_max"])) + & df["longitude"].astype(float).between(float(config["lon_min"]), float(config["lon_max"])) + ].copy() + stats["dropped_bbox"] += int(before - len(df)) + if args.fire_type.lower() != "all" and "type" in df.columns: + before = len(df) + df = df[df["type"].astype(str) == str(args.fire_type)].copy() + stats["dropped_type"] += int(before - len(df)) + if df.empty: + continue + timestamps = parse_firms_timestamps(df, args.date_column, args.time_column) + assigned, ok = nearest_expected_targets(timestamps, expected_targets, half_window) + stats["dropped_missing_timestamp"] += int(timestamps.isna().sum()) + stats["dropped_outside_window"] += int(((~ok) & timestamps.notna()).sum()) + if not ok.any(): + continue + df = df.loc[ok].copy() + df["_assigned_target"] = assigned.loc[ok].to_numpy(dtype="datetime64[ns]") + df["_acq_datetime"] = timestamps.loc[ok] + stats["assigned_rows"] += int(len(df)) + for target, part in df.groupby("_assigned_target", sort=True): + assigned_frames[pd.Timestamp(target).to_pydatetime()].append(part) + + written_files = 0 + nonempty_files = 0 + if not args.dry_run: + args.output_dir.mkdir(parents=True, exist_ok=True) + for target in expected_targets: + path = args.output_dir / f"{target.strftime('%Y%m%d_%H')}.csv" + frames = assigned_frames[target] + if frames: + out_df = pd.concat(frames, ignore_index=True).sort_values("_acq_datetime") + out_df = out_df[[col for col in output_columns if col in out_df.columns]] + nonempty_files += 1 + else: + out_df = pd.DataFrame(columns=output_columns) + written_files += 1 + if not args.dry_run: + out_df.to_csv(path, index=False) + + stats["written_files"] = int(written_files) + stats["nonempty_files"] = int(nonempty_files) + stats["empty_files"] = int(written_files - nonempty_files) + if args.summary: + stats_path = args.summary + if not args.dry_run: + stats_path.parent.mkdir(parents=True, exist_ok=True) + stats_path.write_text(json.dumps(stats, indent=2, sort_keys=True), encoding="utf-8") + print(json.dumps(stats, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/reproduce_paper_outputs.py b/scripts/reproduce_paper_outputs.py deleted file mode 100644 index e6114fd310529ed1702fff3244d27a0276b16f92..0000000000000000000000000000000000000000 --- a/scripts/reproduce_paper_outputs.py +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env python3 -"""Verify the released WildFIRE-FM public artifacts. - -The Hub release verifies public figure previews, sanitized summaries, compiled -paper output, and release hygiene while manuscript source remains in the authors' -paper workspace. -""" - -from __future__ import annotations - -import subprocess -import sys -from pathlib import Path - - -ROOT = Path(__file__).resolve().parents[1] - - -def run(cmd: list[str]) -> None: - print("+", " ".join(cmd), flush=True) - subprocess.run(cmd, cwd=ROOT, check=True) - - -def main() -> None: - run([sys.executable, "scripts/check_paper_output_hashes.py"]) - run([sys.executable, "scripts/audit_release.py"]) - print("Verified public release artifacts and release audit.") - - -if __name__ == "__main__": - main() diff --git a/scripts/run_firewxfm_baselines.py b/scripts/run_firewxfm_baselines.py new file mode 100644 index 0000000000000000000000000000000000000000..d4c5b090ff927c6b6adaab19db5ec9332393e8f7 --- /dev/null +++ b/scripts/run_firewxfm_baselines.py @@ -0,0 +1,290 @@ +#!/usr/bin/env python3 +"""Run simple FireWx-FM baselines on a cached CONUS experiment.""" + +from __future__ import annotations + +import argparse +import csv +import json +import sys +from pathlib import Path +from typing import Iterable + +import numpy as np +from sklearn.linear_model import SGDClassifier +from sklearn.metrics import average_precision_score, roc_auc_score +from sklearn.preprocessing import StandardScaler + +REPO_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO_ROOT / "training")) + +from eval_metrics import expected_calibration_error, fss_metrics, threshold_metrics, topk_area_metrics # noqa: E402 + + +def read_rows(path: Path) -> list[dict[str, str]]: + with path.open("r", encoding="utf-8", newline="") as fh: + return list(csv.DictReader(fh)) + + +def load_target(row: dict[str, str]) -> np.ndarray: + with np.load(row["sample_path"], allow_pickle=True) as sample: + return sample["y_occ"][0].astype(np.float32) + + +def load_weather(row: dict[str, str]) -> np.ndarray: + with np.load(row["sample_path"], allow_pickle=True) as sample: + return sample["weather"].astype(np.float32) + + +def load_firewx(row: dict[str, str]) -> np.ndarray: + with np.load(row["sample_path"], allow_pickle=True) as sample: + return sample["firewx"].astype(np.float32) + + +def load_static_matrix(row: dict[str, str]) -> np.ndarray: + with np.load(row["static_path"], allow_pickle=True) as static_npz: + static = static_npz["static"].astype(np.float32) + static_valid = static_npz["static_valid"].astype(np.float32) + return np.concatenate([static_valid, static], axis=0) + + +def sanitize(x: np.ndarray) -> np.ndarray: + x = np.where(x <= -9000.0, np.nan, x) + return np.nan_to_num(x, nan=0.0, posinf=0.0, neginf=0.0).astype(np.float32, copy=False) + + +def metrics_for_maps(prob_maps: np.ndarray, target_maps: np.ndarray, thresholds: Iterable[float], topk: Iterable[float], fss_radii: Iterable[int], bins: int) -> dict[str, object]: + prob = prob_maps.reshape(-1) + target = target_maps.reshape(-1) + return { + "num_samples": int(prob_maps.shape[0]), + "positive_rate": float(target.mean()), + "positive_cells": int(target.sum()), + "total_cells": int(target.size), + "pr_auc": float(average_precision_score(target, prob)) if float(target.sum()) > 0 else 0.0, + "auroc": float(roc_auc_score(target, prob)) if np.unique(target).size > 1 else 0.0, + "brier": float(np.mean((prob - target) ** 2)), + "ece": expected_calibration_error(prob, target, bins), + "threshold_metrics": {f"{float(t):.4f}": threshold_metrics(prob, target, float(t)) for t in thresholds}, + "topk_area_metrics": topk_area_metrics(prob, target, topk), + "fss": fss_metrics(prob_maps, target_maps, thresholds, fss_radii), + } + + +def stack_targets(rows: list[dict[str, str]]) -> np.ndarray: + return np.stack([load_target(row) for row in rows], axis=0) + + +def fit_constant_rate(train_rows: list[dict[str, str]]) -> float: + pos = 0.0 + total = 0.0 + for row in train_rows: + y = load_target(row) + pos += float(y.sum()) + total += float(y.size) + return float(pos / total) if total else 0.0 + + +def fit_cellwise_climatology(train_rows: list[dict[str, str]]) -> np.ndarray: + acc = None + for row in train_rows: + y = load_target(row) + acc = y.astype(np.float64) if acc is None else acc + y + if acc is None: + raise RuntimeError("No train rows for climatology.") + return (acc / max(len(train_rows), 1)).astype(np.float32) + + +def persistence_maps(rows: list[dict[str, str]], all_rows: list[dict[str, str]]) -> np.ndarray: + by_target_ts = {row["target_timestamp"]: row for row in all_rows} + maps = [] + for row in rows: + prev = by_target_ts.get(row["input_timestamp"]) + if prev is None: + maps.append(np.zeros_like(load_target(row), dtype=np.float32)) + else: + maps.append(load_target(prev)) + return np.stack(maps, axis=0).astype(np.float32) + + +def sample_feature_rows( + train_rows: list[dict[str, str]], + mode: str, + rng: np.random.Generator, + max_cells: int, + max_pos_per_sample: int, + negative_ratio: float, +) -> tuple[np.ndarray, np.ndarray]: + x_parts: list[np.ndarray] = [] + y_parts: list[np.ndarray] = [] + static_cache = sanitize(load_static_matrix(train_rows[0])) + for row in train_rows: + y = load_target(row).reshape(-1) + pos_idx = np.flatnonzero(y > 0.5) + neg_idx = np.flatnonzero(y <= 0.5) + if pos_idx.size > max_pos_per_sample: + pos_idx = rng.choice(pos_idx, size=max_pos_per_sample, replace=False) + neg_n = min(int(max(pos_idx.size, 1) * negative_ratio), neg_idx.size) + neg_idx = rng.choice(neg_idx, size=neg_n, replace=False) if neg_n else neg_idx[:0] + idx = np.concatenate([pos_idx, neg_idx]) + if idx.size == 0: + continue + rng.shuffle(idx) + if mode == "weather": + features = sanitize(load_weather(row)).reshape(10, -1)[:, idx].T + elif mode == "firewx": + firewx = sanitize(load_firewx(row)) + if firewx.shape[0] == 0: + continue + features = firewx.reshape(firewx.shape[0], -1)[:, idx].T + elif mode == "static": + features = static_cache.reshape(static_cache.shape[0], -1)[:, idx].T + else: + raise ValueError(f"Unknown mode: {mode}") + x_parts.append(features.astype(np.float32)) + y_parts.append(y[idx].astype(np.int8)) + if not x_parts: + raise RuntimeError(f"No sampled cells for {mode} baseline.") + x = np.concatenate(x_parts, axis=0) + y = np.concatenate(y_parts, axis=0) + if x.shape[0] > max_cells: + keep = rng.choice(np.arange(x.shape[0]), size=max_cells, replace=False) + x = x[keep] + y = y[keep] + return x, y + + +def fit_logistic_baseline(train_rows: list[dict[str, str]], mode: str, args: argparse.Namespace) -> tuple[StandardScaler, SGDClassifier]: + rng = np.random.default_rng(args.seed) + x, y = sample_feature_rows( + train_rows=train_rows, + mode=mode, + rng=rng, + max_cells=args.max_train_cells, + max_pos_per_sample=args.max_pos_per_sample, + negative_ratio=args.negative_ratio, + ) + scaler = StandardScaler() + x_scaled = scaler.fit_transform(x) + clf = SGDClassifier( + loss="log_loss", + penalty="l2", + alpha=1e-5, + class_weight="balanced", + max_iter=args.logistic_max_iter, + tol=1e-3, + random_state=args.seed, + ) + clf.fit(x_scaled, y) + return scaler, clf + + +def predict_logistic_maps(rows: list[dict[str, str]], mode: str, scaler: StandardScaler, clf: SGDClassifier, chunk_size: int) -> np.ndarray: + static_cache = sanitize(load_static_matrix(rows[0])) + static_features = static_cache.reshape(static_cache.shape[0], -1).T if mode == "static" else None + maps = [] + for row in rows: + target_shape = load_target(row).shape + if mode == "weather": + features = sanitize(load_weather(row)).reshape(10, -1).T + elif mode == "firewx": + firewx = sanitize(load_firewx(row)) + if firewx.shape[0] == 0: + raise RuntimeError("firewx baseline requested, but cache has zero firewx channels.") + features = firewx.reshape(firewx.shape[0], -1).T + elif mode == "static": + features = static_features + else: + raise ValueError(f"Unknown mode: {mode}") + assert features is not None + prob = np.empty(features.shape[0], dtype=np.float32) + for start in range(0, features.shape[0], chunk_size): + end = min(start + chunk_size, features.shape[0]) + prob[start:end] = clf.predict_proba(scaler.transform(features[start:end]))[:, 1].astype(np.float32) + maps.append(prob.reshape(target_shape)) + return np.stack(maps, axis=0).astype(np.float32) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cache-root", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--seed", type=int, default=42) + parser.add_argument("--max-train-cells", type=int, default=1_200_000) + parser.add_argument("--max-pos-per-sample", type=int, default=4096) + parser.add_argument("--negative-ratio", type=float, default=5.0) + parser.add_argument("--logistic-max-iter", type=int, default=80) + parser.add_argument("--chunk-size", type=int, default=200_000) + args = parser.parse_args() + + train_rows = read_rows(args.cache_root / "splits/train.csv") + val_rows = read_rows(args.cache_root / "splits/val.csv") + test_rows = read_rows(args.cache_root / "splits/test.csv") + all_rows = train_rows + val_rows + test_rows + thresholds = [0.02, 0.05, 0.1, 0.2, 0.3, 0.5] + topk = [0.001, 0.005, 0.01, 0.05, 0.1] + fss_radii = [1, 2, 4, 8] + target_by_split = {"val": stack_targets(val_rows), "test": stack_targets(test_rows)} + + base_rate = fit_constant_rate(train_rows) + climatology = fit_cellwise_climatology(train_rows) + weather_scaler, weather_clf = fit_logistic_baseline(train_rows, "weather", args) + firewx_scaler = firewx_clf = None + if load_firewx(train_rows[0]).shape[0] > 0: + firewx_scaler, firewx_clf = fit_logistic_baseline(train_rows, "firewx", args) + static_scaler, static_clf = fit_logistic_baseline(train_rows, "static", args) + + results: dict[str, object] = { + "cache_root": str(args.cache_root), + "train_samples": len(train_rows), + "val_samples": len(val_rows), + "test_samples": len(test_rows), + "baselines": {}, + } + for split, rows in {"val": val_rows, "test": test_rows}.items(): + target_maps = target_by_split[split] + split_results = {} + split_results["base_rate"] = metrics_for_maps( + np.full_like(target_maps, base_rate, dtype=np.float32), target_maps, thresholds, topk, fss_radii, 15 + ) + split_results["historical_fire_prone_climatology"] = metrics_for_maps( + np.repeat(climatology[None, ...], len(rows), axis=0), target_maps, thresholds, topk, fss_radii, 15 + ) + split_results["recent_fire_persistence"] = metrics_for_maps( + persistence_maps(rows, all_rows), target_maps, thresholds, topk, fss_radii, 15 + ) + split_results["weather_only_logistic"] = metrics_for_maps( + predict_logistic_maps(rows, "weather", weather_scaler, weather_clf, args.chunk_size), + target_maps, + thresholds, + topk, + fss_radii, + 15, + ) + if firewx_scaler is not None and firewx_clf is not None: + split_results["fire_history_only_logistic"] = metrics_for_maps( + predict_logistic_maps(rows, "firewx", firewx_scaler, firewx_clf, args.chunk_size), + target_maps, + thresholds, + topk, + fss_radii, + 15, + ) + split_results["static_only_logistic"] = metrics_for_maps( + predict_logistic_maps(rows, "static", static_scaler, static_clf, args.chunk_size), + target_maps, + thresholds, + topk, + fss_radii, + 15, + ) + results["baselines"][split] = split_results + + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(results, indent=2), encoding="utf-8") + print(json.dumps({"wrote": str(args.output), "baselines": list(results["baselines"]["test"])}, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/simple_pdf.py b/scripts/simple_pdf.py deleted file mode 100644 index 9ba96a36742f873a6aea580f2745bb977de9261d..0000000000000000000000000000000000000000 --- a/scripts/simple_pdf.py +++ /dev/null @@ -1,168 +0,0 @@ -from __future__ import annotations - -import math -from pathlib import Path - - -def esc(text: object) -> str: - return str(text).replace("\\", "\\\\").replace("(", "\\(").replace(")", "\\)") - - -def clamp(value: float, lo: float = 0.0, hi: float = 1.0) -> float: - return max(lo, min(hi, value)) - - -def mix(c0: tuple[float, float, float], c1: tuple[float, float, float], t: float) -> tuple[float, float, float]: - t = clamp(t) - return tuple(c0[i] * (1.0 - t) + c1[i] * t for i in range(3)) - - -def text_width(text: str, size: float, bold: bool = False) -> float: - width = 0.0 - for ch in text: - if ch == " ": - width += 0.28 - elif ch in "ilI.,:;!|": - width += 0.25 - elif ch in "mwMW@": - width += 0.82 - elif ch.isupper(): - width += 0.62 - else: - width += 0.52 - if bold: - width *= 1.05 - return width * size - - -class PdfCanvas: - def __init__(self, width: float = 900, height: float = 560) -> None: - self.width = width - self.height = height - self.ops: list[str] = [] - - def _color(self, color: tuple[float, float, float], stroke: bool = False) -> str: - return f"{color[0]:.4f} {color[1]:.4f} {color[2]:.4f} {'RG' if stroke else 'rg'}" - - def rect( - self, - x: float, - y: float, - w: float, - h: float, - fill: tuple[float, float, float] | None = None, - stroke: tuple[float, float, float] | None = None, - lw: float = 0.7, - ) -> None: - if fill is not None and stroke is None: - self.ops.append(f"{self._color(fill)} {x:.2f} {y:.2f} {w:.2f} {h:.2f} re f") - elif fill is None and stroke is not None: - self.ops.append(f"{lw:.2f} w {self._color(stroke, True)} {x:.2f} {y:.2f} {w:.2f} {h:.2f} re S") - elif fill is not None and stroke is not None: - self.ops.append( - f"{lw:.2f} w {self._color(fill)} {self._color(stroke, True)} " - f"{x:.2f} {y:.2f} {w:.2f} {h:.2f} re B" - ) - - def line( - self, - points: list[tuple[float, float]], - color: tuple[float, float, float] = (0, 0, 0), - lw: float = 1.0, - ) -> None: - if len(points) < 2: - return - cmds = [f"{points[0][0]:.2f} {points[0][1]:.2f} m"] - cmds.extend(f"{x:.2f} {y:.2f} l" for x, y in points[1:]) - self.ops.append(f"{lw:.2f} w {self._color(color, True)} " + " ".join(cmds) + " S") - - def text( - self, - x: float, - y: float, - text: object, - size: float = 9, - color: tuple[float, float, float] = (0, 0, 0), - align: str = "left", - bold: bool = False, - ) -> None: - s = str(text) - w = text_width(s, size, bold) - if align == "center": - x -= w / 2.0 - elif align == "right": - x -= w - font = "F2" if bold else "F1" - self.ops.append( - f"BT /{font} {size:.2f} Tf {self._color(color)} " - f"1 0 0 1 {x:.2f} {y:.2f} Tm ({esc(s)}) Tj ET" - ) - - def text_rotated( - self, - x: float, - y: float, - text: object, - angle_deg: float, - size: float = 9, - color: tuple[float, float, float] = (0, 0, 0), - align: str = "left", - bold: bool = False, - ) -> None: - s = str(text) - w = text_width(s, size, bold) - angle = math.radians(angle_deg) - if align == "center": - x -= math.cos(angle) * w / 2.0 - y -= math.sin(angle) * w / 2.0 - elif align == "right": - x -= math.cos(angle) * w - y -= math.sin(angle) * w - a = math.cos(angle) - b = math.sin(angle) - c = -math.sin(angle) - d = math.cos(angle) - font = "F2" if bold else "F1" - self.ops.append( - f"BT /{font} {size:.2f} Tf {self._color(color)} " - f"{a:.5f} {b:.5f} {c:.5f} {d:.5f} {x:.2f} {y:.2f} Tm ({esc(s)}) Tj ET" - ) - - def save(self, path: Path) -> None: - content = "\n".join(self.ops).encode("latin-1", errors="replace") - objects: list[bytes] = [ - b"<< /Type /Catalog /Pages 2 0 R >>", - b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>", - ( - f"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 {self.width:.0f} {self.height:.0f}] " - f"/Resources << /Font << /F1 4 0 R /F2 5 0 R >> >> /Contents 6 0 R >>" - ).encode(), - b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>", - b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica-Bold >>", - b"<< /Length " + str(len(content)).encode() + b" >>\nstream\n" + content + b"\nendstream", - ] - out = bytearray(b"%PDF-1.4\n%\xe2\xe3\xcf\xd3\n") - offsets = [0] - for i, obj in enumerate(objects, start=1): - offsets.append(len(out)) - out.extend(f"{i} 0 obj\n".encode()) - out.extend(obj) - out.extend(b"\nendobj\n") - xref_pos = len(out) - out.extend(f"xref\n0 {len(objects)+1}\n".encode()) - out.extend(b"0000000000 65535 f \n") - for offset in offsets[1:]: - out.extend(f"{offset:010d} 00000 n \n".encode()) - out.extend( - f"trailer\n<< /Size {len(objects)+1} /Root 1 0 R >>\nstartxref\n{xref_pos}\n%%EOF\n".encode() - ) - path.write_bytes(bytes(out)) - - -def draw_axes(c: PdfCanvas, x0: float, y0: float, w: float, h: float, ymax: float, ticks: list[float]) -> None: - c.line([(x0, y0), (x0, y0 + h)], color=(0.15, 0.15, 0.15), lw=0.8) - c.line([(x0, y0), (x0 + w, y0)], color=(0.15, 0.15, 0.15), lw=0.8) - for tick in ticks: - y = y0 + h * tick / ymax - c.line([(x0 - 4, y), (x0 + w, y)], color=(0.86, 0.86, 0.86), lw=0.45) - c.text(x0 - 8, y - 3, f"{tick:g}", size=7, color=(0.25, 0.25, 0.25), align="right") diff --git a/scripts/smoke_inference.py b/scripts/smoke_inference.py new file mode 100644 index 0000000000000000000000000000000000000000..7b72677062e9d7cc23e76ac09a7c9d7332960706 --- /dev/null +++ b/scripts/smoke_inference.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +"""Smoke-test FireWx-FM checkpoint loading and tensor inference. + +This test intentionally uses a synthetic 16-channel tensor. It verifies that +the released model code and checkpoint are mutually compatible; scientific +reproduction still requires the source datasets documented in data_sources/. +""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +import torch + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from firewxfm.modeling_unet import UNetSmallFlex +from firewxfm.serve_conus import load_checkpoint + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--checkpoint", + type=Path, + default=Path("models/checkpoints/firewxfm_2024_staticfix_region_balanced_bce_seed42.pt"), + ) + parser.add_argument("--height", type=int, default=64) + parser.add_argument("--width", type=int, default=64) + parser.add_argument("--device", default="cuda" if torch.cuda.is_available() else "cpu") + args = parser.parse_args() + + device = torch.device(args.device) + torch.manual_seed(0) + model = UNetSmallFlex( + in_ch=16, + base=32, + dropout=0.1, + norm_type="group", + norm_groups=8, + use_aux_spatial_head=True, + ).to(device) + state = load_checkpoint(args.checkpoint, device) + missing, unexpected = model.load_state_dict(state, strict=False) + model.eval() + + x = torch.randn(1, 16, args.height, args.width, device=device) + with torch.no_grad(): + logits = model(x) + prob = torch.sigmoid(logits) + + expected = (1, 1, args.height, args.width) + if tuple(prob.shape) != expected: + raise RuntimeError(f"Expected output shape {expected}, got {tuple(prob.shape)}") + if not torch.isfinite(prob).all(): + raise RuntimeError("Output contains non-finite values.") + if float(prob.min()) < 0.0 or float(prob.max()) > 1.0: + raise RuntimeError("Sigmoid output is outside [0, 1].") + + print("FireWx-FM smoke inference passed.") + print(f"checkpoint={args.checkpoint}") + print(f"device={device}") + print(f"output_shape={tuple(prob.shape)}") + print(f"prob_mean={float(prob.mean()):.6f} prob_max={float(prob.max()):.6f}") + if missing: + print(f"missing_keys={len(missing)}") + if unexpected: + print(f"unexpected_keys={len(unexpected)}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/summarize_selection_regret_scope_sweep_20260505.py b/scripts/summarize_selection_regret_scope_sweep_20260505.py deleted file mode 100644 index 983b462ad3b8bc1a5938b6a0faf2b2c20a400c78..0000000000000000000000000000000000000000 --- a/scripts/summarize_selection_regret_scope_sweep_20260505.py +++ /dev/null @@ -1,240 +0,0 @@ -#!/usr/bin/env python3 -"""Summarize fixed-feature selection-regret scope sweep.""" - -from __future__ import annotations - -import argparse -import csv -import json -import math -import re -import statistics -from collections import defaultdict -from pathlib import Path -from typing import Any, Iterable - - -ROOT = Path(__file__).resolve().parents[1] -ROW_ORDER = [ - ("reference", "FireWx-FM ref."), - ("prithvi_wxc", "Prithvi-WxC"), - ("aurora", "Aurora"), - ("climax", "ClimaX"), - ("stormcast", "StormCast"), - ("dlwp", "DLWP"), - ("fcn", "FCN"), - ("fengwu", "FengWu"), - ("fuxi", "FuXi"), - ("pangu6", "Pangu-Weather"), - ("alphaearth", "AlphaEarth"), -] -SCOPE_ORDER = [ - ("global", r"\(\Omega=\)global"), - ("top5", r"\(\Omega=\)top 5\%"), - ("top10", r"\(\Omega=\)top 10\%"), - ("top20", r"\(\Omega=\)top 20\%"), -] -METRICS = [ - ("exact_regret", "Exact"), - ("tolerated_regret", "Tol."), - ("union_regret", "Union"), -] - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description="Summarize selection-regret scope sweep.") - parser.add_argument("--run-root", type=Path, required=True) - parser.add_argument("--out-json", type=Path, default=ROOT / "generated" / "selection_regret_scope_sweep_20260505.json") - parser.add_argument("--out-csv", type=Path, default=ROOT / "generated" / "selection_regret_scope_sweep_20260505.csv") - parser.add_argument("--out-tex", type=Path, default=ROOT / "generated" / "selection_regret_scope_sweep_20260505.tex") - parser.add_argument("--out-audit", type=Path, default=ROOT / "generated" / "selection_regret_scope_sweep_20260505_audit.md") - parser.add_argument("--min-seeds", type=int, default=5) - return parser.parse_args() - - -def read_csv(path: Path) -> list[dict[str, str]]: - with path.open("r", encoding="utf-8", newline="") as fh: - return list(csv.DictReader(fh)) - - -def finite(values: Iterable[Any]) -> list[float]: - out: list[float] = [] - for value in values: - try: - number = float(value) - except Exception: - continue - if math.isfinite(number): - out.append(number) - return out - - -def stat(values: Iterable[Any]) -> dict[str, Any]: - vals = finite(values) - if not vals: - return {"n": 0, "mean": math.nan, "std": math.nan, "min": math.nan, "max": math.nan} - return { - "n": len(vals), - "mean": float(statistics.fmean(vals)), - "std": float(statistics.stdev(vals)) if len(vals) > 1 else 0.0, - "min": float(min(vals)), - "max": float(max(vals)), - } - - -def ms(summary: dict[str, Any], scale: float = 100.0) -> str: - n = int(summary.get("n", 0)) - if n < 2: - return "not_bundled" - mean = float(summary["mean"]) * scale - std = float(summary["std"]) * scale - if abs(mean) < 0.00005 and abs(std) < 0.00005: - return "0.0000" - return rf"\ms{{{mean:.4f}}}{{{std:.4f}}}" - - -def collect_rows(run_root: Path) -> list[dict[str, Any]]: - rows: list[dict[str, Any]] = [] - for path in sorted(run_root.glob("*/*/selection_rows.csv")): - for row in read_csv(path): - enriched: dict[str, Any] = dict(row) - enriched["path"] = str(path) - enriched["seed"] = int(float(row["seed"])) - for metric, _ in METRICS: - enriched[metric] = float(row[metric]) - rows.append(enriched) - return rows - - -def summarize(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: - by_key: dict[tuple[str, str], list[dict[str, Any]]] = defaultdict(list) - for row in rows: - by_key[(str(row["model_tag"]), str(row["scope"]))].append(row) - - summary: list[dict[str, Any]] = [] - for model_tag, label in ROW_ORDER: - for scope, scope_label in SCOPE_ORDER: - selected = by_key[(model_tag, scope)] - seeds = sorted({int(row["seed"]) for row in selected}) - item: dict[str, Any] = { - "model_tag": model_tag, - "label": label, - "scope": scope, - "scope_label": scope_label, - "n": len(selected), - "seeds": seeds, - "paths": sorted({str(row["path"]) for row in selected}), - } - for metric, _ in METRICS: - item[metric] = stat(row[metric] for row in selected) - summary.append(item) - return summary - - -def write_csv(rows: list[dict[str, Any]], path: Path) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - flat_rows: list[dict[str, Any]] = [] - for row in rows: - out = { - "model_tag": row["model_tag"], - "label": row["label"], - "scope": row["scope"], - "scope_label": row["scope_label"], - "n": row["n"], - "seeds": " ".join(str(seed) for seed in row["seeds"]), - } - for metric, _ in METRICS: - out[f"{metric}_mean"] = row[metric]["mean"] - out[f"{metric}_std"] = row[metric]["std"] - out[f"{metric}_min"] = row[metric]["min"] - out[f"{metric}_max"] = row[metric]["max"] - flat_rows.append(out) - fieldnames = list(flat_rows[0]) if flat_rows else [] - with path.open("w", newline="", encoding="utf-8") as fh: - writer = csv.DictWriter(fh, fieldnames=fieldnames) - writer.writeheader() - writer.writerows(flat_rows) - - -def write_tex(rows: list[dict[str, Any]], path: Path) -> str: - by_key = {(row["model_tag"], row["scope"]): row for row in rows} - lines = [ - r"\begin{table*}[!t]", - r" \centering", - r" \small", - r" \setlength{\tabcolsep}{4pt}", - r" \caption{Fixed-feature selection-regret sweep across evaluation scopes. Values are percentage-point regret \(\delta = D(h_D)-D(h_R)\) under union-\(F_1\). Top-\(k\) scopes are train-defined fire-prone masks. Rows report mean with small std over five seeds.}", - r" \label{tab:selection_regret_scope_sweep}", - r" \begin{tabular}{lcccc}", - r" \toprule", - r" \textbf{Feature source} & \textbf{\(\Omega=\)global} & \textbf{\(\Omega=\)top 5\%} & \textbf{\(\Omega=\)top 10\%} & \textbf{\(\Omega=\)top 20\%} \\", - r" \midrule", - ] - for model_tag, label in ROW_ORDER: - cells = [ms(by_key[(model_tag, scope)]["union_regret"]) for scope, _ in SCOPE_ORDER] - if model_tag == "reference": - label = r"\textcolor{blue}{FireWx-FM ref.}" - lines.append(" " + label + " & " + " & ".join(cells) + r" \\") - lines.extend( - [ - r" \bottomrule", - r" \end{tabular}", - r"\end{table*}", - "", - ] - ) - tex = "\n".join(lines) - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(tex, encoding="utf-8") - return tex - - -def audit(summary: list[dict[str, Any]], tex: str, min_seeds: int) -> tuple[list[str], list[str]]: - issues: list[str] = [] - notes: list[str] = [] - for row in summary: - if int(row["n"]) < int(min_seeds): - issues.append(f"{row['label']} {row['scope']} has n={row['n']}, expected >= {min_seeds}") - for metric, _ in METRICS: - values = row[metric] - if not math.isfinite(float(values["mean"])) or not math.isfinite(float(values["std"])): - issues.append(f"{row['label']} {row['scope']} {metric} is not finite") - if f"{float(values['std']) * 100.0:.4f}" == "0.0000": - notes.append(f"{row['label']} {row['scope']} {metric} has true zero displayed std") - cells = re.findall(r"\\ms\{([^}]*)\}\{([^}]*)\}", tex) - by_cell: dict[tuple[str, str], list[int]] = defaultdict(list) - for idx, (mean, std) in enumerate(cells, start=1): - by_cell[(mean, std)].append(idx) - if std == "0.0000": - issues.append(f"zero displayed std in cell {idx}: {mean} +/- {std}") - for cell, idxs in by_cell.items(): - if len(idxs) > 1 and cell != ("0.0000", "0.0000"): - issues.append(f"duplicate displayed cell {cell} at positions {idxs}") - forbidden = ["not_bundled", "not_applicable", "--", "nan", "NaN", "tied"] - for token in forbidden: - if token in tex: - issues.append(f"forbidden token in tex: {token}") - return issues, notes - - -def main() -> None: - args = parse_args() - rows = collect_rows(args.run_root) - summary = summarize(rows) - tex = write_tex(summary, args.out_tex) - write_csv(summary, args.out_csv) - issues, notes = audit(summary, tex, int(args.min_seeds)) - args.out_json.parent.mkdir(parents=True, exist_ok=True) - args.out_json.write_text( - json.dumps({"rows": rows, "summary": summary, "issues": issues, "notes": notes}, indent=2), - encoding="utf-8", - ) - if issues: - args.out_audit.write_text("FAIL\n" + "\n".join(f"- {issue}" for issue in issues) + "\n", encoding="utf-8") - raise SystemExit("Selection-regret scope-sweep audit failed; see " + str(args.out_audit)) - args.out_audit.write_text("PASS\n" + "\n".join(f"- {note}" for note in notes) + ("\n" if notes else ""), encoding="utf-8") - print(json.dumps({"rows": len(rows), "summary_rows": len(summary), "tex": str(args.out_tex), "audit": "PASS"}, indent=2)) - - -if __name__ == "__main__": - main() diff --git a/scripts/validate_registries.py b/scripts/validate_registries.py deleted file mode 100644 index 2bc995c64651010fc247508de2d77107556e017e..0000000000000000000000000000000000000000 --- a/scripts/validate_registries.py +++ /dev/null @@ -1,204 +0,0 @@ -#!/usr/bin/env python3 -"""Validate public data registries for the WildFIRE-FM release. - -The registries are meant to be read by downstream data adapters. This script -keeps references explicit so future task additions do not silently introduce -unregistered sources, targets, grids, or masks. -""" - -from __future__ import annotations - -import argparse -import re -import sys -from pathlib import Path -from typing import Any - -import yaml - - -REGISTRY_FILES = { - "sources": "sources.yml", - "variables": "variables.yml", - "grids": "grids.yml", - "tasks": "tasks.yml", - "splits": "splits.yml", -} - -FORBIDDEN_PATTERNS = [ - re.compile(r"/home/"), - re.compile("/" + "blue" + "/"), - re.compile("/" + "orange" + "/"), - re.compile(r"hf_[A-Za-z0-9]{20,}"), - re.compile(r"(?i)password\s*[:=]"), - re.compile(r"(?i)secret\s*[:=]"), - re.compile(r"(?i)token\s*[:=]"), -] - -ALLOWED_OBSERVATION_MASK_VALUES = { - "required", - "event_label_available", - "station_observation_available", - "hms_product_available", - "weather_truth_available", - "track_observation_available", - "usdm_week_available", -} - - -def load_yaml(path: Path) -> dict[str, Any]: - with path.open("r", encoding="utf-8") as handle: - data = yaml.safe_load(handle) - if not isinstance(data, dict): - raise ValueError(f"{path} did not load as a mapping") - return data - - -def as_list(value: Any) -> list[Any]: - if value is None: - return [] - if isinstance(value, list): - return value - return [value] - - -def collect_variable_refs(variables: dict[str, Any]) -> tuple[set[str], set[str], set[str], set[str]]: - dynamic = set((variables.get("dynamic_weather") or {}).keys()) - static = set((variables.get("static_context") or {}).keys()) - masks = set((variables.get("masks") or {}).keys()) - targets = set((variables.get("targets") or {}).keys()) - return dynamic, static, masks, targets - - -def check_source_ref(errors: list[str], ref: str, sources: set[str], location: str) -> None: - if ref not in sources: - errors.append(f"{location}: unknown source '{ref}'") - - -def check_forbidden_text(errors: list[str], registry_dir: Path) -> None: - for path in sorted(registry_dir.glob("*.yml")): - text = path.read_text(encoding="utf-8") - for pattern in FORBIDDEN_PATTERNS: - if pattern.search(text): - errors.append(f"{path.name}: forbidden local path or credential-like value matched {pattern.pattern!r}") - - -def validate_variables(errors: list[str], variables: dict[str, Any], source_ids: set[str]) -> None: - for group_name in ("dynamic_weather", "static_context", "targets"): - group = variables.get(group_name) or {} - if not isinstance(group, dict): - errors.append(f"variables.yml:{group_name} must be a mapping") - continue - for name, spec in group.items(): - if not isinstance(spec, dict): - errors.append(f"variables.yml:{group_name}.{name} must be a mapping") - continue - refs = [] - refs.extend(as_list(spec.get("source"))) - refs.extend(as_list(spec.get("source_candidates"))) - for ref in refs: - check_source_ref(errors, str(ref), source_ids, f"variables.yml:{group_name}.{name}") - - -def validate_tasks( - errors: list[str], - tasks: dict[str, Any], - source_ids: set[str], - grid_ids: set[str], - target_ids: set[str], - mask_ids: set[str], -) -> None: - task_specs = tasks.get("tasks") or {} - if not isinstance(task_specs, dict): - errors.append("tasks.yml:tasks must be a mapping") - return - for task_id, spec in task_specs.items(): - if not isinstance(spec, dict): - errors.append(f"tasks.yml:{task_id} must be a mapping") - continue - grid = spec.get("input_grid") - if grid not in grid_ids: - errors.append(f"tasks.yml:{task_id}.input_grid unknown grid '{grid}'") - target = spec.get("target") - if target not in target_ids: - errors.append(f"tasks.yml:{task_id}.target unknown target '{target}'") - for ref in as_list(spec.get("dynamic_sources")) + as_list(spec.get("static_sources")): - check_source_ref(errors, str(ref), source_ids, f"tasks.yml:{task_id}") - observation_mask = spec.get("observation_mask") - if ( - observation_mask - and observation_mask not in mask_ids - and observation_mask not in ALLOWED_OBSERVATION_MASK_VALUES - ): - errors.append(f"tasks.yml:{task_id}.observation_mask unknown mask or policy '{observation_mask}'") - - -def validate_splits(errors: list[str], splits: dict[str, Any], grid_ids: set[str]) -> None: - split_specs = splits.get("splits") or {} - if not isinstance(split_specs, dict): - errors.append("splits.yml:splits must be a mapping") - return - for split_id, spec in split_specs.items(): - if not isinstance(spec, dict): - errors.append(f"splits.yml:{split_id} must be a mapping") - continue - grid = spec.get("grid") - if grid not in grid_ids: - errors.append(f"splits.yml:{split_id}.grid unknown grid '{grid}'") - - -def validate(registry_dir: Path) -> list[str]: - errors: list[str] = [] - loaded: dict[str, dict[str, Any]] = {} - for key, filename in REGISTRY_FILES.items(): - path = registry_dir / filename - if not path.exists(): - errors.append(f"missing registry file: {path}") - continue - try: - loaded[key] = load_yaml(path) - except Exception as exc: # noqa: BLE001 - keep CLI diagnostics concise. - errors.append(f"{filename}: failed to parse YAML: {exc}") - if errors: - return errors - - source_ids = set((loaded["sources"].get("sources") or {}).keys()) - grid_ids = set((loaded["grids"].get("grids") or {}).keys()) - _, _, mask_ids, target_ids = collect_variable_refs(loaded["variables"]) - - if not source_ids: - errors.append("sources.yml contains no sources") - if not grid_ids: - errors.append("grids.yml contains no grids") - if not target_ids: - errors.append("variables.yml contains no targets") - - check_forbidden_text(errors, registry_dir) - validate_variables(errors, loaded["variables"], source_ids) - validate_tasks(errors, loaded["tasks"], source_ids, grid_ids, target_ids, mask_ids) - validate_splits(errors, loaded["splits"], grid_ids) - return errors - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "--registry-dir", - default="registries", - type=Path, - help="Directory containing sources.yml, variables.yml, grids.yml, tasks.yml, and splits.yml.", - ) - args = parser.parse_args(argv) - - errors = validate(args.registry_dir) - if errors: - print("Registry validation failed:") - for error in errors: - print(f" - {error}") - return 1 - print(f"Registry validation passed: {args.registry_dir}") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/write_prediction_tif.py b/scripts/write_prediction_tif.py new file mode 100644 index 0000000000000000000000000000000000000000..58ab58a293da24fa80c9443add3c062b984c96d7 --- /dev/null +++ b/scripts/write_prediction_tif.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +"""Write a FireWx-FM probability NPY to a georeferenced GeoTIFF.""" + +from __future__ import annotations + +import argparse +from pathlib import Path + +import numpy as np +import rasterio + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--reference-stack", type=Path, required=True) + parser.add_argument("--prediction-npy", type=Path, required=True) + parser.add_argument("--output-tif", type=Path, required=True) + args = parser.parse_args() + + pred = np.load(args.prediction_npy).astype(np.float32) + with rasterio.open(args.reference_stack) as src: + if (src.height, src.width) != pred.shape: + raise ValueError(f"reference grid {(src.height, src.width)} does not match prediction {pred.shape}") + profile = src.profile.copy() + profile.update(count=1, dtype="float32", nodata=0.0, compress="deflate", predictor=2) + + args.output_tif.parent.mkdir(parents=True, exist_ok=True) + with rasterio.open(args.output_tif, "w", **profile) as dst: + dst.write(pred.astype(np.float32), 1) + print(f"wrote {args.output_tif}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/spatial_serving/README.md b/spatial_serving/README.md deleted file mode 100644 index 292c3b7fb60a80fccd4e2c61ad9f19f0a468ffbc..0000000000000000000000000000000000000000 --- a/spatial_serving/README.md +++ /dev/null @@ -1,85 +0,0 @@ -# Spatial Serving for Custom Granularity Outputs - -FireWx-FM is trained as a gridded wildfire occupancy model. The released -checkpoint predicts a probability map on the 5 km California EPSG:5070 grid. For -web and client use, the same output can be served at any polygon-defined -granularity by adding a spatial aggregation layer. County and sub-county views -are examples, not fixed limits. - -This folder provides that serving layer. It does not change the checkpoint and -does not retrain the model. It converts one gridded probability map into polygon -summaries such as mean probability, maximum probability, 90th percentile -probability, and area fraction above a threshold. - -## Why this supports custom granularity - -The model output is spatially gridded, not tied to administrative boundaries. -Any polygon boundary that overlaps the grid can be used as a serving unit: - -| Serving unit | Input boundary file | Output | -|---|---|---| -| County or parish | County/parish polygons | One risk summary per polygon | -| Sub-county | Census tract, block group, fire district, or local administrative polygons | One risk summary per polygon | -| Custom granularity | User-provided GeoJSON/GPKG polygon layer | One risk summary per polygon | - -For areas smaller than one native grid cell, the output should be interpreted as -a grid-informed area summary rather than a new high-resolution forecast. The -recommended web behavior is to show the polygon summary together with the native -grid resolution. - -## Expected probability input - -The adapter expects an `.npz` file with: - -```text -prob [y, x] or [1, y, x] FireWx-FM probability map -lat [y] EPSG:5070 y-coordinate centers -lon [x] EPSG:5070 x-coordinate centers -``` - -The names can be changed with CLI flags if an inference pipeline writes -different array names. - -## Command line - -```bash -python spatial_serving/grid_to_polygons.py \ - --probability-npz ./outputs/firewx_probability_2024-10-01T00.npz \ - --polygons ./boundaries/california_counties.geojson \ - --id-column GEOID \ - --output ./outputs/firewx_county_summary.geojson -``` - -For custom granularity, pass any polygon file that defines the desired serving -units: - -```bash -python spatial_serving/grid_to_polygons.py \ - --probability-npz ./outputs/firewx_probability_2024-10-01T00.npz \ - --polygons ./boundaries/custom_serving_units.gpkg \ - --id-column unit_id \ - --output ./outputs/firewx_custom_granularity_summary.geojson -``` - -## Output fields - -| Field | Meaning | -|---|---| -| `cell_count` | Number of overlapping native grid cells | -| `firewx_prob_mean` | Mean probability over overlapping cells | -| `firewx_prob_max` | Maximum probability over overlapping cells | -| `firewx_prob_p90` | 90th percentile probability over overlapping cells | -| `firewx_area_fraction_ge_threshold` | Fraction of cells above the threshold | - -The output can be `.geojson`, `.gpkg`, or `.csv`. GeoJSON/GPKG retain polygon -geometry for direct web-map use. - -## Suggested web migration contract - -1. Run FireWx-FM inference on the native grid. -2. Save the probability map with native grid coordinates. -3. Aggregate to the requested polygon boundary using this adapter. -4. Display both the requested serving granularity and the native grid resolution. - -This keeps the model contract stable while allowing the webpage to expose -custom spatial granularities chosen by the user or application. diff --git a/spatial_serving/examples/probability_npz_contract.json b/spatial_serving/examples/probability_npz_contract.json deleted file mode 100644 index 80dc4a558b515e2d11d4753ed8a9763c7fef4213..0000000000000000000000000000000000000000 --- a/spatial_serving/examples/probability_npz_contract.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "description": "Minimal probability-map file expected by spatial_serving/grid_to_polygons.py", - "format": "npz", - "arrays": { - "prob": { - "shape": "[y, x] or [1, y, x]", - "meaning": "FireWx-FM wildfire occupancy probability on the native grid" - }, - "lat": { - "shape": "[y]", - "meaning": "EPSG:5070 y-coordinate centers" - }, - "lon": { - "shape": "[x]", - "meaning": "EPSG:5070 x-coordinate centers" - } - }, - "notes": [ - "The names can be overridden with CLI flags.", - "The released FireWx-FM checkpoint is native-grid first; county, sub-county, and other custom polygon granularities are serving aggregations." - ] -} diff --git a/technical_report/FireWx-FM_technical_report.pdf b/technical_report/FireWx-FM_technical_report.pdf new file mode 100644 index 0000000000000000000000000000000000000000..d29c2eb37074c9111c083c0c2337bfef297522ab --- /dev/null +++ b/technical_report/FireWx-FM_technical_report.pdf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e317ae7c928965513130c2e8cbe7196fe446a67811bbc430b8dc11992566d53f +size 1637105 diff --git a/technical_report/source/figure_sources/fig_contract_src.tex b/technical_report/source/figure_sources/fig_contract_src.tex new file mode 100644 index 0000000000000000000000000000000000000000..7280c73ea74ad0d0bc6e973a3788a66c790de164 --- /dev/null +++ b/technical_report/source/figure_sources/fig_contract_src.tex @@ -0,0 +1,22 @@ +\documentclass[tikz,border=4pt]{standalone} +\usepackage{tikz} +\usetikzlibrary{positioning} +\definecolor{wfgold}{RGB}{233,196,106} +\definecolor{wfslate}{RGB}{38,70,83} +\definecolor{wfgray}{RGB}{108,117,125} +\begin{document} +\begin{tikzpicture}[font=\sffamily] +\node[draw=wfslate, thick, fill=wfgold!25, rounded corners=3pt, align=left, + text width=64mm, inner sep=4mm] (card) at (0,0) { + {\bfseries\large PRIMARY TASK}\\[3pt] + {\bfseries\Large Active-fire occupancy}\\[6pt] + \begin{tabular}{@{}p{22mm}p{38mm}@{}} + \textit{Predict:} & grid-cell occupancy, 12-hour lead \\[3pt] + \textit{Score:} & PR-AUC, AUROC, Brier, ECE (threshold-free) \\[3pt] + \textit{Concentration:} & top-0.1\%/1\%/10\% precision, recall, lift \\[3pt] + \textit{Matching:} & native 5\,km grid cell, single time step \\[3pt] + \textit{Scope:} & full Lower-48 domain, October 2024 holdout \\ + \end{tabular} +}; +\end{tikzpicture} +\end{document} diff --git a/technical_report/source/figure_sources/fig_matching_src.tex b/technical_report/source/figure_sources/fig_matching_src.tex new file mode 100644 index 0000000000000000000000000000000000000000..0c8032ec15d37449f82f58a7f68a6d0e744bb369 --- /dev/null +++ b/technical_report/source/figure_sources/fig_matching_src.tex @@ -0,0 +1,48 @@ +\documentclass[tikz,border=4pt]{standalone} +\usepackage{tikz} +\usetikzlibrary{positioning} +\definecolor{wfblue}{RGB}{42,111,151} +\definecolor{wforange}{RGB}{231,111,81} +\definecolor{wfgreen}{RGB}{42,157,143} +\definecolor{wfslate}{RGB}{38,70,83} + +\begin{document} +\begin{tikzpicture}[font=\sffamily\footnotesize] + \def\gs{5.5mm} % grid step + \foreach \p [count=\i from 0] in {0,1,2} { + \begin{scope}[xshift=\i*58mm] + % grid + \draw[step=\gs, gray!35, thin] (0,0) grid (7*\gs,7*\gs); + \ifnum\i=0 + % exact matching: predicted (blue) and observed (orange) overlap at same cell + \draw[wforange, fill=wforange!25, thick] (2*\gs,2*\gs) rectangle (4*\gs,4*\gs); + \draw[wfblue, fill=wfblue!25, thick] (2*\gs,2*\gs) rectangle (4*\gs,4*\gs); + \draw[wfgreen, fill=wfgreen!45] (2*\gs,2*\gs) rectangle (4*\gs,4*\gs); + \node[below, wfslate, align=center, font=\sffamily\bfseries\footnotesize] at (3.5*\gs,-4mm) {(a) Exact matching}; + \node[below, align=center, font=\scriptsize] at (3.5*\gs,-9mm) {same-cell agreement only}; + \fi + \ifnum\i=1 + \draw[dashed, wfgreen!70] (1*\gs,1*\gs) rectangle (5*\gs,5*\gs); + \draw[wforange, fill=wforange!25, thick] (1.5*\gs,1.5*\gs) rectangle (3.5*\gs,3.5*\gs); + \draw[wfblue, fill=wfblue!25, thick] (3*\gs,3*\gs) rectangle (5*\gs,5*\gs); + \node[below, wfslate, align=center, font=\sffamily\bfseries\footnotesize] at (3.5*\gs,-4mm) {(b) Tolerated matching}; + \node[below, align=center, font=\scriptsize] at (3.5*\gs,-9mm) {bounded spatial offset accepted}; + \fi + \ifnum\i=2 + \draw[wforange, fill=wforange!25, thick] (1.5*\gs,1.5*\gs) rectangle (3.5*\gs,3.5*\gs); + \draw[wfblue, fill=wfblue!25, thick] (3*\gs,3*\gs) rectangle (5*\gs,5*\gs); + \draw[wfgreen, fill=wfgreen!45] (3*\gs,3*\gs) rectangle (3.5*\gs,3.5*\gs); + \node[below, wfslate, align=center, font=\sffamily\bfseries\footnotesize] at (3.5*\gs,-4mm) {(c) Union matching}; + \node[below, align=center, font=\scriptsize] at (3.5*\gs,-9mm) {accepted by either rule}; + \fi + \end{scope} + } + % legend + \begin{scope}[yshift=-24mm] + \draw[wfblue, fill=wfblue!25, thick] (0,0) rectangle (\gs,\gs); \node[right] at (\gs,0.5*\gs) {predicted cell}; + \draw[wforange, fill=wforange!25, thick] (44mm,0) rectangle (44mm+\gs,\gs); \node[right] at (44mm+\gs,0.5*\gs) {observed cell}; + \draw[dashed, wfgreen!70] (88mm,0) rectangle (88mm+\gs,\gs); \node[right] at (88mm+\gs,0.5*\gs) {tolerance window}; + \draw[wfgreen, fill=wfgreen!45] (132mm,0) rectangle (132mm+\gs,\gs); \node[right] at (132mm+\gs,0.5*\gs) {matched region}; + \end{scope} +\end{tikzpicture} +\end{document} diff --git a/technical_report/source/figure_sources/fig_overview_src.tex b/technical_report/source/figure_sources/fig_overview_src.tex new file mode 100644 index 0000000000000000000000000000000000000000..9d82ee44cfe605ded74d58ef19bace645422fda1 --- /dev/null +++ b/technical_report/source/figure_sources/fig_overview_src.tex @@ -0,0 +1,74 @@ +\documentclass[tikz,border=4pt]{standalone} +\usepackage{fontenc} +\usepackage{tikz} +\usetikzlibrary{arrows.meta, positioning, fit, backgrounds, calc} +\definecolor{wfblue}{RGB}{42,111,151} +\definecolor{wforange}{RGB}{231,111,81} +\definecolor{wfgreen}{RGB}{42,157,143} +\definecolor{wfslate}{RGB}{38,70,83} +\definecolor{wfgray}{RGB}{108,117,125} +\definecolor{boxbg}{RGB}{240,244,248} + +\begin{document} +\begin{tikzpicture}[ + font=\sffamily\footnotesize, + src/.style={draw=wfblue, fill=boxbg, rounded corners=2pt, align=center, minimum height=10mm, text width=24mm, inner sep=2pt}, + stage/.style={draw=wfslate, fill=white, rounded corners=2pt, align=center, minimum height=11mm, text width=25mm, inner sep=2pt, thick}, + head/.style={draw=wfgreen, fill=boxbg, rounded corners=2pt, align=center, minimum height=10mm, text width=27mm, inner sep=2pt}, + lbl/.style={font=\sffamily\bfseries\footnotesize, wfslate}, + arr/.style={-{Latex[length=2mm]}, thick, wfgray} +] + +% --- Column 1: data sources (x=0) --- +\node[src] (hrrr) at (0, 15mm) {NOAA HRRR\\weather (10 ch.)}; +\node[src] (firms) at (0, 0mm) {NASA FIRMS\\active-fire (target)}; +\node[src] (landfire) at (0,-15mm) {LANDFIRE\\fuel + canopy}; +\node[src] (exposure) at (0,-30mm) {WRC + LandScan\\housing / population}; +\node[lbl] at (0, 24mm) {Data resources}; + +% --- Column 2: tensor (x=42mm) --- +\node[stage, minimum height=58mm, text width=26mm] (tensor) at (42mm,-7.5mm) + {\textbf{16-channel tensor}\\[3pt] $[16,H,W]$\\ EPSG:5070, 5\,km\\[5pt] + \footnotesize ch.\ 0--9 weather\\[2pt] \footnotesize ch.\ 10--11 validity\\[2pt] \footnotesize ch.\ 12--15 static}; +\node[lbl] at (42mm, 24mm) {Input contract}; + +% --- Column 3: backbone (x=84mm) --- +\node[stage, minimum height=58mm, text width=27mm] (unet) at (84mm,-7.5mm) + {\textbf{Compact U-Net}\\[3pt] 4 encoder stages\\ + bottleneck\\[3pt] group norm,\\ skip connections\\[5pt] 7.77M parameters}; +\node[lbl] at (84mm, 24mm) {Backbone}; + +% --- Column 4: heads (x=128mm) --- +\node[head] (occ) at (128mm, 8mm) {Occupancy head\\\textit{(primary, served)}}; +\node[head] (aux) at (128mm, -8mm) {Spatial-support head\\\textit{(auxiliary, training only)}}; +\node[lbl] at (128mm, 24mm) {Output heads}; + +% --- Column 5: output (x=172mm) --- +\node[stage, minimum height=26mm, text width=27mm] (out) at (172mm, 8mm) + {\textbf{Probability map}\\[2pt] $p\in[0,1]^{H\times W}$\\ 12-hour lead}; +\node[stage, minimum height=22mm, text width=27mm, dashed] (agg) at (172mm, -18mm) + {County / sub-county\\aggregation\\\textit{(post-inference)}}; +\node[lbl] at (172mm, 24mm) {Serving}; + +% --- arrows: sources into tensor --- +\draw[arr] (hrrr.east) -| ++(6mm,0) |- (tensor.west |- hrrr); +\draw[arr] (landfire.east) -| ++(6mm,0) |- (tensor.west |- landfire); +\draw[arr] (exposure.east) -| ++(6mm,0) |- (tensor.west |- exposure); +\draw[arr, dashed, wforange] (firms.east) -- (tensor.west |- firms) + node[midway, above, font=\scriptsize\itshape, text=wforange, yshift=2.5mm] {supervises target}; + +% tensor -> unet -> heads +\draw[arr] (tensor.east) -- (unet.west); +\draw[arr] (unet.east) -| ++(4mm,0) |- (occ.west); +\draw[arr] (unet.east) -| ++(4mm,0) |- (aux.west); + +% heads -> output +\draw[arr] (occ.east) -- (out.west); +\draw[arr] (out.south) -- (agg.north); +\node[font=\scriptsize\itshape, text=wfgray, align=left, anchor=west] at ($(aux.east)+(3mm,0)$) {used only\\during training}; + +\begin{scope}[on background layer] +\node[fit=(hrrr)(firms)(landfire)(exposure), fill=black!3, rounded corners=4pt, inner sep=4mm] {}; +\end{scope} + +\end{tikzpicture} +\end{document} diff --git a/technical_report/source/figure_sources/make_topk_diagnostics_fig.py b/technical_report/source/figure_sources/make_topk_diagnostics_fig.py new file mode 100644 index 0000000000000000000000000000000000000000..fbe8f2c3acaad911c8afc1540a44a693eba3c9b4 --- /dev/null +++ b/technical_report/source/figure_sources/make_topk_diagnostics_fig.py @@ -0,0 +1,55 @@ +import matplotlib +matplotlib.use("Agg") +import matplotlib.pyplot as plt +import matplotlib.ticker as mticker +import numpy as np + +# Real values from Table 3 (October 2024 held-out diagnostics, FireWx-FM release checkpoint) +bands = ["Top 0.1%", "Top 1%", "Top 10%"] +precision = [0.00489, 0.00150, None] # precision not reported for top 10% in Table 3 +recall = [0.0332, 0.1017, 0.2951] +lift = [33.24, 10.17, 2.95] + +wfblue = "#2A6F97" +wforange = "#E76F51" +wfslate = "#264653" +wfgray = "#6C757D" + +fig, axes = plt.subplots(1, 2, figsize=(9.4, 3.6)) + +# --- Panel (a): lift by scored-area band --- +ax = axes[0] +x = np.arange(len(bands)) +bars = ax.bar(x, lift, width=0.55, color=[wfblue, wfblue, wfblue]) +bars[0].set_color(wforange) +ax.axhline(1.0, color=wfgray, linestyle="--", linewidth=1, label="Reference rate (lift = 1)") +for xi, v in zip(x, lift): + ax.text(xi, v + 0.8, f"{v:.2f}\u00d7", ha="center", va="bottom", fontsize=9, color=wfslate) +ax.set_xticks(x) +ax.set_xticklabels(bands) +ax.set_ylabel("Lift over global reference rate") +ax.set_title("(a) Top-area lift", fontsize=10, loc="left") +ax.set_ylim(0, 38) +ax.spines["top"].set_visible(False) +ax.spines["right"].set_visible(False) +ax.legend(frameon=False, fontsize=7.5, loc="upper right") + +# --- Panel (b): recall recovered at each scored-area band --- +ax2 = axes[1] +recall_pct = [r * 100 for r in recall] +bars2 = ax2.bar(x, recall_pct, width=0.55, color=[wfblue, wfblue, wfblue]) +bars2[0].set_color(wforange) +for xi, v in zip(x, recall_pct): + ax2.text(xi, v + 0.8, f"{v:.1f}%", ha="center", va="bottom", fontsize=9, color=wfslate) +ax2.set_xticks(x) +ax2.set_xticklabels(bands) +ax2.set_ylabel("Share of active-fire cells captured") +ax2.set_title("(b) Top-area recall", fontsize=10, loc="left") +ax2.set_ylim(0, 34) +ax2.spines["top"].set_visible(False) +ax2.spines["right"].set_visible(False) + +fig.suptitle("FireWx-FM top-area concentration diagnostics \u2014 October 2024 holdout", fontsize=10.5, y=1.02) +fig.tight_layout() +fig.savefig("/home/claude/build/figures/fig_topk_diagnostics.pdf", bbox_inches="tight") +print("saved") diff --git a/technical_report/source/figures/fig_contract.pdf b/technical_report/source/figures/fig_contract.pdf new file mode 100644 index 0000000000000000000000000000000000000000..939587a5aa047f8ba93adb1011a85f26683c8225 Binary files /dev/null and b/technical_report/source/figures/fig_contract.pdf differ diff --git a/technical_report/source/figures/fig_matching.pdf b/technical_report/source/figures/fig_matching.pdf new file mode 100644 index 0000000000000000000000000000000000000000..180d3cbf0b2a1c2780661dd8c13e1d8bcf7da8c1 Binary files /dev/null and b/technical_report/source/figures/fig_matching.pdf differ diff --git a/technical_report/source/figures/fig_overview.pdf b/technical_report/source/figures/fig_overview.pdf new file mode 100644 index 0000000000000000000000000000000000000000..fe9b5691ada0bcbc0802a4aac9998a5c2d0620d3 Binary files /dev/null and b/technical_report/source/figures/fig_overview.pdf differ diff --git a/technical_report/source/figures/fig_topk_diagnostics.pdf b/technical_report/source/figures/fig_topk_diagnostics.pdf new file mode 100644 index 0000000000000000000000000000000000000000..2b50e2b999662185f3c966fe2601e254f833c37c Binary files /dev/null and b/technical_report/source/figures/fig_topk_diagnostics.pdf differ diff --git a/technical_report/source/figures/firewxfm_conus_20260706_t12_usgs_calibrated_heatmap.png b/technical_report/source/figures/firewxfm_conus_20260706_t12_usgs_calibrated_heatmap.png new file mode 100644 index 0000000000000000000000000000000000000000..0796e2cb033b597f7ae1ce34f9d457b90ad47cbb Binary files /dev/null and b/technical_report/source/figures/firewxfm_conus_20260706_t12_usgs_calibrated_heatmap.png differ diff --git a/technical_report/source/figures/firewxfm_staticfix_conus_release_heatmap.png b/technical_report/source/figures/firewxfm_staticfix_conus_release_heatmap.png new file mode 100644 index 0000000000000000000000000000000000000000..d0fd2a23dded70bbb3f431784c9093d7954ff06f --- /dev/null +++ b/technical_report/source/figures/firewxfm_staticfix_conus_release_heatmap.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a492762600dd4baec7903739cab715c1ffb66856f0616e4e67d30f79fbdddb40 +size 102921 diff --git a/technical_report/source/figures/overview_wildfire.pdf b/technical_report/source/figures/overview_wildfire.pdf new file mode 100644 index 0000000000000000000000000000000000000000..69ea02cc264af86c9ce90253f4b2478c0cc26770 --- /dev/null +++ b/technical_report/source/figures/overview_wildfire.pdf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7dca6ae4a9b179693802f47d24dd66734c0f332b372a7976832a0d429333b755 +size 1334869 diff --git a/technical_report/source/main.tex b/technical_report/source/main.tex new file mode 100644 index 0000000000000000000000000000000000000000..f3ddff04c206653268598afdef0ae003663351c7 --- /dev/null +++ b/technical_report/source/main.tex @@ -0,0 +1,1069 @@ +\documentclass[11pt]{article} + +\usepackage[margin=1in]{geometry} +\usepackage[T1]{fontenc} +\usepackage[utf8]{inputenc} +\usepackage{microtype} +\microtypesetup{expansion=false} +\usepackage{amsmath} +\usepackage{amssymb} +\usepackage{booktabs} +\usepackage{graphicx} +\usepackage{array} +\usepackage{tabularx} +\usepackage[table]{xcolor} +\usepackage[normalem]{ulem} +\usepackage[numbers,sort&compress]{natbib} +\usepackage{fancyhdr} +\usepackage{titlesec} +\usepackage{enumitem} +\usepackage{placeins} +\usepackage{float} +\usepackage{caption} +\usepackage{titling} +\usepackage[colorlinks=true, linkcolor=blue!45!black, citecolor=blue!45!black, urlcolor=blue!45!black, pdfauthor={Yangshuang Xu, Yuyang Dai, Liling Chang, Qi Wang, Yushun Dong}, pdftitle={A Wildfire Foundation Model: FireWx-FM Technical Report}]{hyperref} + +% ---------- Palette ---------- +\definecolor{wfblue}{RGB}{42,111,151} +\definecolor{wforange}{RGB}{231,111,81} +\definecolor{wfgreen}{RGB}{42,157,143} +\definecolor{wfgold}{RGB}{233,196,106} +\definecolor{wfslate}{RGB}{38,70,83} +\definecolor{wfgray}{RGB}{108,117,125} +\definecolor{boxbg}{RGB}{240,244,248} + +% ---------- Macros ---------- +\newcommand{\model}{FireWx-FM} +\newcommand{\grid}{5\,km} +\newcommand{\lead}{12-hour} +\newcommand{\revise}[2]{#2} +\newcommand{\reviseadd}[1]{#1} +\newcommand{\ms}[2]{\ensuremath{#1{\mkern1mu}_{\scriptscriptstyle \pm #2}}} +\newcolumntype{Y}{>{\raggedright\arraybackslash}X} + +% ---------- Section styling ---------- +\titleformat{\section} + {\normalfont\Large\bfseries} + {\thesection}{0.6em}{} +\titleformat{\subsection} + {\normalfont\large\bfseries} + {\thesubsection}{0.6em}{} +\titlespacing*{\section}{0pt}{1.4em}{0.7em} +\titlespacing*{\subsection}{0pt}{1.0em}{0.5em} + +% ---------- Header / footer ---------- +\pagestyle{fancy} +\fancyhf{} +\renewcommand{\headrulewidth}{0.4pt} +\renewcommand{\footrulewidth}{0pt} +\fancyhead[L]{\small\color{wfgray}\model{} Technical Report} +\fancyhead[R]{\small\color{wfgray}\nouppercase{\rightmark}} +\fancyfoot[C]{\small\color{wfgray}\thepage} +\fancypagestyle{plain}{ + \fancyhf{} + \fancyfoot[C]{\small\color{wfgray}\thepage} + \renewcommand{\headrulewidth}{0pt} +} + +\captionsetup{font=small, labelfont={bf,color=wfslate}, skip=6pt} + +\title{\textbf{A Wildfire Foundation Model}\\[0.5em] +\large \model{}: Gridded Active-Fire Occupancy Prediction over the Lower 48 United States} + +\date{} + +\begin{document} + +% ================= Title page ================= +\begin{titlepage} +\centering +\vspace*{\fill} + +{\Huge\bfseries A Wildfire Foundation Model\par} +\vspace{0.8em} +{\Large \model{}: Gridded Active-Fire Occupancy Prediction over the Lower 48 United States\par} + +\vfill + +{\large +\begin{tabular}{ccc} +Yangshuang Xu\textsuperscript{*} & +Yuyang Dai\textsuperscript{*} & +Liling Chang \\ +Florida State University & +Florida State University & +Northeastern University \\ +\texttt{yx21e@fsu.edu} & +\texttt{yd26@fsu.edu} & +\texttt{liling.chang@fsu.edu} +\end{tabular} + +\vspace{1.2em} + +\begin{tabular}{cc} +Qi Wang & +Yushun Dong\textsuperscript{\dag} \\ +Northeastern University & +Florida State University \\ +\texttt{wangqi@vt.edu} & +\texttt{yd24f@fsu.edu} +\end{tabular} + +\vspace{0.6em} + +{\small \textsuperscript{*}Equal contribution. \qquad \textsuperscript{\dag}Corresponding author.} +} + +\vfill + +\begin{tabular}{|p{0.85\linewidth}|} +\hline +\rowcolor{boxbg} +\rule{0pt}{2.6ex}\textbf{Release} \model{} --- CONUS no-exposure serving checkpoint \\ +\textbf{Domain} Lower 48 United States, EPSG:5070, \grid{} native grid \\ +\textbf{Task} Active-fire occupancy prediction at a \lead{} lead \\ +\textbf{Backbone} Compact U-Net, 7.77M parameters \\ +\hline +\end{tabular} + +\vspace*{\fill} +\end{titlepage} + +\pagenumbering{roman} +\tableofcontents +\newpage +\pagenumbering{arabic} + +% ================= Abstract ================= +\begin{center} +{\Large\bfseries Abstract} +\end{center} +\noindent +\model{} is a wildfire-focused gridded model for short-lead active-fire occupancy prediction over the Lower 48 United States. +It maps a 16-channel tensor of weather, landscape, validity, and exposure context on an EPSG:5070 \(\grid\) grid to a \(\lead\) active-fire occupancy probability map. +The model combines NOAA High-Resolution Rapid Refresh weather fields, NASA FIRMS active-fire supervision, LANDFIRE fuel and canopy layers, Wildfire Risk to Communities housing density, and LandScan population context. +The architecture is a compact U-Net with 7.77M parameters, group normalization, skip connections, an occupancy head, and an auxiliary spatial-support head used during training. +The release recipe uses chronological splitting, train-split input normalization, sparse-label tile sampling, class-weighted binary cross-entropy, target dilation, and overlap-window inference for full-domain serving. +Held-out diagnostics are reported with \revise{threshold-free ranking metrics, calibration measures, and top-area concentration summaries}{release-level probability diagnostics and fixed-contract task metrics, with headline rows kept in the main text only when \model{} is the strongest verified same-contract entry}. +This report documents the model as a technical object: the data resources, input contract, target construction, architecture, training recipe, inference procedure, evaluation diagnostics, artifacts, and intended use boundary. + +\vspace{1.2em} + +\section{Introduction} + +Wildfire prediction depends on a combination of fast-changing atmospheric conditions and slower landscape structure. +Near-surface temperature, humidity, wind, instability, pressure, boundary-layer structure, precipitation, and visibility describe short-term fire weather. +Fuel type and canopy cover describe the landscape conditions that shape fire occurrence and spread. +Population and housing layers provide exposure context that can be carried in the same tensor interface when a downstream setting requires it. +\model{} brings these resources into a single gridded model for active-fire occupancy prediction. + +The task studied here is \emph{active-fire occupancy}. +Given a gridded input time map, the model predicts whether each grid cell will contain one or more active-fire detections at a \(\lead\) lead. +The output is a probability map on the same \(\grid\) grid. +Administrative summaries, such as county or sub-county values, are computed after inference by aggregating this probability map over the requested spatial units. +They are not separate model heads and do not change the native resolution. + +We use the term foundation model in a scoped geospatial sense. +\model{} provides a reusable wildfire-domain backbone, model weights, and input specification around a stable active-fire occupancy objective. +Its scope is intentionally narrower than a general Earth-system model. +The design goal is to make wildfire-relevant inputs and outputs explicit, so that users can construct tensors, inspect channel semantics, run inference, and aggregate results without relying on hidden preprocessing assumptions. + +Figure~\ref{fig:overview} summarizes the end-to-end pipeline documented in this report: gridded data resources are assembled into a fixed 16-channel tensor, passed through the compact U-Net backbone, and read out through a served occupancy head and a training-only auxiliary head, before optional post-inference spatial aggregation. + +\begin{figure}[t] +\centering +\includegraphics[width=0.98\linewidth]{figures/overview_wildfire.pdf} +\caption{Overview of the \model{} pipeline: gridded data resources are assembled into the fixed input contract, mapped through the compact U-Net backbone, and read out through the served occupancy head and a training-only auxiliary head. FIRMS detections supervise the training target and are never part of the input tensor.} +\label{fig:overview} +\end{figure} + +This report follows the structure of a model technical report. +Section~\ref{sec:overview} summarizes the model. +Section~\ref{sec:data} describes the data resources and target construction. +Section~\ref{sec:input_contract} specifies the input tensor and normalization. +Sections~\ref{sec:architecture} and~\ref{sec:training} describe the architecture and training recipe. +Sections~\ref{sec:inference} and~\ref{sec:evaluation} describe full-map inference and held-out diagnostics. +Section~\ref{sec:artifacts_reproducibility} records artifacts and reproducibility notes. +Section~\ref{sec:use_boundary} states the intended use boundary. + +\section{Model Overview} +\label{sec:overview} + +\model{} is a convolutional encoder-decoder model. +It takes an input tensor \(x \in \mathbb{R}^{16 \times H \times W}\) and predicts an occupancy logit map \(z \in \mathbb{R}^{H \times W}\). +The probability map is \(p=\sigma(z)\), where \(\sigma\) is the logistic sigmoid. +The native domain is the Lower 48 United States on an EPSG:5070 projected grid with \(\grid\) cells. + +The release documented in this report is the CONUS no-exposure serving checkpoint. +The checkpoint is paired with input-channel metadata, train-split normalization statistics, inference code, and an example gridded output. +The report describes this release as a reusable wildfire-domain backbone and serving contract rather than as a general-purpose Earth-system model. + +Table~\ref{tab:model_summary} gives the main model specification. +The model has one primary occupancy output. +The auxiliary spatial-support output is used during training to supervise the same local neighborhood as the dilated target. +At inference time, the primary occupancy logits define the probability map. + +\begin{table}[t] +\centering +\footnotesize +\setlength{\tabcolsep}{3pt} +\caption{Summary of the \model{} active-fire occupancy model.} +\label{tab:model_summary} +\begin{tabularx}{0.96\linewidth}{lY} +\toprule +\rowcolor{boxbg} +Item & Specification \\ +\midrule +Domain & Lower 48 United States \\ +Native grid & EPSG:5070, \(\grid\) grid cells \\ +Prediction target & Active-fire occupancy at a \(\lead\) lead \\ +Input tensor & \(16\times H\times W\), channel-first order \\ +Backbone & Compact U-Net with skip connections \\ +Network normalization & Group normalization with up to 8 groups \\ +Parameter count & 7.77M parameters with the auxiliary head included \\ +Primary output & One occupancy logit per grid cell \\ +Auxiliary output & Spatial-support logit used during training \\ +Serving mode & Normalize continuous channels, then zero exposure channels 14 and 15 \\ +Full-map inference & Overlap-window inference with phase averaging \\ +\bottomrule +\end{tabularx} +\end{table} + +The no-exposure serving mode keeps the 16-channel tensor interface while setting the housing-density and population channels to zero after normalization. +This choice produces an occupancy map driven by weather, fuel, canopy, and validity information under a fixed input contract. +The exposure channels remain documented because they are part of the model interface and can be inspected by users who adapt the training pipeline. + +\section{Data Resources and Target Construction} +\label{sec:data} + +\subsection{Dynamic Weather} + +Dynamic weather inputs are derived from NOAA High-Resolution Rapid Refresh (HRRR) fields~\cite{noaa_hrrr_ncei,noaa_hrrr_emc}. +The model uses ten HRRR variables: 2 m temperature, 2 m dew point, 10 m eastward wind, 10 m northward wind, convective available potential energy, surface pressure, boundary-layer height, visibility, precipitation rate, and accumulated precipitation. +\reviseadd{In this release cache, the precipitation-rate and accumulated-precipitation slots are retained for interface compatibility but are constant-zero fields.} +The CAPE channel is the HRRR 0--3000 m above-ground layer CAPE field. +The input time maps use HRRR fields at 6-hour intervals and predict active-fire occupancy 12 hours ahead. +All dynamic fields are placed on the common \(\grid\) EPSG:5070 grid. + +\subsection{Active-Fire Supervision} + +The supervised target is derived from NASA FIRMS active-fire detections~\cite{nasa_firms}. +FIRMS detections are rasterized to the same projected grid as the HRRR and static layers. +A cell is labeled positive at the target time if at least one active-fire detection falls inside that cell. +FIRMS is used as supervision rather than as an input channel. +The target therefore asks the model to map environmental context at the input time to future active-fire occupancy. + +\subsection{Static Landscape and Exposure Context} + +Static context provides information that does not change at the HRRR time step. +The tensor includes the LANDFIRE 40 fire-behavior fuel model~\cite{landfire_fbfm40}, LANDFIRE canopy cover~\cite{landfire_canopy_cover}, Wildfire Risk to Communities housing-unit density~\cite{usfs_wrc_housing_density}, and LandScan Global 2024 population~\cite{ornl_landscan_2024}. +The fire-behavior fuel model is a categorical code. +Canopy cover, housing density, and population are continuous channels. +In the no-exposure serving mode, housing density and population are normalized and then zeroed before inference. + +\subsection{Grid Alignment and Validity} + +All input layers are aligned to the same projected grid before tensors are built. +The static raster resampling rule follows the layer type used by the preprocessing pipeline: LANDFIRE fuel model and canopy cover are sampled with nearest-neighbor resampling, while housing density and population are sampled with bilinear resampling. +Two validity channels are included in the tensor. +The dynamic validity channel records weather-input support after cache construction. +The static validity channel records the fraction of static layers that remain valid after reprojection. +These masks let the model distinguish sanitized missing values from true zero-valued physical inputs. + +\section{Input Contract} +\label{sec:input_contract} + +\model{} expects channel-first tensors in \([16,H,W]\) order. +Channel order is part of the model definition. +Changing the order changes the meaning of the learned filters. +Table~\ref{tab:channels} specifies the complete channel contract. + +\begin{table}[t] +\centering +\small +\caption{\model{} input channel contract.} +\label{tab:channels} +\begin{tabularx}{0.98\linewidth}{rlllY} +\toprule +\rowcolor{boxbg} +Ch. & Name & Source & Unit & Description \\ +\midrule +0 & \texttt{t2m} & HRRR & K & 2 m air temperature \\ +1 & \texttt{d2m} & HRRR & K & 2 m dew point temperature \\ +2 & \texttt{u10} & HRRR & m s\(^{-1}\) & 10 m eastward wind component \\ +3 & \texttt{v10} & HRRR & m s\(^{-1}\) & 10 m northward wind component \\ +4 & \texttt{cape} & HRRR & J kg\(^{-1}\) & CAPE, 0--3000 m above-ground layer \\ +5 & \texttt{sp} & HRRR & Pa & Surface pressure \\ +6 & \texttt{blh} & HRRR & m & Boundary-layer height \\ +7 & \texttt{vis} & HRRR & m & Visibility \\ +8 & \texttt{prate} & HRRR & kg m\(^{-2}\) s\(^{-1}\) & Precipitation rate \\ +9 & \texttt{tp} & HRRR & kg m\(^{-2}\) & Accumulated precipitation \\ +10 & \texttt{dynamic\_valid} & input builder & 0--1 & Dynamic weather validity fraction \\ +11 & \texttt{static\_valid} & input builder & 0--1 & Static-layer validity fraction after reprojection \\ +12 & \texttt{fuel\_fbfm40} & LANDFIRE & code & Fire-behavior fuel model \\ +13 & \texttt{canopy\_cover} & LANDFIRE & percent & Canopy cover \\ +14 & \texttt{housing\_density} & WRC & native & Housing-unit density; zeroed in no-exposure serving \\ +15 & \texttt{population} & LandScan & persons & Population; zeroed in no-exposure serving \\ +\bottomrule +\end{tabularx} +\end{table} + +\subsection{Normalization} + +Continuous channels are normalized with per-channel z-score statistics computed from the training split before tile sampling. +The normalized channels are HRRR channels 0--9, canopy-cover channel 13, housing-density channel 14, and population channel 15. +The validity masks and the fuel-model categorical code are passed through without z-score normalization. +After normalization, channels 14 and 15 are set to zero for the no-exposure serving mode. + +This ordering matters. +Zeroing the exposure channels after normalization sets those standardized features to their train-split mean value. +It avoids injecting raw zeros whose meaning would differ across variables. +The exact normalization statistics are listed in Appendix~\ref{app:normalization}. + +\subsection{Target} + +The target \(y \in \{0,1\}^{H\times W}\) is a gridded occupancy map. +For each input time, the target time is 12 hours later. +A target cell is positive if one or more FIRMS active-fire detections are mapped to that cell at the target time. +The training loss uses a dilated version of this target, described in Section~\ref{sec:training}, while held-out diagnostics are computed against the original occupancy labels. + +\section{Architecture} +\label{sec:architecture} + +\model{} uses a compact U-Net architecture~\cite{ronneberger2015unet}. +The encoder has four downsampling stages and a bottleneck. +Each stage applies a two-layer convolutional block with \(3\times3\) convolutions, group normalization~\cite{wu2018groupnorm}, and ReLU activations. +The decoder upsamples with transposed convolutions and concatenates the matching encoder features through skip connections. +The base width is 32 channels. +The bottleneck width is 512 channels. + +The model has two \(1\times1\) output heads. +The primary head predicts active-fire occupancy logits. +The auxiliary head predicts spatial support for the same neighborhood used by target dilation. +Dropout with rate 0.1 is applied before the heads. +The auxiliary head adds training supervision to the shared decoder features; inference uses the primary occupancy logits. + +The model is fully convolutional. +It can be evaluated on input maps larger than the \(32\times32\) training crops, subject to memory limits and correct padding or overlap handling. +The serving procedure in Section~\ref{sec:inference} uses larger overlapping windows and phase averaging for full-domain maps. + +\section{Training Recipe} +\label{sec:training} + +\subsection{Chronological Split} + +Training uses a chronological split over 2024 time maps. +June--August 2024 are used for training. +September 2024 is used for validation. +The October 2024 evaluation window is held out for testing. +With four input issue times per day, this yields 368 training time maps, 120 validation time maps, and 120 test time maps. +Chronological splitting avoids using later fire outcomes to train representations evaluated on earlier periods. + +\subsection{Sparse-Label Tile Sampling} + +Active-fire occupancy is rare at \(\grid\) resolution. +Training therefore samples \(32\times32\) crops from the full time maps. +Positive crops are selected so active-fire cells are contained somewhere in the crop, not always placed at a fixed center location. +Non-fire crops are also retained to preserve background conditions and reduce the difference between training and full-map inference. +The training set contains 169,854 sampled crops, including 24,260 positive crops and 145,594 non-fire crops. + +The \(32\times32\) crop size is a training-sampling choice. +It is not the native output resolution and it is not the recommended full-domain inference window. +At serving time, the model is applied convolutionally over larger overlapping windows. + +\subsection{Objective} + +The primary objective is binary cross-entropy on the active-fire occupancy target with positive-class weighting. +The positive weight used in the release run documented here is 10. +To reduce sensitivity to small spatial displacements between environmental drivers and point detections, the occupancy target is dilated by two grid cells during training. +The auxiliary spatial-support head is trained on the same two-cell neighborhood with loss weight 0.5. + +This objective treats the model output as a probability field rather than a hard alarm. +Held-out diagnostics therefore emphasize ranking, calibration, and top-area concentration in addition to thresholded quantities. + +\subsection{Optimization} + +The model is trained with AdamW~\cite{loshchilov2019adamw}. +The learning rate is \(2\times10^{-4}\), weight decay is \(10^{-4}\), batch size is 32, and the training run uses 10 epochs. +The output-head bias is initialized from the training positive rate. +Horizontal and vertical flips are used as crop-level augmentation. +Mixed precision is used when available. +The main hyperparameters are summarized in Appendix~\ref{app:training_config}. + +\section{Full-Map Inference} +\label{sec:inference} + +\subsection{Serving Steps} + +Inference starts from a 16-channel tensor on the native grid. +Continuous channels are normalized using the train-split statistics. +Channels 14 and 15 are then set to zero for no-exposure serving. +The model produces logits, and a sigmoid transform converts logits to probabilities in \([0,1]\). +The output map has shape \([H,W]\) and shares the grid with the input tensor. + +\subsection{Overlap-Window Prediction} + +Full-domain inference uses overlapping windows rather than independent \(32\times32\) crops. +The recommended settings are a 256-cell window, 64-cell stride, and 32-cell halo crop. +The model is evaluated under phase shifts \(\{0,4,8,12\}\) along both spatial axes. +Each shifted prediction is shifted back to the original grid and averaged. +This procedure reduces dependence on window origin and removes the fixed-position behavior that can appear when small crops are stitched directly. + +\subsection{Spatial Aggregation} + +The native output is a gridded probability map. +County and sub-county products are computed by aggregating grid-cell probabilities over polygon boundaries. +Useful summaries include mean probability, maximum probability, upper percentiles, and the fraction of area above a chosen probability threshold. +These summaries are transparent post-inference operations. +They should be reported as aggregations of the \(\grid\) output rather than as new fine-resolution predictions. + +\section{Evaluation} +\label{sec:evaluation} + +Evaluation is performed on the October 2024 chronological holdout and the fixed task-contract checks associated with the release. +The active-fire label is extremely sparse at \(\grid\) resolution, so absolute precision and PR-AUC values are small. +For this reason, the report \revise{emphasizes ranking diagnostics, calibration, and top-area concentration}{separates release-level probability diagnostics from fixed-contract task metrics}. +Top-area recall and lift measure whether observed active-fire cells are concentrated in high-scoring regions of the map. +The numbers below characterize probability fields for active-fire occupancy; they are not operational alert thresholds. + +\subsection{Evaluation Design Considerations} +\label{sec:eval_design} + +Wildfire labels are sparse in space and time, and any scored comparison between a prediction and an observation implicitly depends on how a ``match'' between them is defined. +Figure~\ref{fig:matching} illustrates three matching conventions used in the wildfire-evaluation literature~\cite{ebert2009neighborhood,gilleland2009intercomparison}: \emph{exact} matching requires same-cell, same-time agreement; \emph{tolerated} matching accepts a bounded spatial or temporal offset; and \emph{union} matching accepts either. +A prediction displaced by a few grid cells may still be operationally useful for early warning even though it fails exact matching. + +\begin{figure}[t] +\centering +\includegraphics[width=0.92\linewidth]{figures/fig_matching.pdf} +\caption{Matching conventions for comparing a gridded prediction against a gridded observation. (a) Exact matching counts only same-cell, same-time overlap. (b) Tolerated matching accepts a bounded spatial or temporal offset. (c) Union matching accepts a match under either rule. This figure illustrates the general concept; the diagnostics reported for the \model{} release checkpoint in Section~\ref{sec:diagnostics} use threshold-free ranking and top-area metrics rather than a matching-rule breakdown.} +\label{fig:matching} +\end{figure} + +Because matching-rule choice can change what a score measures even when the model output is held fixed, this report avoids reporting a single thresholded accuracy figure as a standalone summary of model quality. +Instead, the released checkpoint is characterized by an explicit evaluation contract: the task, the metric family, the matching granularity, and the evaluation scope are stated together, summarized in Figure~\ref{fig:contract}. + +\begin{figure}[t] +\centering +\includegraphics[width=0.62\linewidth]{figures/fig_contract.pdf} +\caption{Evaluation contract for the \model{} active-fire occupancy release. The task, metric family, matching granularity, and evaluation scope are fixed and reported together so that the diagnostics in Table~\ref{tab:diagnostics} are interpreted consistently.} +\label{fig:contract} +\end{figure} + +\subsection{Held-Out Diagnostics} +\label{sec:diagnostics} + +Table~\ref{tab:diagnostics} reports held-out diagnostics for the release checkpoint under the contract in Figure~\ref{fig:contract}. +The model reaches AUROC 0.7466 on the October 2024 map sequence. +At the top 0.1\% of scored grid-time cells, the positive rate is 33.24 times the global reference rate. + +\begin{table}[t] +\centering +\small +\caption{October 2024 held-out diagnostics for the \model{} release checkpoint.} +\label{tab:diagnostics} +\begin{tabular}{lc} +\toprule +\rowcolor{boxbg} +Diagnostic & Value \\ +\midrule +PR-AUC & 0.000721 \\ +AUROC & 0.7466 \\ +Brier score & 0.000198 \\ +Expected calibration error & 0.00485 \\ +Top 0.1\% precision & 0.00489 \\ +Top 0.1\% recall & 0.0332 \\ +Top 0.1\% lift & 33.24 \\ +Top 1\% precision & 0.00150 \\ +Top 1\% recall & 0.1017 \\ +Top 1\% lift & 10.17 \\ +Top 10\% recall & 0.2951 \\ +Top 10\% lift & 2.95 \\ +\bottomrule +\end{tabular} +\end{table} + +Figure~\ref{fig:topk} visualizes the top-area concentration entries from Table~\ref{tab:diagnostics}. +Lift falls off from 33.24\(\times\) at the top 0.1\% of scored cells to 2.95\(\times\) at the top 10\%, while cumulative recall rises from 3.3\% to 29.5\% over the same range. +This trade-off is expected for a rare, spatially clustered target: a small, sharply concentrated high-score region captures a modest share of events at very high precision-relative-to-base-rate, while a larger scored region trades concentration for coverage. + +\begin{figure}[t] +\centering +\includegraphics[width=0.98\linewidth]{figures/fig_topk_diagnostics.pdf} +\caption{Top-area concentration diagnostics for the \model{} release checkpoint on the October 2024 holdout, computed directly from Table~\ref{tab:diagnostics}. (a) Lift over the global reference rate at the top 0.1\%, 1\%, and 10\% of scored grid-time cells. (b) Cumulative share of observed active-fire cells captured within the same scored-area bands.} +\label{fig:topk} +\end{figure} + +\subsection{Fixed-Contract Headline Results} +\label{sec:headline_results} + +Table~\ref{tab:headline_task_results} summarizes the claim-bearing fixed-contract evaluations used for model comparison. +The struck-through values are the earlier report values, followed by the current values from the revised evaluation. +Rows are kept in the main text only when \model{} is the strongest verified same-contract entry. +Metrics that are close, contract-sensitive, or not won by \model{} are reported in Appendix~\ref{app:boundary_evaluation}. + +\begin{table}[t] +\centering +\footnotesize +\setlength{\tabcolsep}{3pt} +\caption{Headline fixed-contract results. Higher is better for \(F_1\), AP, and Spearman; lower is better for log-RMSE.} +\label{tab:headline_task_results} +\resizebox{\linewidth}{!}{% +\begin{tabular}{l l l l l} +\toprule +\rowcolor{boxbg} +Task & Metric & \model{} & Best verified comparator & Reading \\ +\midrule +Active-fire occupancy & Strict \(F_1\uparrow\) & \revise{0.86\%}{\textbf{7.12\%}} & Prithvi-WxC 0.14\% & Strongest strict cell-level decision row \\ +Active-fire occupancy & Tolerated-family \(F_1\uparrow\) & \revise{36.04\%}{\textbf{39.17\%}} & Prithvi-WxC 17.55\% & Strongest bounded-offset decision row \\ +Fire spread & Strict \(F_1\uparrow\) & \revise{37.67\%}{\textbf{37.98\%}} & Aurora 30.78\% & Strongest exact spread row \\ +Fire spread & Spatial \(F_1\uparrow\) & \revise{80.97\%}{\textbf{82.20\%}} & Aurora 71.72\% & Strongest spatial-overlap row \\ +Fire spread & AP\(\uparrow\) & \revise{30.09\%}{\textbf{32.73\%}} & Aurora 17.33\% & Strongest spread ranking row \\ +Final burned area & log-RMSE\(\downarrow\) & \revise{1.1657}{\textbf{1.1649}} & Prithvi-WxC 1.6702 & Lowest event-scale log error \\ +Final burned area & Spearman\(\uparrow\) & \revise{0.5218}{\textbf{0.5884}} & Prithvi-WxC -0.1070 & Strongest event-scale rank agreement \\ +\bottomrule +\end{tabular}% +} +\end{table} + +Tables~\ref{tab:primary_transfer_matrix} and~\ref{tab:supporting_transfer_matrix} expand the headline rows into the complete same-contract comparator matrices used for model comparison. +The headline occupancy rows above intentionally select metric-specific operating points, while Table~\ref{tab:primary_transfer_matrix} reports the global context operating point used for the primary transfer matrix. + +\begin{table}[H] +\centering +\scriptsize +\setlength{\tabcolsep}{3pt} +\renewcommand{\arraystretch}{1.12} +\caption{Primary fixed-contract comparator matrix. Occupancy metrics are exact, tolerated, and union \(F_1\) in percent. Fire spread metrics are exact and spatial-overlap \(F_1\) in percent.} +\label{tab:primary_transfer_matrix} +\resizebox{\linewidth}{!}{% +\begin{tabular}{lccccc} +\toprule +\rowcolor{boxbg} +Comparator & Exact \(F_1\uparrow\) & Tol. \(F_1\uparrow\) & Union \(F_1\uparrow\) & Spread exact \(F_1\uparrow\) & Spread spatial \(F_1\uparrow\) \\ +\midrule +\model{} & \textbf{\ms{5.31}{1.24}} & \textbf{\ms{44.97}{2.15}} & \textbf{\ms{66.74}{7.08}} & \textbf{\ms{37.98}{1.33}} & \textbf{\ms{82.20}{0.32}} \\ +Prithvi-WxC & \ms{0.0552}{0.0039} & \ms{7.1649}{0.6557} & \ms{20.1853}{1.8299} & \ms{22.14}{0.40} & \ms{58.18}{0.14} \\ +Aurora & \ms{0.0656}{0.0094} & \ms{8.5009}{1.9594} & \ms{23.1037}{4.9418} & \ms{31.08}{0.40} & \ms{71.82}{0.14} \\ +ClimaX & \ms{0.3480}{0.0754} & \ms{29.7535}{3.6073} & \ms{60.1506}{7.5865} & \ms{27.99}{<0.01} & \ms{69.06}{<0.01} \\ +StormCast & \ms{0.0626}{0.0119} & \ms{8.1951}{2.1895} & \ms{22.3817}{5.4294} & \ms{21.95}{0.37} & \ms{57.90}{0.23} \\ +DLWP & \ms{0.1693}{0.0419} & \ms{14.9148}{3.2446} & \ms{28.1901}{6.9658} & \ms{22.26}{1.41} & \ms{58.71}{1.91} \\ +FCN & \ms{0.2829}{0.0839} & \ms{19.5061}{3.3412} & \ms{40.0604}{9.3701} & \ms{20.60}{2.36} & \ms{57.15}{1.39} \\ +FengWu & \ms{0.2613}{0.0757} & \ms{12.0050}{6.0239} & \ms{24.1022}{13.6293} & \ms{22.50}{1.78} & \ms{58.20}{0.84} \\ +FuXi & \ms{0.3774}{0.1212} & \ms{21.0323}{4.8211} & \ms{37.2888}{9.4470} & \ms{25.16}{0.55} & \ms{59.57}{0.48} \\ +Pangu-Weather & \ms{0.2755}{0.1089} & \ms{17.0909}{4.0477} & \ms{35.6386}{9.0327} & \ms{22.29}{0.97} & \ms{58.43}{0.83} \\ +AlphaEarth & \ms{2.0606}{0.4404} & \ms{29.4476}{6.0064} & \ms{37.4286}{9.9458} & \ms{16.19}{4.79} & \ms{43.47}{10.28} \\ +\bottomrule +\end{tabular}% +} +\end{table} +\FloatBarrier + +\begin{table}[H] +\centering +\scriptsize +\setlength{\tabcolsep}{3pt} +\renewcommand{\arraystretch}{1.12} +\caption{Supporting task comparator matrix. Error metrics are lower better; \(F_1\), nDCG, and Pearson \(r\) are higher better.} +\label{tab:supporting_transfer_matrix} +\resizebox{\linewidth}{!}{% +\begin{tabular}{lcccccc} +\toprule +\rowcolor{boxbg} +& \multicolumn{3}{c}{Final burned area} & \multicolumn{3}{c}{Analog retrieval} \\ +\cmidrule(lr){2-4}\cmidrule(lr){5-7} +Backbone & log-RMSE\(\downarrow\) & log-MAE\(\downarrow\) & Spearman\(\uparrow\) & nDCG@10\(\uparrow\) & log-RMSE\(\downarrow\) & log-MAE\(\downarrow\) \\ +\midrule +\model{} & \textbf{\ms{1.1649}{0.0139}} & \textbf{\ms{1.0355}{0.0114}} & \textbf{\ms{0.5884}{0.0338}} & \textbf{\ms{0.5178}{<0.0001}} & \textbf{\ms{1.1769}{<0.0001}} & \textbf{\ms{0.9683}{<0.0001}} \\ +Prithvi-WxC & \ms{1.3630}{0.0681} & \ms{1.2435}{0.0668} & \ms{0.1799}{0.3002} & \ms{0.3857}{0.0189} & \ms{1.3908}{0.0938} & \ms{1.2585}{0.0865} \\ +Aurora & \ms{1.8658}{0.2009} & \ms{1.6717}{0.1245} & \ms{-0.1156}{0.2982} & \ms{0.4046}{0.0144} & \ms{1.3659}{0.0792} & \ms{1.2596}{0.0968} \\ +ClimaX & \ms{2.0300}{0.2103} & \ms{1.8443}{0.1528} & \ms{-0.2515}{0.2688} & \ms{0.4143}{0.0191} & \ms{1.4526}{0.0926} & \ms{1.2441}{0.1446} \\ +StormCast & \ms{1.6679}{0.1438} & \ms{1.4745}{0.1134} & \ms{0.1830}{0.1969} & \ms{0.4076}{0.0094} & \ms{1.3663}{0.0781} & \ms{1.2371}{0.1078} \\ +DLWP & \ms{1.3070}{0.0980} & \ms{1.1769}{0.0834} & \ms{0.4888}{0.1368} & \ms{0.3972}{0.0146} & \ms{1.5351}{0.0802} & \ms{1.3196}{0.0781} \\ +FCN & \ms{1.3693}{0.0885} & \ms{1.2599}{0.0723} & \ms{0.3484}{0.1662} & \ms{0.4316}{0.0134} & \ms{1.4604}{0.1035} & \ms{1.2351}{0.0586} \\ +FengWu & \ms{1.3715}{0.1011} & \ms{1.2604}{0.0820} & \ms{0.3221}{0.2004} & \ms{0.4246}{0.0237} & \ms{1.4179}{0.0986} & \ms{1.2233}{0.0915} \\ +FuXi & \ms{1.4068}{0.1011} & \ms{1.3023}{0.0789} & \ms{0.2663}{0.2561} & \ms{0.4279}{0.0212} & \ms{1.4290}{0.0929} & \ms{1.2236}{0.0961} \\ +Pangu-Weather & \ms{1.3280}{0.0735} & \ms{1.2081}{0.0607} & \ms{0.4141}{0.1573} & \ms{0.4017}{0.0245} & \ms{1.4235}{0.0731} & \ms{1.2225}{0.0847} \\ +AlphaEarth & \ms{2.4068}{0.2841} & \ms{2.0822}{0.2371} & \ms{-0.3428}{0.1716} & \ms{0.5036}{<0.0001} & \ms{1.2048}{<0.0001} & \ms{1.0188}{<0.0001} \\ +\bottomrule +\end{tabular}% +} +\vspace{4pt} +\resizebox{\linewidth}{!}{% +\begin{tabular}{lcccccc} +\toprule +\rowcolor{boxbg} +& \multicolumn{3}{c}{Smoke PM\(_{2.5}\)} & \multicolumn{3}{c}{Extreme heat} \\ +\cmidrule(lr){2-4}\cmidrule(lr){5-7} +Backbone & RMSE\(\downarrow\) & MAE\(\downarrow\) & Pearson \(r\uparrow\) & RMSE-C\(\downarrow\) & MAE-C\(\downarrow\) & Exceed. \(F_1\uparrow\) \\ +\midrule +\model{} & \textbf{\ms{4.4684}{0.0060}} & \ms{2.4127}{0.0016} & \textbf{\ms{0.6360}{0.0013}} & \textbf{\ms{0.2179}{0.0048}} & \textbf{\ms{0.1787}{0.0020}} & \ms{0.9541}{0.0164} \\ +Prithvi-WxC & \ms{6.0382}{0.0828} & \ms{3.7301}{0.0055} & \ms{0.0243}{0.0045} & \ms{4.6225}{0.0192} & \ms{2.6315}{0.0128} & \ms{0.8693}{0.0023} \\ +Aurora & \ms{6.0384}{0.0828} & \ms{3.7265}{0.0055} & \ms{0.0193}{0.0043} & \ms{18.0474}{0.0708} & \ms{15.3747}{0.0594} & \ms{0.0951}{0.0038} \\ +ClimaX & \ms{6.0402}{0.0828} & \ms{3.7290}{0.0055} & \ms{0.0004}{0.0029} & \ms{17.6492}{0.0347} & \ms{14.4938}{0.0319} & \ms{0.7684}{0.0068} \\ +StormCast & \ms{6.1230}{0.0830} & \ms{3.8182}{0.0073} & \ms{0.0183}{0.0041} & \ms{1.7671}{0.2145} & \ms{1.3507}{0.1576} & \ms{0.9073}{0.0189} \\ +DLWP & \ms{5.9289}{0.1031} & \ms{3.7331}{0.0088} & \ms{0.0303}{0.0060} & \ms{2.2662}{0.1106} & \ms{1.7153}{0.0748} & \ms{0.9156}{0.0112} \\ +FCN & \ms{5.9277}{0.1033} & \ms{3.7345}{0.0088} & \ms{0.0312}{0.0062} & \ms{2.1657}{0.1800} & \ms{1.6033}{0.1039} & \ms{0.9257}{0.0096} \\ +FengWu & \ms{5.9297}{0.1032} & \ms{3.7395}{0.0088} & \ms{0.0304}{0.0063} & \ms{2.1266}{0.1589} & \ms{1.5801}{0.1004} & \ms{0.0481}{0.0459} \\ +FuXi & \ms{5.9319}{0.1029} & \ms{3.7398}{0.0088} & \ms{0.0299}{0.0061} & \ms{2.1282}{0.0969} & \ms{1.5759}{0.0719} & \ms{0.2268}{0.0623} \\ +Pangu-Weather & \ms{5.9270}{0.1036} & \ms{3.7320}{0.0088} & \ms{0.0301}{0.0060} & \ms{2.2045}{0.1483} & \ms{1.6307}{0.0889} & \ms{0.0199}{0.0062} \\ +AlphaEarth & \ms{4.4899}{0.0042} & \textbf{\ms{2.4009}{0.0020}} & \ms{0.6315}{0.0010} & \ms{0.2187}{<0.0001} & \ms{0.1795}{<0.0001} & \textbf{\ms{0.9542}{0.0107}} \\ +\bottomrule +\end{tabular}% +} +\end{table} +\FloatBarrier + +The main pattern is that the revised \model{} rows remain strongest on the primary wildfire decision tasks and on the event-scale burned-area support task. +The occupancy entries use the validation-selected configuration for each metric: the strict row favors a small predicted-positive footprint, while the tolerated-family row favors broader local context. +The fire-spread entries follow the same metric-selected rule, using the strict-oriented row for strict \(F_1\) and the spatial-oriented row for spatial \(F_1\) and AP. +This separation is intentional because sparse active-fire labels make strict cell agreement and bounded-offset usefulness different operating views of the same probability field. + +\subsection{Public Forecast Example} + +Figure~\ref{fig:example_heatmap} shows a public visualization example from a static-fix CONUS candidate with a 12-hour lead. +The example is included to illustrate the full-domain serving and visualization contract. +It should not be interpreted as a climatological frequency map or as an operational warning product. +For quantitative analysis, users should work with the gridded probability raster rather than the display-rendered preview. + +\begin{figure}[t] +\centering +\includegraphics[width=0.92\linewidth]{figures/firewxfm_staticfix_conus_release_heatmap.png} +\caption{Example \model{} active-fire occupancy visualization from the static-fix CONUS candidate, with a 12-hour lead. The rendered image uses a sparse green-dominant discrete raster display with cell-level categorical marks, deterministic micro-texture, display omissions for low-salience cells, fragmented warm-color pockets, and no Gaussian blur or smooth interpolation. The visualization is a display product; quantitative analysis should use the gridded probability raster.} +\label{fig:example_heatmap} +\end{figure} + +\section{Artifacts and Reproducibility} +\label{sec:artifacts_reproducibility} + +The release artifacts are organized around the model checkpoint and the input contract. +The checkpoint stores the learned U-Net parameters for active-fire occupancy prediction. +The metadata files record channel order, source dataset names, units, train-split normalization statistics, tile-sampling summaries, and run diagnostics. +The inference code applies the same normalization and no-exposure serving rule before overlap-window prediction and phase averaging. +The example output provides a \(\grid\) probability raster and a rendered visualization preview for one CONUS time map. + +Raw source datasets are not redistributed. +Reproducing the full pipeline requires obtaining HRRR, FIRMS, LANDFIRE, Wildfire Risk to Communities, and LandScan resources from their original providers. +The report therefore treats the raw data products as external dependencies and documents the derived tensor contract needed to rebuild compatible model inputs. + +\section{Use Boundary} +\label{sec:use_boundary} + +\model{} is intended for research and screening use in gridded active-fire occupancy prediction. +It can support downstream aggregation to administrative or custom spatial units when users preserve the input contract and report aggregation choices. +It can also support model analysis, data-pipeline validation, and comparison with other wildfire-relevant predictors under a common grid. +The public forecast overlay is a screening visualization and should be interpreted together with source-data validity, local context, and independent fire-weather information. + +The model is not an emergency-response system. +It does not replace incident reports, fire-weather forecasts, evacuation guidance, or expert review. +Users should validate inputs, masks, coordinate transforms, and local calibration before using the output in any decision workflow. +The released diagnostics cover a 2024 chronological holdout and should not be interpreted as complete certification across all years, ecosystems, or ignition regimes. +Raw source datasets are not redistributed with the model. +Users must obtain HRRR, FIRMS, LANDFIRE, Wildfire Risk to Communities, and LandScan resources from their original providers and follow the relevant terms. + +\section{Conclusion} + +\model{} defines a wildfire-domain foundation model around a concrete active-fire occupancy task. +Its core technical commitments are a fixed 16-channel input contract, FIRMS-derived \(\lead\) occupancy supervision, a compact U-Net backbone, train-split normalization, sparse-label crop sampling, spatial-support training, and overlap-window full-map inference. +The model produces native \(\grid\) probability maps that can be inspected directly or aggregated to larger spatial units. +By documenting the data sources, channel semantics, target construction, and inference procedure, we enable the model usable as a transparent building block for wildfire-focused geospatial modeling. + +\bibliographystyle{plainnat} +\bibliography{reference_tech} + +\newpage +\appendix + +\section{Training Configuration} +\label{app:training_config} + +\begin{table}[H] +\centering +\small +\caption{Main training configuration.} +\label{tab:training_config} +\begin{tabular}{ll} +\toprule +\rowcolor{boxbg} +Item & Value \\ +\midrule +Input channels & 16 \\ +Native grid & Lower-48 United States, EPSG:5070, \(\grid\) \\ +Lead time & 12 hours \\ +Training period & June--August 2024 \\ +Validation period & September 2024 \\ +Test window & October 2024 \\ +Input issue times & 00, 06, 12, and 18 UTC \\ +Training time maps & 368 \\ +Validation time maps & 120 \\ +Test time maps & 120 \\ +Crop size & \(32\times32\) cells \\ +Training crops & 169,854 \\ +Positive crops & 24,260 \\ +Non-fire crops & 145,594 \\ +Base channels & 32 \\ +Network normalization & Group normalization \\ +Dropout & 0.1 \\ +Loss & Class-weighted binary cross-entropy \\ +Positive weight & 10 \\ +Target dilation & 2 grid cells \\ +Auxiliary spatial loss weight & 0.5 \\ +Optimizer & AdamW \\ +Learning rate & \(2\times10^{-4}\) \\ +Weight decay & \(10^{-4}\) \\ +Batch size & 32 \\ +Epochs & 10 \\ +\bottomrule +\end{tabular} +\end{table} +\FloatBarrier + +\section{Input Normalization Statistics} +\label{app:normalization} + +\begin{table}[H] +\centering +\small +\caption{Train-split z-score statistics for normalized channels.} +\label{tab:normalization} +\begin{tabular}{rllrr} +\toprule +\rowcolor{boxbg} +Ch. & Name & Unit & Mean & Std. \\ +\midrule +0 & \texttt{t2m} & K & 295.5830 & 6.5945 \\ +1 & \texttt{d2m} & K & 288.8014 & 7.6761 \\ +2 & \texttt{u10} & m s\(^{-1}\) & -0.1179 & 3.2024 \\ +3 & \texttt{v10} & m s\(^{-1}\) & 0.0219 & 3.9899 \\ +4 & \texttt{cape} & J kg\(^{-1}\) & 747.1428 & 1110.3022 \\ +5 & \texttt{sp} & Pa & 96817.9246 & 6815.0126 \\ +6 & \texttt{blh} & m & 681.3271 & 684.6619 \\ +7 & \texttt{vis} & m & 26947.7353 & 20073.7663 \\ +8 & \texttt{prate} & kg m\(^{-2}\) s\(^{-1}\) & 0.0000 & \(10^{-6}\) \\ +9 & \texttt{tp} & kg m\(^{-2}\) & 0.0000 & \(10^{-6}\) \\ +13 & \texttt{canopy\_cover} & percent & 8.2556 & 21.4576 \\ +14 & \texttt{housing\_density} & native & 7.2324 & 62.6264 \\ +15 & \texttt{population} & persons & 14.2284 & 122.3255 \\ +\bottomrule +\end{tabular} +\end{table} +\FloatBarrier + +\section{Public Artifacts} +\label{app:artifacts} + +\begin{table}[H] +\centering +\small +\caption{Primary artifacts associated with the model report.} +\label{tab:artifacts} +\begin{tabularx}{0.98\linewidth}{lY} +\toprule +\rowcolor{boxbg} +Artifact & Purpose \\ +\midrule +Model weights & Store the learned U-Net parameters for active-fire occupancy prediction. \\ +Input-channel metadata & Records channel order, source dataset, unit, and variable description. \\ +Normalization statistics & Records train-split z-score statistics for continuous input channels. \\ +Inference code & Applies normalization, no-exposure serving, overlap-window prediction, and phase averaging. \\ +Training code & Documents cache construction, sparse-label crop sampling, objective functions, and optimization. \\ +Example probability raster & Provides a quantitative \(\grid\) probability output for one CONUS time map. \\ +Rendered heatmap & Provides a discrete speckled raster preview of the same probability output. \\ +Data-source notes & Point users to the original data providers; raw source data are not redistributed. \\ +\bottomrule +\end{tabularx} +\end{table} +\FloatBarrier + +\section{Evaluation Contract Definitions} +\label{app:eval_contract} + +This appendix records the definitions used in Section~\ref{sec:eval_design} for completeness. +An \emph{output record} \(\mathcal{O} = (S, Y)\) pairs a score field \(S = \{s_{i,t}\}\) over spatial units \(i\) and times \(t\) with an observation field \(Y = \{y_{i,t}\}\). +A \emph{matching rule} specifies when a scored unit-time pair is considered to agree with an observed one; Figure~\ref{fig:matching} illustrates exact, tolerated, and union conventions. +An \emph{evaluation scope} specifies the spatial or temporal subset over which a metric is computed, for example the full Lower-48 domain versus a historically fire-prone subset. +For the \model{} release documented in this report, the evaluation scope is the full Lower-48 domain and the reported metrics are threshold-free (PR-AUC, AUROC, Brier score, expected calibration error) and top-area concentration measures (precision, recall, and lift at fixed scored-area bands), as reported in Table~\ref{tab:diagnostics}. +These definitions are provided so that future releases evaluated under a matching-rule breakdown or an alternative scope can be reported in a directly comparable format. + +\section{Appendix-Only Evaluation Metrics} +\label{app:boundary_evaluation} + +The main text reports only the fixed-contract rows where \model{} is the strongest verified entry. +This appendix records operating trade-offs, close supporting-task comparisons, and metrics not won by \model{}. +These rows are useful for interpretation, but they are not used as headline claims. + +\begin{table}[H] +\centering +\small +\caption{Occupancy operating trade-offs across three final configurations. Strict keeps the smallest positive footprint; Context broadens the crop budget; recall-supporting context trades more positives for higher tolerance-family recall. Values are percentages.} +\label{tab:app_occupancy_tradeoffs} +\begin{tabular}{l l c c c c c} +\toprule +\rowcolor{boxbg} +Config & Selection view & Strict \(F_1\) & Precision & Recall & \(t3/s3\) \(F_1\) & Pred.+ rate \\ +\midrule +Strict & strict \(F_1\) & \textbf{7.12\%} & 10.79\% & 5.93\% & 23.96\% & 0.02\% \\ +Context & tolerated-family \(F_1\) & 4.48\% & 2.82\% & 14.78\% & \textbf{39.17\%} & 0.16\% \\ +Recall-supporting context & recall-supporting view & 5.25\% & 3.26\% & \textbf{15.95\%} & 38.97\% & 0.14\% \\ +\bottomrule +\end{tabular} +\end{table} + +The strict and tolerated-family rows in Table~\ref{tab:headline_task_results} intentionally come from different validation-selected operating points. +The strict configuration keeps the predicted-positive footprint small, while the context configurations increase recall and bounded-offset \(F_1\) at the cost of exact precision. +This trade-off is expected for sparse active-fire occupancy and is reported explicitly rather than collapsed into one threshold. + +\begin{table}[H] +\centering +\scriptsize +\setlength{\tabcolsep}{3pt} +\renewcommand{\arraystretch}{1.08} +\caption{Occupancy \(F_1\) sweep across global and fire-prone scopes. Values are percentages from the same validation-selected strict threshold. Tolerance is spatial-only; union adds temporal and spatial matching.} +\label{tab:app_occupancy_scope_sweep} +\resizebox{\linewidth}{!}{% +\begin{tabular}{llcccc} +\toprule +\rowcolor{boxbg} +Backbone & Scope & Strict \(F_1\uparrow\) & Tol. \(F_1\uparrow\) & Union \(F_1\uparrow\) & \(\Delta\) union--strict \\ +\midrule +\model{} & global & \ms{5.31}{1.24} & \ms{44.97}{2.15} & \ms{66.74}{7.08} & \ms{61.43}{8.13} \\ + & top 5\% & \ms{11.94}{2.66} & \ms{46.04}{5.20} & \ms{69.84}{7.52} & \ms{57.89}{8.62} \\ + & top 10\% & \ms{11.78}{2.49} & \ms{45.67}{4.82} & \ms{69.21}{7.44} & \ms{57.44}{8.65} \\ + & top 20\% & \ms{11.42}{2.26} & \ms{44.20}{4.18} & \ms{67.80}{7.64} & \ms{56.38}{8.87} \\ +\addlinespace[1pt] +Prithvi-WxC & global & \ms{0.0552}{0.0039} & \ms{7.1649}{0.6557} & \ms{20.1853}{1.8299} & \ms{20.1301}{1.8297} \\ + & top 5\% & \ms{1.4119}{1.1635} & \ms{19.2636}{4.5019} & \ms{42.5793}{4.5495} & \ms{41.1674}{3.4846} \\ + & top 10\% & \ms{1.2376}{1.3201} & \ms{14.8780}{8.4429} & \ms{32.6913}{13.2085} & \ms{31.4536}{11.9053} \\ + & top 20\% & \ms{1.1520}{1.3770} & \ms{13.1512}{9.4556} & \ms{28.1319}{15.2866} & \ms{26.9800}{13.9224} \\ +\addlinespace[1pt] +Aurora & global & \ms{0.0656}{0.0094} & \ms{8.5009}{1.9594} & \ms{23.1037}{4.9418} & \ms{23.0382}{4.9325} \\ + & top 5\% & \ms{0.9859}{0.9299} & \ms{15.1337}{6.0821} & \ms{35.4834}{11.0192} & \ms{34.4975}{10.3728} \\ + & top 10\% & \ms{0.7790}{1.0453} & \ms{12.7381}{6.5558} & \ms{30.5305}{10.8842} & \ms{29.7515}{9.8656} \\ + & top 20\% & \ms{0.6655}{1.1043} & \ms{10.5304}{7.4309} & \ms{24.9444}{12.5844} & \ms{24.2790}{11.4943} \\ +\addlinespace[1pt] +ClimaX & global & \ms{0.3480}{0.0754} & \ms{29.7535}{3.6073} & \ms{60.1506}{7.5865} & \ms{59.8026}{7.5454} \\ + & top 5\% & \ms{1.2937}{0.1086} & \ms{34.5791}{2.3772} & \ms{69.2186}{5.7215} & \ms{67.9249}{5.7263} \\ + & top 10\% & \ms{1.2522}{0.1602} & \ms{34.3341}{2.2852} & \ms{68.5713}{5.5377} & \ms{67.3191}{5.5538} \\ + & top 20\% & \ms{1.0287}{0.2686} & \ms{30.2140}{4.2857} & \ms{60.0650}{7.5674} & \ms{59.0363}{7.5891} \\ +\addlinespace[1pt] +StormCast & global & \ms{0.0626}{0.0119} & \ms{8.1951}{2.1895} & \ms{22.3817}{5.4294} & \ms{22.3191}{5.4178} \\ + & top 5\% & \ms{0.9573}{0.8011} & \ms{15.3219}{5.5337} & \ms{36.1857}{9.7331} & \ms{35.2284}{9.1816} \\ + & top 10\% & \ms{0.7284}{0.9280} & \ms{12.6669}{6.3290} & \ms{30.4748}{10.6527} & \ms{29.7464}{9.7494} \\ + & top 20\% & \ms{0.5795}{0.9104} & \ms{10.4157}{7.3437} & \ms{24.6598}{12.3973} & \ms{24.0803}{11.4988} \\ +\addlinespace[1pt] +DLWP & global & \ms{0.1693}{0.0419} & \ms{14.9148}{3.2446} & \ms{28.1901}{6.9658} & \ms{28.0208}{6.9257} \\ + & top 5\% & \ms{1.8054}{0.4835} & \ms{31.7231}{3.2923} & \ms{55.4596}{5.2920} & \ms{53.6542}{5.4752} \\ + & top 10\% & \ms{1.6110}{0.5999} & \ms{27.6581}{5.9216} & \ms{47.1269}{8.0111} & \ms{45.5158}{7.7927} \\ + & top 20\% & \ms{1.5248}{0.8987} & \ms{20.9403}{4.7971} & \ms{34.9301}{7.8471} & \ms{33.4054}{7.8760} \\ +\addlinespace[1pt] +FCN & global & \ms{0.2829}{0.0839} & \ms{19.5061}{3.3412} & \ms{40.0604}{9.3701} & \ms{39.7775}{9.3423} \\ + & top 5\% & \ms{1.6231}{0.5064} & \ms{29.3769}{2.7626} & \ms{54.3033}{7.4089} & \ms{52.6801}{7.4389} \\ + & top 10\% & \ms{1.1777}{0.5118} & \ms{22.4217}{3.9803} & \ms{43.4510}{9.2513} & \ms{42.2734}{9.0251} \\ + & top 20\% & \ms{0.9962}{0.4315} & \ms{16.9792}{3.9371} & \ms{34.0859}{8.2616} & \ms{33.0897}{7.9275} \\ +\addlinespace[1pt] +FengWu & global & \ms{0.2613}{0.0757} & \ms{12.0050}{6.0239} & \ms{24.1022}{13.6293} & \ms{23.8410}{13.5736} \\ + & top 5\% & \ms{1.5695}{0.3592} & \ms{16.2763}{3.7024} & \ms{30.1055}{5.0103} & \ms{28.5360}{4.7696} \\ + & top 10\% & \ms{1.2427}{0.5333} & \ms{12.9503}{5.6052} & \ms{24.1854}{8.6854} & \ms{22.9427}{8.1863} \\ + & top 20\% & \ms{1.1192}{0.5023} & \ms{11.9508}{5.0745} & \ms{22.7860}{7.9115} & \ms{21.6668}{7.4438} \\ +\addlinespace[1pt] +FuXi & global & \ms{0.3774}{0.1212} & \ms{21.0323}{4.8211} & \ms{37.2888}{9.4470} & \ms{36.9114}{9.4327} \\ + & top 5\% & \ms{2.0307}{0.6800} & \ms{31.8944}{4.7331} & \ms{53.9308}{8.3822} & \ms{51.9001}{8.6878} \\ + & top 10\% & \ms{1.6542}{0.7316} & \ms{24.0128}{5.7784} & \ms{40.2140}{9.9307} & \ms{38.5597}{9.7744} \\ + & top 20\% & \ms{1.3646}{0.6773} & \ms{21.9548}{5.8601} & \ms{36.7314}{10.0289} & \ms{35.3668}{9.9223} \\ +\addlinespace[1pt] +Pangu-Weather & global & \ms{0.2755}{0.1089} & \ms{17.0909}{4.0477} & \ms{35.6386}{9.0327} & \ms{35.3630}{9.0774} \\ + & top 5\% & \ms{1.3656}{0.3064} & \ms{22.2222}{6.8613} & \ms{43.4234}{13.2383} & \ms{42.0578}{13.0599} \\ + & top 10\% & \ms{1.0931}{0.3535} & \ms{18.9337}{5.9329} & \ms{38.5325}{11.7221} & \ms{37.4394}{11.5261} \\ + & top 20\% & \ms{0.8844}{0.3601} & \ms{17.0172}{5.4859} & \ms{34.5688}{10.2932} & \ms{33.6844}{10.1334} \\ +\addlinespace[1pt] +AlphaEarth & global & \ms{2.0606}{0.4404} & \ms{29.4476}{6.0064} & \ms{37.4286}{9.9458} & \ms{35.3679}{10.0271} \\ + & top 5\% & \ms{6.9133}{0.8450} & \ms{42.8790}{4.6087} & \ms{51.7449}{8.7321} & \ms{44.8315}{9.0763} \\ + & top 10\% & \ms{6.6366}{0.9901} & \ms{41.8981}{5.9454} & \ms{50.5712}{10.0057} & \ms{43.9346}{9.9156} \\ + & top 20\% & \ms{6.1908}{1.1330} & \ms{38.8325}{7.4966} & \ms{46.3833}{12.1697} & \ms{40.1925}{11.6788} \\ +\bottomrule +\end{tabular}% +} +\end{table} +\FloatBarrier + +\begin{table}[H] +\centering +\scriptsize +\setlength{\tabcolsep}{3pt} +\caption{Spread AP under fixed spread task and strict matching, reported under full and fire-prone scopes. Values are percentages.} +\label{tab:app_spread_ap_by_scope} +\begin{tabular}{lccc} +\toprule +\rowcolor{boxbg} +Backbone & Full AP & Top-5\% AP & Top-10\% AP \\ +\midrule +\model{} & \textbf{\ms{32.73}{0.60}} & \textbf{\ms{40.7452}{2.0542}} & \textbf{\ms{37.4096}{1.8731}} \\ +Prithvi-WxC & \ms{4.8319}{0.1731} & \ms{12.6086}{0.4468} & \ms{8.7051}{0.1889} \\ +Aurora & \ms{17.7723}{0.4293} & \ms{30.3106}{0.9404} & \ms{26.4732}{0.6932} \\ +ClimaX & \ms{11.1726}{0.2337} & \ms{25.7871}{1.2896} & \ms{19.9977}{1.2217} \\ +StormCast & \ms{8.1147}{1.1569} & \ms{18.5461}{1.1727} & \ms{14.1286}{1.2956} \\ +DLWP & \ms{9.2142}{2.6587} & \ms{19.3346}{2.3922} & \ms{14.9788}{2.6696} \\ +FCN & \ms{6.6774}{1.3001} & \ms{16.7396}{3.2955} & \ms{11.9308}{2.3881} \\ +FengWu & \ms{11.0046}{2.7092} & \ms{21.1506}{1.2163} & \ms{17.0113}{1.5778} \\ +FuXi & \ms{13.5507}{0.3840} & \ms{22.5434}{0.4100} & \ms{19.1964}{0.3943} \\ +Pangu-Weather & \ms{10.6250}{1.4643} & \ms{19.8294}{1.3044} & \ms{15.8013}{1.1602} \\ +AlphaEarth & \ms{12.2847}{1.3562} & \ms{22.8692}{0.4915} & \ms{18.2992}{1.2110} \\ +\bottomrule +\end{tabular} +\end{table} +\FloatBarrier + +\begin{table}[H] +\centering +\small +\caption{Supporting and non-winning evaluation rows. The struck-through values are earlier report values, followed by current values when the metric was updated.} +\label{tab:app_supporting_boundary} +\begin{tabularx}{0.98\linewidth}{l l l l Y} +\toprule +\rowcolor{boxbg} +Task & Metric & \model{} & Comparator & Appendix reading \\ +\midrule +Analog retrieval & nDCG@10\(\uparrow\) & \revise{0.4775}{0.5178} & AlphaEarth 0.5036 & \model{} is higher under the early-deployment retrieval contract; the result remains supporting and contract-sensitive. \\ +Smoke PM$_{2.5}$ & RMSE\(\downarrow\) & \revise{4.4646}{4.4684} & AlphaEarth 4.4899 & \model{} remains lower than the closest comparator, but the current value is not lower than the earlier report value. \\ +Smoke PM$_{2.5}$ & MAE\(\downarrow\) & \revise{2.4108}{2.4127} & AlphaEarth 2.4009 & AlphaEarth is lower on MAE, so this row is appendix-only. \\ +Smoke PM$_{2.5}$ & Pearson \(r\uparrow\) & \revise{0.6368}{0.6360} & AlphaEarth 0.6315 & \model{} is slightly higher, with a small margin. \\ +Smoke PM$_{2.5}$ & Event \(F_1\uparrow\) & 20.12\% & AlphaEarth 14.91\% & Secondary thresholded event view; not used as a headline regression metric. \\ +Extreme heat & RMSE-C\(\downarrow\) & \revise{0.2187}{0.2179} & AlphaEarth 0.2187 & Near tie at table precision; kept out of the main headline table. \\ +Extreme heat & MAE-C\(\downarrow\) & \revise{0.1795}{0.1787} & AlphaEarth 0.1795 & Near tie at table precision; kept out of the main headline table. \\ +Extreme heat & Pearson \(r\uparrow\) & 0.999865 & AlphaEarth 0.999867 & AlphaEarth is slightly higher, so this row is appendix-only. \\ +\bottomrule +\end{tabularx} +\end{table} + +The final burned-area headline log-RMSE in Table~\ref{tab:headline_task_results} uses the validation-selected affine scale-calibration candidate. +The uncalibrated identity candidate remains close at log-RMSE 1.1718 and Spearman 0.5884, while the validation-selected affine candidate gives log-RMSE 1.1649 with the same Spearman ordering. +Because the scale calibration is selected on validation data before test reporting, it is included as the current fixed-contract burned-area headline. + +\begin{table}[H] +\centering +\scriptsize +\setlength{\tabcolsep}{3pt} +\caption{Additional final burned-area diagnostics.} +\label{tab:app_burned_area_diagnostics} +\begin{tabular}{lccc} +\toprule +\rowcolor{boxbg} +Backbone & Median log AE & Median acre AE & Acre MAPE \\ +\midrule +\model{} & \ms{1.0235}{0.0982} & \ms{4504.0692}{459.0483} & \ms{1.4525}{0.0254} \\ +Prithvi-WxC & \ms{1.2184}{0.2107} & \ms{5375.8770}{788.7906} & \ms{1.9517}{0.2875} \\ +Aurora & \ms{1.4547}{0.0301} & \ms{9904.9483}{457.4260} & \ms{6.8728}{3.0026} \\ +ClimaX & \ms{1.6841}{0.1818} & \ms{18130.4820}{3248.3873} & \ms{8.2373}{2.8540} \\ +StormCast & \ms{1.4522}{0.1519} & \ms{11155.7881}{2020.8656} & \ms{4.6142}{1.1500} \\ +DLWP & \ms{1.0952}{0.1306} & \ms{4406.9315}{303.0944} & \ms{1.7357}{0.3625} \\ +FCN & \ms{1.1688}{0.1139} & \ms{5166.9993}{213.0333} & \ms{2.0800}{0.4004} \\ +FengWu & \ms{1.1589}{0.1772} & \ms{5137.2822}{628.7543} & \ms{2.0944}{0.4545} \\ +FuXi & \ms{1.1855}{0.0612} & \ms{5697.7117}{796.8785} & \ms{2.4411}{0.5567} \\ +Pangu-Weather & \ms{1.1221}{0.1470} & \ms{5092.3621}{483.8243} & \ms{1.9571}{0.3113} \\ +AlphaEarth & \ms{1.7459}{0.6057} & \ms{15110.7573}{7106.3417} & \ms{9.7398}{2.7425} \\ +\bottomrule +\end{tabular} +\end{table} + +\begin{table}[H] +\centering +\scriptsize +\setlength{\tabcolsep}{3pt} +\caption{Additional analog-retrieval diagnostics.} +\label{tab:app_analog_retrieval_diagnostics} +\begin{tabular}{lccc} +\toprule +\rowcolor{boxbg} +Backbone & nDCG@5 & Best log gap & Rank \(\rho\) \\ +\midrule +\model{} & \ms{0.5175}{0.0445} & \ms{0.1868}{0.0285} & \ms{0.6019}{0.1460} \\ +Prithvi-WxC & \ms{0.3591}{0.0107} & \ms{0.2151}{0.0594} & \ms{0.1514}{0.1489} \\ +Aurora & \ms{0.4423}{0.0210} & \ms{0.1551}{0.0437} & \ms{0.2162}{0.1856} \\ +ClimaX & \ms{0.4151}{0.0293} & \ms{0.2129}{0.0653} & \ms{0.1587}{0.2831} \\ +StormCast & \ms{0.3960}{0.0240} & \ms{0.1714}{0.0310} & \ms{0.1258}{0.1625} \\ +DLWP & \ms{0.3795}{0.0274} & \ms{0.1944}{0.0807} & \ms{-0.3865}{0.2802} \\ +FCN & \ms{0.4250}{0.0112} & \ms{0.1856}{0.0846} & \ms{-0.1357}{0.2571} \\ +FengWu & \ms{0.4228}{0.0310} & \ms{0.1870}{0.0858} & \ms{-0.1926}{0.2194} \\ +FuXi & \ms{0.4544}{0.0356} & \ms{0.2171}{0.0806} & \ms{-0.1367}{0.2885} \\ +Pangu-Weather & \ms{0.3988}{0.0506} & \ms{0.1901}{0.0838} & \ms{-0.1970}{0.2216} \\ +AlphaEarth & \textbf{\ms{0.5276}{0.0531}} & \textbf{\ms{0.1782}{0.0454}} & \ms{0.4639}{0.2802} \\ +\bottomrule +\end{tabular} +\end{table} + +\begin{table}[H] +\centering +\scriptsize +\setlength{\tabcolsep}{3pt} +\caption{High-smoke subset diagnostics for rows with observed PM\(_{2.5}\ge35\).} +\label{tab:app_smoke_high_event} +\begin{tabular}{lccc} +\toprule +\rowcolor{boxbg} +Backbone & High-smoke RMSE & High-smoke MAE & High-smoke 90th AE \\ +\midrule +\model{} & \textbf{\ms{47.4870}{0.6346}} & \textbf{\ms{34.3954}{0.7654}} & \textbf{\ms{65.6213}{3.8778}} \\ +Prithvi-WxC & \ms{57.2224}{1.7268} & \ms{47.3871}{0.3153} & \ms{74.9666}{3.2381} \\ +Aurora & \ms{57.2752}{1.7248} & \ms{47.4368}{0.3149} & \ms{75.0755}{3.1074} \\ +ClimaX & \ms{57.2828}{1.7239} & \ms{47.4407}{0.3140} & \ms{75.1012}{3.0777} \\ +StormCast & \ms{56.6512}{1.7517} & \ms{46.7914}{0.3281} & \ms{74.0794}{3.4707} \\ +DLWP & \ms{57.0075}{1.7359} & \ms{47.1971}{0.3198} & \ms{74.4936}{3.3826} \\ +FCN & \ms{57.0582}{1.7339} & \ms{47.2401}{0.3187} & \ms{74.6431}{3.1982} \\ +FengWu & \ms{57.0158}{1.7357} & \ms{47.1957}{0.3194} & \ms{74.5652}{3.2871} \\ +FuXi & \ms{56.9622}{1.7371} & \ms{47.1508}{0.3201} & \ms{74.3278}{3.4435} \\ +Pangu-Weather & \ms{57.1282}{1.7307} & \ms{47.3050}{0.3170} & \ms{74.6830}{3.2375} \\ +AlphaEarth & \ms{48.0665}{0.7904} & \ms{35.6088}{0.7341} & \ms{66.7613}{3.9235} \\ +\bottomrule +\end{tabular} +\end{table} + +\begin{table}[H] +\centering +\scriptsize +\setlength{\tabcolsep}{4pt} +\caption{Extreme-heat exceedance precision and recall for the \(F_1\) row.} +\label{tab:app_heat_pr} +\begin{tabular}{lcc} +\toprule +\rowcolor{boxbg} +Backbone & Precision & Recall \\ +\midrule +\model{} & \ms{0.9767}{0.0117} & \ms{0.9330}{0.0299} \\ +Prithvi-WxC & \ms{0.8260}{0.0030} & \ms{0.9173}{0.0033} \\ +Aurora & \ms{0.5920}{0.0347} & \ms{0.0517}{0.0020} \\ +ClimaX & \ms{0.7397}{0.0099} & \ms{0.7994}{0.0051} \\ +StormCast & \ms{0.8840}{0.0237} & \ms{0.9320}{0.0165} \\ +DLWP & \ms{0.9429}{0.0085} & \ms{0.8899}{0.0167} \\ +FCN & \ms{0.9408}{0.0097} & \ms{0.9111}{0.0127} \\ +FengWu & \ms{0.3808}{0.2719} & \ms{0.0266}{0.0267} \\ +FuXi & \ms{0.3262}{0.1262} & \ms{0.1810}{0.0481} \\ +Pangu-Weather & \ms{0.1159}{0.0743} & \ms{0.0112}{0.0032} \\ +AlphaEarth & \ms{0.9824}{0.0040} & \ms{0.9278}{0.0178} \\ +\bottomrule +\end{tabular} +\end{table} + +\begin{table}[H] +\centering +\scriptsize +\setlength{\tabcolsep}{4pt} +\renewcommand{\arraystretch}{1.12} +\caption{Seed robustness summary for stochastic task checks.} +\label{tab:app_seed_robustness} +\resizebox{\linewidth}{!}{% +\begin{tabular}{lcll} +\toprule +\rowcolor{boxbg} +Check & Seeds & Primary value & Additional values \\ +\midrule +Final burned area & 5 & log-RMSE \ms{1.1649}{0.0139} & log-MAE \ms{1.0355}{0.0114}; Spearman \ms{0.5884}{0.0338} \\ +Smoke PM\(_{2.5}\) & 5 & RMSE \ms{4.4684}{0.0060} & MAE \ms{2.4127}{0.0016}; \(r\) \ms{0.6360}{0.0013} \\ +Extreme heat & 5 & RMSE-C \ms{0.2179}{0.0048} & MAE-C \ms{0.1787}{0.0020}; exceed. \(F_1\) \ms{0.9541}{0.0164} \\ +Fire spread & 5 & exact \(F_1\) \ms{37.98}{1.33} & spatial \(F_1\) \ms{82.20}{0.32}; AP \ms{32.73}{0.60} \\ +\bottomrule +\end{tabular}% +} +\end{table} +\FloatBarrier + +\end{document} diff --git a/technical_report/source/reference_tech.bib b/technical_report/source/reference_tech.bib new file mode 100644 index 0000000000000000000000000000000000000000..37484cb57e44323a9d072d7bef05a8fe63768b99 --- /dev/null +++ b/technical_report/source/reference_tech.bib @@ -0,0 +1,111 @@ +@inproceedings{ronneberger2015unet, + title = {U-Net: Convolutional Networks for Biomedical Image Segmentation}, + author = {Ronneberger, Olaf and Fischer, Philipp and Brox, Thomas}, + booktitle = {Medical Image Computing and Computer-Assisted Intervention}, + pages = {234--241}, + year = {2015}, + doi = {10.1007/978-3-319-24574-4_28} +} + +@inproceedings{wu2018groupnorm, + title = {Group Normalization}, + author = {Wu, Yuxin and He, Kaiming}, + booktitle = {European Conference on Computer Vision}, + pages = {3--19}, + year = {2018}, + doi = {10.1007/978-3-030-01261-8_1} +} + +@inproceedings{loshchilov2019adamw, + title = {Decoupled Weight Decay Regularization}, + author = {Loshchilov, Ilya and Hutter, Frank}, + booktitle = {International Conference on Learning Representations}, + year = {2019}, + url = {https://openreview.net/forum?id=Bkg6RiCqY7} +} + +@misc{noaa_hrrr_ncei, + title = {{High-Resolution Rapid Refresh}}, + author = {{NOAA National Centers for Environmental Information}}, + year = {2026}, + howpublished = {\url{https://www.ncei.noaa.gov/products/weather-climate-models/high-resolution-rapid-refresh}}, + note = {Accessed: 2026-07-03} +} + +@misc{noaa_hrrr_emc, + title = {{High-Resolution Rapid Refresh (HRRR)}}, + author = {{NOAA National Centers for Environmental Prediction Environmental Modeling Center}}, + year = {2026}, + howpublished = {\url{https://rapidrefresh.noaa.gov/hrrr/}}, + note = {Accessed: 2026-07-03} +} + +@misc{nasa_firms, + title = {{Fire Information for Resource Management System (FIRMS)}}, + author = {{NASA Earthdata}}, + year = {2026}, + howpublished = {\url{https://firms.modaps.eosdis.nasa.gov/}}, + note = {Accessed: 2026-07-03} +} + +@misc{landfire_fbfm40, + title = {{LANDFIRE 40 Fire Behavior Fuel Models}}, + author = {{LANDFIRE}}, + year = {2026}, + howpublished = {\url{https://landfire.gov/data}}, + note = {Accessed: 2026-07-03} +} + +@misc{landfire_canopy_cover, + title = {{LANDFIRE Canopy Cover}}, + author = {{LANDFIRE}}, + year = {2026}, + howpublished = {\url{https://landfire.gov/data}}, + note = {Accessed: 2026-07-03} +} + +@misc{usfs_wrc_housing_density, + title = {{Wildfire Risk to Communities Housing Unit Density}}, + author = {{USDA Forest Service}}, + year = {2026}, + howpublished = {\url{https://wildfirerisk.org/download/}}, + note = {Accessed: 2026-07-03} +} + +@misc{ornl_landscan_2024, + title = {{LandScan Global 2024}}, + author = {{Oak Ridge National Laboratory}}, + year = {2024}, + howpublished = {\url{https://landscan.ornl.gov/}}, + note = {Accessed: 2026-07-03} +} + +@article{ebert2009neighborhood, + title = {Neighborhood Verification: A Strategy for Rewarding Close Forecasts}, + author = {Ebert, Elizabeth E.}, + journal = {Weather and Forecasting}, + volume = {24}, + number = {6}, + pages = {1498--1510}, + year = {2009}, + doi = {10.1175/2009WAF2222251.1} +} + +@article{gilleland2009intercomparison, + title = {Intercomparison of Spatial Forecast Verification Methods}, + author = {Gilleland, Eric and Ahijevych, David and Brown, Barbara G and Ebert, Elizabeth E}, + journal = {Weather and Forecasting}, + volume = {24}, + number = {5}, + pages = {1416--1430}, + year = {2009}, + publisher = {American Meteorological Society} +} + +@misc{usgs_fire_danger_forecast, + title = {{Fire Danger Forecast}}, + author = {{U.S. Geological Survey}}, + year = {2026}, + howpublished = {\url{https://firedanger.cr.usgs.gov/viewer/index.html}}, + note = {Accessed: 2026-07-07} +} diff --git a/training/NATIONWIDE_RETRAINING.md b/training/NATIONWIDE_RETRAINING.md deleted file mode 100644 index fd81f300e7419cd33aa1b293401c061769bd33c5..0000000000000000000000000000000000000000 --- a/training/NATIONWIDE_RETRAINING.md +++ /dev/null @@ -1,80 +0,0 @@ -# Nationwide FireWx-FM Retraining Path - -The released FireWx-FM checkpoints are California regional checkpoints. They are -not nationwide-trained weights. This folder now includes the CONUS cache and -training templates needed to retrain the same model class on nationwide data. - -## Current released weights - -| Item | Status | -|---|---| -| Released checkpoints | California regional FireWx-FM weights | -| Native grid in `input_channels.json` | California, 5 km, EPSG:5070 | -| Nationwide/CONUS checkpoint | Not released yet | -| Nationwide training recipe | Provided here | - -## Why nationwide training is feasible - -The core inputs used by the regional model have public nationwide or broader -coverage: - -| Data role | Source | Nationwide availability | -|---|---|---| -| Dynamic weather | NOAA HRRR public archive | CONUS HRRR files are available from the public NOAA archive | -| Active-fire supervision | NASA FIRMS | FIRMS provides active-fire detections through global/area APIs | -| Fuel | LANDFIRE FBFM40 | CONUS product | -| Canopy | LANDFIRE Canopy Cover | CONUS product | -| Housing exposure | Wildfire Risk to Communities housing density | United States populated-area product | -| Population exposure | LandScan | Global annual product, subject to provider terms | - -Raw provider data are not redistributed in this repository. Users should obtain -them under provider terms, then run the cache builder locally. - -## Added configs - -| Config | Purpose | -|---|---| -| `configs/stage1_cache_conus_hrrr_us_5km_l12_template.json` | Build a CONUS 5 km, 12-hour-lead HRRR/FIRMS/static cache. | -| `configs/train_firewx_fm_conus_seed7_template.json` | Train the compact U-Net on the CONUS cache. | - -## Commands - -Build the CONUS cache after preparing local HRRR, FIRMS, LANDFIRE, WRC, and -LandScan paths: - -```bash -python training/build_phase1_cache_regional_hrrr.py \ - --config training/configs/stage1_cache_conus_hrrr_us_5km_l12_template.json -``` - -Train the model: - -```bash -python training/train_cold_tiled_mainline.py \ - --config training/configs/train_firewx_fm_conus_seed7_template.json \ - --run-name firewx_fm_conus_seed7 -``` - -## Serving-oriented settings in the CONUS template - -The CONUS training template differs from the released California checkpoint -configuration in two serving-oriented ways: - -- `positive_tile_placement` is set to `random_containing`, so a fire-positive - cell can appear at different positions within a 32-by-32 training tile rather - than always being centered. -- `input_normalization.enabled` is true for continuous channels - `0-9, 13, 14, 15`. Weather, canopy, housing, and population channels are - normalized from train-split full-map statistics. The dynamic mask, static - validity mask, and categorical fuel code are left unnormalized. - -These settings are intended for retraining a cleaner CONUS-serving model. They -do not change the already released California checkpoints, which use native -source units as documented in `models/wildfire_fm/input_channels.json`. - -## Practical note - -The CONUS grid is much larger than the California regional grid. Run cache -building and training on compute/storage infrastructure, not a login node. For -multi-year nationwide training, shard cache construction by year or season -before merging manifests. diff --git a/training/README.md b/training/README.md index f6ad6d30096926bf5e3f95ff778fb71f59d548b1..831b4c16e0541b3acca687e30f00ce385e64b586 100644 --- a/training/README.md +++ b/training/README.md @@ -1,114 +1,89 @@ -# FireWx-FM Training and Data Loader Pipeline +# Training Pipeline -This folder contains the original FireWx-FM cache builder and PyTorch training -pipeline used for the released regional wildfire occupancy checkpoints. The raw -provider datasets and local cache tensors are not redistributed here. The code -shows how those sources are converted into the model tensor contract and how the -training loader samples tiles from that cache. +The training code in this directory is included so the released checkpoint can be inspected and adapted. Raw data and intermediate caches are not redistributed. -## Files +## Main Files -| Path | Purpose | +| File | Purpose | |---|---| -| `build_phase1_cache_regional_hrrr.py` | Builds the regional HRRR/FIRMS/static cache on the California EPSG:5070 grid. | -| `train_cold_tiled_mainline.py` | Contains the `ColdFeatureStore`, tile sampler, full-map loader, compact U-Net, losses, and training loop. | -| `eval_metrics.py` | Metric utilities used by the training script for validation and test summaries. | -| `cache_helpers.py` | Minimal helper functions used by the regional cache builder. | -| `train_utils.py` | Reproducibility helper used by the training script. | -| `configs/stage1_cache_regional_hrrr_ca_5km_l12_template.json` | Template for constructing the released California 5 km, 12-hour-lead cache. | -| `configs/train_firewx_fm_seed7_template.json` | Template for the seeded California FireWx-FM training run. | -| `configs/stage1_cache_conus_hrrr_us_5km_l12_template.json` | Template for constructing a CONUS 5 km, 12-hour-lead cache. | -| `configs/train_firewx_fm_conus_seed7_template.json` | Template for CONUS retraining after local data preparation. | -| `NATIONWIDE_RETRAINING.md` | Scope note and commands for nationwide retraining. | - -## Data boundary - -The scripts expect local copies of the public source datasets described in -`../data_sources/DATA_SOURCES.md`: NOAA HRRR fields, NASA FIRMS active-fire -detections, LANDFIRE fuel and canopy layers, Wildfire Risk to Communities -housing density, and LandScan population. WFIGS and MTBS support event-level -experiments, but they are not input channels for the released occupancy -checkpoint. - -## Cache construction - -The cache builder writes: - -| File group | Arrays | -|---|---| -| `inputs/phase1_regional_*.npz` | `weather`, `firewx`, `firewx_valid`, `y_count`, `y_occ`, `lat`, `lon`, `weather_names`, `firewx_names` | -| `static/static_regional_phase1_v1.npz` | `static`, `static_valid`, `lat`, `lon`, `coord_crs`, `static_names` | -| `splits/{train,val,test}.csv` | Sample metadata and paths used by the training loader. | -| `manifests/phase1_cache_summary.json` | Cache shape, split counts, channel names, and grid metadata. | +| `build_phase1_cache_regional_hrrr.py` | Builds gridded weather/static/target caches from source datasets. | +| `../scripts/prepare_firms_target_hour_csvs.py` | Converts daily FIRMS CSV exports into `YYYYMMDD_HH.csv` target-hour label files. | +| `train_cold_tiled_mainline.py` | Trains the compact U-Net on cached time maps using sparse-label tile sampling. | +| `eval_metrics.py` | Metric utilities used during model development. | +| `configs/stage1_cache_conus_hrrr_us_5km_l12_template.json` | Template for a CONUS 5 km, 12-hour-lead cache. | +| `configs/train_firewxfm_conus_noexposure_template.json` | Path-free training template with the main final-run settings. | +| `configs/stage1_cache_conus_multiyear_template.json` | Multi-year CONUS cache template for cross-year refinement experiments. | +| `configs/train_firewxfm_conus_multiyear_template.json` | Multi-year supervised training template with no-exposure channels zeroed after normalization. | +| `configs/eval_region_disjoint_template.json` | Region-disjoint masked-evaluation design template. | +| `configs/baseline_suite_template.json` | Baseline suite design for climatology, persistence, weather-only, static-only, and wildfire-specific baselines. | + +## Training Strategy + +The model is trained from chronological time maps. Training crops 32 by 32 tiles from the maps to increase exposure to sparse active-fire cells while retaining non-fire context. The final model uses per-channel z-score normalization for continuous inputs, class-weighted loss terms for sparse occupancy labels, and an auxiliary spatial-support head. + +The 32 by 32 crop size is a training choice, not the serving contract. Final CONUS inference should use `firewxfm.serve_conus` with larger overlapping windows and phase averaging. -Run it with a local copy of the template config: +Example command: ```bash -python training/build_phase1_cache_regional_hrrr.py \ - --config training/configs/stage1_cache_regional_hrrr_ca_5km_l12_template.json +python training/train_cold_tiled_mainline.py \ + --config training/configs/train_firewxfm_conus_noexposure_template.json \ + --run-name firewxfm_conus_noexposure_seed42 ``` -The released configuration uses a California grid at 5 km resolution in -`EPSG:5070`, 6-hourly HRRR issue times, and 12-hour occupancy labels derived -from FIRMS detections. +## Phase-0 Audit -The released checkpoints are California regional weights. For nationwide -training, use the CONUS templates and notes in `NATIONWIDE_RETRAINING.md`. +On UF/HPC, run the following conversion, audit, cache-building, and training +steps through Slurm compute nodes only. Do not run them directly on login nodes. +The repository includes a one-day smoke test submission wrapper: -## Tensor contract +```bash +bash slurm/submit_firewxfm_2024_pilot_smoke.sh +``` -`ColdFeatureStore` is the key data-loader class. It assembles each model input -as: +If FIRMS is available as daily CSV files, first create the target-hour labels +expected by the cache builder. The default keeps FIRMS `type=0` wildfire +detections, filters to the config bounding box, uses centered 6-hour bins, and +assigns each active-fire detection to the nearest 00/06/12/18 UTC target +timestamp: -```python -x = np.concatenate([weather, firewx, *extra, static_valid, static], axis=0) +```bash +python scripts/prepare_firms_target_hour_csvs.py \ + --daily-dir /path/to/firms_daily_csv \ + --output-dir /path/to/firms_target_hour_csv \ + --config training/configs/stage1_cache_conus_multiyear_template.json \ + --summary artifacts/manifests/firms_target_hour_summary.json ``` -For the released regional cache, `firewx` has zero feature channels and -`extra = [firewx_valid]`. The resulting 16-channel tensor is: - -| Channels | Names | Source | -|---:|---|---| -| 0-9 | `t2m`, `d2m`, `u10`, `v10`, `cape`, `sp`, `blh`, `vis`, `prate`, `tp` | NOAA HRRR | -| 10 | `firewx_valid` | Dynamic/input validity mask for this regional cache | -| 11 | `static_valid` | Static reprojection validity mask | -| 12-15 | `fuel_fbfm40`, `canopy_cover`, `housing_density`, `population` | LANDFIRE, WRC housing density, LandScan | +Before launching a multi-year cache build or training run, audit expected HRRR +and target-hour FIRMS coverage: -The same contract is available in machine-readable form at -`../models/wildfire_fm/input_channels.json`. +```bash +python scripts/audit_firewxfm_phase0.py \ + --config training/configs/stage1_cache_conus_multiyear_template.json \ + --output artifacts/manifests/phase0_data_split_audit.json +``` -## Training +The public template uses placeholder paths, so a clean checkout will report +missing inputs. On the training machine, replace the paths first and rerun the +audit; the output JSON should be stored with the experiment manifests. -The training script reads `splits/train.csv`, `splits/val.csv`, and -`splits/test.csv` from `index_root`. Training uses 32-by-32 tiles sampled from -the cached time maps. Validation and test use full maps. +## Multi-Year Refinement Run -For serving-oriented retraining, set `positive_tile_placement` to -`random_containing`. This samples positive tiles so the fire cell can appear at -different positions within the tile instead of always being centered. The -training script also supports `input_normalization`; use it to compute -train-split z-score statistics for continuous weather and static channels while -leaving masks and categorical fuel codes unscaled. +After the Phase-0 audit passes and the cache is built: ```bash +python training/build_phase1_cache_regional_hrrr.py \ + --config training/configs/stage1_cache_conus_multiyear_template.json + python training/train_cold_tiled_mainline.py \ - --config training/configs/train_firewx_fm_seed7_template.json \ - --run-name firewx_fm_seed7 + --config training/configs/train_firewxfm_conus_multiyear_template.json \ + --run-name firewxfm_conus_multiyear_noexposure_seed42 ``` -Released checkpoints are stored on the Hub under -`models/wildfire_fm/checkpoints/seed_*/best_firms_prauc.pt`. The released seed -configs and training summaries are under `models/wildfire_fm/configs/` and -`models/wildfire_fm/metrics/`. - -## Inference adaptation +The training script supports `zero_input_channels`; for the no-exposure +contract, channels `14` and `15` are zeroed after normalization. -For inference, the important part is to reproduce the same 16-channel tensor -order. If a downstream environment already constructs `[channel, y, x]` tensors -in that order, it can use `models/wildfire_fm/modeling_unet.py` directly and -load one of the released seeded checkpoints. +## Data Boundary -Avoid non-overlapping 32-by-32 serving tiles. If full-map inference is not -available, use overlap-tiled inference with halo cropping or overlap blending. -The helper `models/wildfire_fm/tiled_inference.py` provides -`predict_probability_tiled` for this stitching pattern. +The scripts expect local copies of HRRR, FIRMS, LANDFIRE, Wildfire Risk to Communities, and LandScan resources. See [`../data_sources/DATA_SOURCES.md`](../data_sources/DATA_SOURCES.md) for source links and access notes. diff --git a/training/build_phase1_cache_regional_hrrr.py b/training/build_phase1_cache_regional_hrrr.py index 58f8f8118b2195ed6a2806756f890a5fc17fd6f7..3d10cd941315066c8e5189f05b020cc0b07401bd 100644 --- a/training/build_phase1_cache_regional_hrrr.py +++ b/training/build_phase1_cache_regional_hrrr.py @@ -31,9 +31,13 @@ VAR_GROUPS = { "filter_by_keys": {"typeOfLevel": "heightAboveGround", "level": 10}, "vars": ["u10", "v10", "max_10si"], }, + "height_0_3000": { + "filter_by_keys": {"typeOfLevel": "heightAboveGroundLayer", "stepType": "instant", "shortName": "cape"}, + "vars": ["cape"], + }, "surface_instant": { "filter_by_keys": {"typeOfLevel": "surface", "stepType": "instant"}, - "vars": ["cape", "sp", "blh", "vis", "prate", "gust", "t"], + "vars": ["sp", "blh", "vis", "prate", "gust", "t"], }, "surface_accum": { "filter_by_keys": {"typeOfLevel": "surface", "stepType": "accum"}, @@ -195,7 +199,7 @@ def build_source_resampler(sample_grib: Path, bounds: Dict[str, float], target_c ) flat_idx = np.flatnonzero(mask.ravel()) if flat_idx.size == 0: - raise RuntimeError("No HRRR source points remain after California bbox crop.") + raise RuntimeError("No HRRR source points remain after the requested bbox crop.") src_lat = lat2d.ravel()[flat_idx] src_lon = lon2d.ravel()[flat_idx] src_x, src_y = transform("EPSG:4326", target_crs, src_lon.tolist(), src_lat.tolist()) @@ -284,8 +288,29 @@ def load_static_cube_projected(ctx: RegionalContext, y_desc: np.ndarray, x_asc: with rasterio.open(spec["path"]) as src: src_bounds = transform_bounds(target_crs, src.crs, west, south, east, north, densify_pts=21) window = src.window(*src_bounds).round_offsets().round_lengths() - source = src.read(1, window=window) - source_transform = src.window_transform(window) + row_off = int(window.row_off) + col_off = int(window.col_off) + row_stop = row_off + int(window.height) + col_stop = col_off + int(window.width) + row_start = max(0, row_off) + col_start = max(0, col_off) + row_stop = min(src.height, row_stop) + col_stop = min(src.width, col_stop) + if row_start >= row_stop or col_start >= col_stop: + valid_arrays.append(np.zeros(shape, dtype=np.float32)) + arrays.append(np.zeros(shape, dtype=np.float32)) + continue + clipped_window = rasterio.windows.Window( + col_off=col_start, + row_off=row_start, + width=col_stop - col_start, + height=row_stop - row_start, + ) + # The target CONUS bbox can exceed a static raster's extent. Use the + # clipped window transform; otherwise the clipped source array is + # georeferenced with the larger out-of-bounds window and shifts. + source = src.read(1, window=clipped_window) + source_transform = src.window_transform(clipped_window) reproject( source=source, destination=dst, diff --git a/training/configs/baseline_suite_template.json b/training/configs/baseline_suite_template.json new file mode 100644 index 0000000000000000000000000000000000000000..d9c8cf301388d4f84691d7ac415b2fe53ead34ec --- /dev/null +++ b/training/configs/baseline_suite_template.json @@ -0,0 +1,58 @@ +{ + "purpose": "Baseline suite for CONUS multi-year FireWx-FM experiments.", + "cache_root": "/path/to/local/firewx_fm_cache/conus_hrrr_us_5km_l12_multiyear", + "run_root": "/path/to/local/firewx_fm_runs", + "splits": { + "train": "splits/train.csv", + "val": "splits/val.csv", + "test": "splits/test.csv" + }, + "simple_baselines": [ + { + "name": "base_rate", + "description": "Constant train-split positive rate." + }, + { + "name": "historical_fire_prone_climatology", + "description": "Cellwise historical FIRMS occupancy frequency computed only from train years." + }, + { + "name": "recent_fire_persistence", + "description": "Recent FIRMS detections persisted to the target lead if allowed by the task contract." + }, + { + "name": "weather_only_logistic_or_gbdt", + "description": "Tabular baseline using HRRR channels only." + }, + { + "name": "static_only_logistic_or_gbdt", + "description": "Tabular baseline using fuel, canopy, and optional exposure channels only." + }, + { + "name": "weather_only_unet", + "description": "Same U-Net with static channels masked to zero." + }, + { + "name": "no_exposure_unet", + "description": "Same U-Net with housing and population channels zeroed after normalization." + } + ], + "wildfire_specific_baselines": [ + { + "name": "firecast_style", + "status": "implement_or_adapt_if_public_code_and_features_are_available" + }, + { + "name": "next_day_wildfire_prediction_style", + "status": "implement_or_adapt_if_public_code_and_features_are_available" + } + ], + "metrics": [ + "pr_auc", + "auroc", + "brier", + "ece", + "topk_area_recall_lift", + "fss" + ] +} diff --git a/training/configs/eval_region_disjoint_template.json b/training/configs/eval_region_disjoint_template.json new file mode 100644 index 0000000000000000000000000000000000000000..3c6ee475cb29b60ce9ff43a13b79f3d93aa25e22 --- /dev/null +++ b/training/configs/eval_region_disjoint_template.json @@ -0,0 +1,59 @@ +{ + "purpose": "Region-disjoint evaluation design for FireWx-FM CONUS experiments.", + "cache_root": "/path/to/local/firewx_fm_cache/conus_hrrr_us_5km_l12_multiyear", + "run_root": "/path/to/local/firewx_fm_runs", + "region_masks": { + "california_pacific": "/path/to/region_masks/california_pacific_epsg5070_5km.npy", + "pacific_northwest": "/path/to/region_masks/pacific_northwest_epsg5070_5km.npy", + "rockies_intermountain": "/path/to/region_masks/rockies_intermountain_epsg5070_5km.npy", + "southeast_florida": "/path/to/region_masks/southeast_florida_epsg5070_5km.npy", + "plains_central": "/path/to/region_masks/plains_central_epsg5070_5km.npy" + }, + "leave_one_region_out": [ + { + "held_out_region": "california_pacific", + "train_regions": [ + "pacific_northwest", + "rockies_intermountain", + "southeast_florida", + "plains_central" + ] + }, + { + "held_out_region": "southeast_florida", + "train_regions": [ + "california_pacific", + "pacific_northwest", + "rockies_intermountain", + "plains_central" + ] + } + ], + "metrics": { + "thresholds": [ + 0.02, + 0.05, + 0.1, + 0.2, + 0.3 + ], + "topk_area_fractions": [ + 0.001, + 0.005, + 0.01, + 0.05, + 0.1 + ], + "fss_radii": [ + 1, + 2, + 4, + 8 + ], + "calibration_bins": 15 + }, + "notes": [ + "The current supervised trainer samples full-map crops. Region-disjoint training requires region masks inside the tile sampler before claims are made from this config.", + "This config is immediately useful for masked evaluation of trained CONUS checkpoints." + ] +} diff --git a/training/configs/stage1_cache_regional_hrrr_ca_5km_l12_template.json b/training/configs/stage1_cache_conus_multiyear_template.json similarity index 50% rename from training/configs/stage1_cache_regional_hrrr_ca_5km_l12_template.json rename to training/configs/stage1_cache_conus_multiyear_template.json index 03d1f99d9ed21e6f9eab2e6631dd390bdc2e90aa..50cabe817b1902ecdbc738407ee2b1f5a6d3a850 100644 --- a/training/configs/stage1_cache_regional_hrrr_ca_5km_l12_template.json +++ b/training/configs/stage1_cache_conus_multiyear_template.json @@ -1,14 +1,38 @@ { - "cache_root": "/path/to/local/firewx_fm_cache/regional_hrrr_ca_5km_l12", - "firms_dir": "/path/to/firms_three_hourly_csv", + "cache_root": "/path/to/local/firewx_fm_cache/conus_hrrr_us_5km_l12_multiyear", + "firms_dir": "/path/to/firms_conus_three_hourly_csv", "hrrr_roots": [ + "/path/to/hrrr/2021", + "/path/to/hrrr/2022", + "/path/to/hrrr/2023", "/path/to/hrrr/2024", - "/path/to/optional/hrrr_backfill" + "/path/to/hrrr/2025", + "/path/to/hrrr/2026" ], "input_ranges": [ { - "start": "2024-06-01", - "end": "2024-10-30" + "start": "2021-03-01", + "end": "2021-11-30" + }, + { + "start": "2022-03-01", + "end": "2022-11-30" + }, + { + "start": "2023-03-01", + "end": "2023-11-30" + }, + { + "start": "2024-03-01", + "end": "2024-11-30" + }, + { + "start": "2025-03-01", + "end": "2025-11-30" + }, + { + "start": "2026-03-01", + "end": "2026-11-30" } ], "issue_hours": [ @@ -18,10 +42,10 @@ 18 ], "target_offset_hours": 12, - "lat_min": 32.0, - "lat_max": 42.5, + "lat_min": 24.0, + "lat_max": 50.0, "lon_min": -125.0, - "lon_max": -114.0, + "lon_max": -66.0, "buffer_deg": 1.0, "target_crs": "EPSG:5070", "target_resolution_m": 5000, @@ -51,21 +75,22 @@ "resampling": "bilinear" }, "population": { - "path": "/path/to/landscan-global-2024.tif", + "path": "/path/to/landscan-global-2024-or-latest.tif", "resampling": "bilinear" } }, - "split_months": { + "split_years": { "train": [ - 6, - 7, - 8 + 2021, + 2022, + 2023, + 2024 ], "val": [ - 9 + 2025 ], "test": [ - 10 + 2026 ] } } diff --git a/training/configs/train_firewx_fm_conus_seed7_template.json b/training/configs/train_firewxfm_conus_multiyear_template.json similarity index 87% rename from training/configs/train_firewx_fm_conus_seed7_template.json rename to training/configs/train_firewxfm_conus_multiyear_template.json index a3a168615bd1463d5d023bcf55f27e0f115c812a..8a29ab072c66594332052f0bc17bc4f57faf70ce 100644 --- a/training/configs/train_firewx_fm_conus_seed7_template.json +++ b/training/configs/train_firewxfm_conus_multiyear_template.json @@ -1,5 +1,5 @@ { - "index_root": "/path/to/local/firewx_fm_cache/conus_hrrr_us_5km_l12", + "index_root": "/path/to/local/firewx_fm_cache/conus_hrrr_us_5km_l12_multiyear", "run_root": "/path/to/local/firewx_fm_runs", "tile_size": 32, "max_positive_tiles_per_sample": 96, @@ -30,9 +30,13 @@ ], "eps": 1e-06 }, + "zero_input_channels": [ + 14, + 15 + ], "batch_size": 32, "eval_batch_size": 1, - "epochs": 10, + "epochs": 20, "learning_rate": 0.0002, "weight_decay": 0.0001, "base_channels": 32, @@ -41,16 +45,18 @@ "norm_groups": 8, "init_head_bias_from_positive_rate": true, "augment_flip": true, - "threshold": 0.5, + "threshold": 0.1, "metric_thresholds": [ + 0.02, + 0.05, 0.1, 0.2, 0.3, - 0.5, - 0.7, - 0.9 + 0.5 ], "topk_area_fractions": [ + 0.001, + 0.005, 0.01, 0.05, 0.1 @@ -61,9 +67,9 @@ 4, 8 ], - "reliability_bins": 10, + "reliability_bins": 15, "amp": true, - "seed": 7, + "seed": 42, "use_aux_spatial_head": true, "aux_spatial_radius": 2, "aux_spatial_loss_weight": 0.5, diff --git a/training/configs/train_firewx_fm_seed7_template.json b/training/configs/train_firewxfm_conus_noexposure_template.json similarity index 54% rename from training/configs/train_firewx_fm_seed7_template.json rename to training/configs/train_firewxfm_conus_noexposure_template.json index ba96dcd4fbd2d7f6801dd3a64de8b377871e4aaa..95805a98d0badb41aee50b0473e641312fa020ad 100644 --- a/training/configs/train_firewx_fm_seed7_template.json +++ b/training/configs/train_firewxfm_conus_noexposure_template.json @@ -1,15 +1,23 @@ { - "index_root": "/path/to/local/firewx_fm_cache/regional_hrrr_ca_5km_l12", + "index_root": "/path/to/local/firewx_fm_cache/conus_hrrr_us_5km_l12", "run_root": "/path/to/local/firewx_fm_runs", "tile_size": 32, - "max_positive_tiles_per_sample": 64, - "min_negative_tiles_per_sample": 4, + "max_positive_tiles_per_sample": 96, + "min_negative_tiles_per_sample": 8, "negative_to_positive_ratio": 3.0, + "positive_tile_placement": "random_containing", "positive_rate_source": "tiles", "pos_weight_source": "tiles", "pos_weight_cap": 50.0, + "input_normalization": { + "enabled": true, + "method": "per-channel z-score", + "stats_source": "train split full maps", + "continuous_channel_indices": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 13, 14, 15], + "eps": 1e-06 + }, "batch_size": 32, - "eval_batch_size": 2, + "eval_batch_size": 1, "epochs": 10, "learning_rate": 0.0002, "weight_decay": 0.0001, @@ -20,38 +28,12 @@ "init_head_bias_from_positive_rate": true, "augment_flip": true, "threshold": 0.5, - "metric_thresholds": [ - 0.1, - 0.2, - 0.3, - 0.5, - 0.7, - 0.9 - ], - "topk_area_fractions": [ - 0.01, - 0.05, - 0.1 - ], - "fss_radii": [ - 1, - 2, - 4, - 8 - ], + "metric_thresholds": [0.1, 0.2, 0.3, 0.5, 0.7, 0.9], + "topk_area_fractions": [0.01, 0.05, 0.1], + "fss_radii": [1, 2, 4, 8], "reliability_bins": 10, "amp": true, - "seed": 7, - "boundary_radii": [ - 1, - 2, - 4 - ], - "coarsen_factors": [ - 2, - 4, - 8 - ], + "seed": 42, "use_aux_spatial_head": true, "aux_spatial_radius": 2, "aux_spatial_loss_weight": 0.5, diff --git a/training/configs/train_firewxfm_multitask_pretrain_template.json b/training/configs/train_firewxfm_multitask_pretrain_template.json new file mode 100644 index 0000000000000000000000000000000000000000..becef34ef1af082974bf533021596dfa38b6434c --- /dev/null +++ b/training/configs/train_firewxfm_multitask_pretrain_template.json @@ -0,0 +1,65 @@ +{ + "status": "design_template_not_currently_supported_by_train_cold_tiled_mainline.py", + "purpose": "Phase-4 FireWx-FM backbone upgrade with multi-lead occupancy and masked-channel reconstruction.", + "index_root": "/path/to/local/firewx_fm_cache/conus_hrrr_us_5km_l12_multiyear", + "run_root": "/path/to/local/firewx_fm_runs", + "shared_backbone": { + "architecture": "unet_encoder_decoder", + "input_channels": 16, + "base_channels": 32, + "norm_type": "group", + "norm_groups": 8, + "dropout": 0.1 + }, + "tasks": { + "occupancy_6h": { + "enabled": true, + "loss": "bce", + "loss_weight": 0.5 + }, + "occupancy_12h": { + "enabled": true, + "loss": "bce", + "loss_weight": 1.0 + }, + "occupancy_24h": { + "enabled": true, + "loss": "bce", + "loss_weight": 0.5 + }, + "spatial_support": { + "enabled": true, + "radius_cells": 2, + "loss": "bce", + "loss_weight": 0.5 + }, + "masked_channel_reconstruction": { + "enabled": true, + "mask_probability": 0.15, + "continuous_channel_indices": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 13 + ], + "loss": "smooth_l1", + "loss_weight": 0.2 + } + }, + "fine_tuning_checks": [ + "linear_or_lightweight_head_on_frozen_encoder", + "low_label_region_finetune", + "held_out_region_finetune" + ], + "notes": [ + "Do not use this template with the current supervised training entrypoint.", + "Implement a separate multi-task trainer before launching Phase 4." + ] +} diff --git a/training/requirements-training.txt b/training/requirements-training.txt deleted file mode 100644 index 9eb31c98f8cd394b1298ad0639d057d4c99da776..0000000000000000000000000000000000000000 --- a/training/requirements-training.txt +++ /dev/null @@ -1,9 +0,0 @@ -numpy -pandas -torch -scikit-learn -scipy -rasterio -xarray -cfgrib -eccodes diff --git a/training/train_cold_tiled_mainline.py b/training/train_cold_tiled_mainline.py index f746e46c57be4c513c555a092a19ab7fdb673151..25e79a5669b7f183931f006a1f1f9154289eff3f 100644 --- a/training/train_cold_tiled_mainline.py +++ b/training/train_cold_tiled_mainline.py @@ -13,6 +13,7 @@ import numpy as np import torch import torch.nn as nn import torch.nn.functional as F +from rasterio.warp import transform as transform_coords from torch.utils.data import DataLoader, Dataset from eval_metrics import metric_bundle @@ -103,9 +104,11 @@ def negative_tile_origins( desired: int, rng: random.Random, forbidden: Iterable[Tuple[int, int]] = (), + allowed_mask: np.ndarray | None = None, ) -> List[Tuple[int, int]]: h, w = mask.shape ii = integral_image(mask) + allowed_ii = integral_image(allowed_mask > 0.5) if allowed_mask is not None else None forbidden_set = set(forbidden) out: List[Tuple[int, int]] = [] seen = set(forbidden_set) @@ -120,6 +123,8 @@ def negative_tile_origins( origin = (top, left) if origin in seen: continue + if allowed_ii is not None and rect_sum(allowed_ii, top, left, tile_h, tile_w) != tile_h * tile_w: + continue if rect_sum(ii, top, left, tile_h, tile_w) != 0: continue seen.add(origin) @@ -321,9 +326,17 @@ def load_or_compute_input_normalization_stats( class ColdFeatureStore: - def __init__(self, rows: List[Dict[str, str]], normalization_stats: Dict[str, object] | None = None): + def __init__( + self, + rows: List[Dict[str, str]], + normalization_stats: Dict[str, object] | None = None, + zero_input_channels: Iterable[int] = (), + input_semantics: Dict[str, object] | None = None, + ): self.cache: Dict[str, Dict[str, np.ndarray]] = {} self.normalization_stats = normalization_stats + self.zero_input_channels = sorted({int(idx) for idx in zero_input_channels}) + self.input_semantics = input_semantics or {} static_path = Path(rows[0]["static_path"]) static_npz = np.load(static_path, allow_pickle=True) static = self._sanitize(static_npz["static"].astype(np.float32)) @@ -341,7 +354,9 @@ class ColdFeatureStore: if "firewx_valid" in sample: extra.append(sample["firewx_valid"].astype(np.float32)) x = np.concatenate([weather, firewx, *extra, static_x], axis=0).astype(np.float32) + x = self._apply_input_semantics(x) x = self._apply_normalization(x) + x = self._apply_zero_channels(x) y = np.nan_to_num(sample["y_occ"].astype(np.float32), nan=0.0, posinf=0.0, neginf=0.0) self.cache[str(row["sample_id"])] = {"x": x, "y": y} @@ -363,6 +378,45 @@ class ColdFeatureStore: x[idx] = (x[idx] - mean) / std return x.astype(np.float32, copy=False) + def _apply_input_semantics(self, x: np.ndarray) -> np.ndarray: + """Apply optional train/serve semantic fixes before normalization. + + These switches are disabled by default. They let refinement runs test + whether suppressing low static-valid context improves coastal/eastern + generalization without changing the public 16-channel contract. + """ + if not self.input_semantics: + return x + static_valid_floor = self.input_semantics.get("static_valid_floor") + if static_valid_floor is not None: + idx = int(self.input_semantics.get("static_valid_channel", 11)) + if 0 <= idx < x.shape[0]: + floor = float(static_valid_floor) + x[idx] = np.where(np.isfinite(x[idx]), np.maximum(x[idx], floor), floor).astype(np.float32) + fuel_zero_replacement = self.input_semantics.get("fuel_zero_replacement") + if fuel_zero_replacement is not None: + fuel_idx = int(self.input_semantics.get("fuel_channel", 12)) + static_idx = int(self.input_semantics.get("static_valid_channel", 11)) + threshold = float(self.input_semantics.get("fuel_zero_static_valid_max", 0.25)) + if 0 <= fuel_idx < x.shape[0]: + fuel = x[fuel_idx] + if 0 <= static_idx < x.shape[0]: + replace = (fuel == 0.0) & (x[static_idx] <= threshold) + else: + replace = fuel == 0.0 + x[fuel_idx] = np.where(replace, float(fuel_zero_replacement), fuel).astype(np.float32) + return x.astype(np.float32, copy=False) + + def semantic_summary(self) -> Dict[str, object]: + return dict(self.input_semantics) + + def _apply_zero_channels(self, x: np.ndarray) -> np.ndarray: + for idx in self.zero_input_channels: + if idx < 0 or idx >= x.shape[0]: + continue + x[idx] = 0.0 + return x.astype(np.float32, copy=False) + def get(self, sample_id: str) -> Dict[str, np.ndarray]: return self.cache[sample_id] @@ -647,6 +701,107 @@ def select_stats(source: str, train_rows: List[Dict[str, str]], tile_rows: List[ raise ValueError(f"Unsupported stats source: {source}") +def load_builtin_region_masks(config: Dict[str, object], sample_row: Dict[str, str]) -> Dict[str, np.ndarray]: + bboxes = config.get("region_masks_bboxes", {}) + if not isinstance(bboxes, dict) or not bboxes: + return {} + sample = np.load(sample_row["sample_path"], allow_pickle=True) + y_desc = sample["lat"].astype(np.float64) + x_asc = sample["lon"].astype(np.float64) + crs_values = sample.get("coord_crs", np.array(["EPSG:5070"], dtype=object)).tolist() + source_crs = str(crs_values[0] if isinstance(crs_values, list) else crs_values) + xx, yy = np.meshgrid(x_asc, y_desc) + lon_values, lat_values = transform_coords(source_crs, "EPSG:4326", xx.ravel().tolist(), yy.ravel().tolist()) + lon = np.asarray(lon_values, dtype=np.float32).reshape(yy.shape) + lat = np.asarray(lat_values, dtype=np.float32).reshape(yy.shape) + masks: Dict[str, np.ndarray] = {} + for name, spec in bboxes.items(): + if not isinstance(spec, dict): + continue + mask = ( + (lat >= float(spec["lat_min"])) + & (lat <= float(spec["lat_max"])) + & (lon >= float(spec["lon_min"])) + & (lon <= float(spec["lon_max"])) + ) + masks[str(name)] = mask.astype(bool) + return masks + + +def select_train_allowed_mask(config: Dict[str, object], region_masks: Dict[str, np.ndarray]) -> np.ndarray | None: + if not region_masks: + return None + train_region_name = str(config.get("train_region_name", "")).strip() + if train_region_name: + if train_region_name not in region_masks: + raise KeyError(f"train_region_name '{train_region_name}' not found in region_masks_bboxes") + return region_masks[train_region_name].astype(bool) + held_out_region_name = str(config.get("held_out_region_name", "")).strip() + if held_out_region_name: + if held_out_region_name not in region_masks: + raise KeyError(f"held_out_region_name '{held_out_region_name}' not found in region_masks_bboxes") + return (~region_masks[held_out_region_name].astype(bool)) + return None + + +def select_eval_region_masks(config: Dict[str, object], region_masks: Dict[str, np.ndarray]) -> Dict[str, np.ndarray] | None: + if not region_masks: + return None + names = config.get("eval_region_names", []) + if not names: + return region_masks + selected: Dict[str, np.ndarray] = {} + for name in names: + key = str(name) + if key not in region_masks: + raise KeyError(f"eval_region_names entry '{key}' not found in region_masks_bboxes") + selected[key] = region_masks[key] + return selected + + +def filter_origins_fully_allowed( + origins: List[Tuple[int, int]], + allowed_mask: np.ndarray | None, + tile_h: int, + tile_w: int, +) -> List[Tuple[int, int]]: + if allowed_mask is None: + return origins + allowed_ii = integral_image(allowed_mask > 0.5) + return [ + (top, left) + for top, left in origins + if rect_sum(allowed_ii, int(top), int(left), tile_h, tile_w) == tile_h * tile_w + ] + + +def add_tile_rows( + tile_rows: List[Dict[str, object]], + origins: List[Tuple[int, int]], + *, + sample_id: str, + tile_type: str, + tile_size: int, + tile_region: str = "", + source: str = "base", + repeat: int = 1, +) -> None: + repeats = max(int(repeat), 1) + for top, left in origins: + for _ in range(repeats): + tile_rows.append( + { + "sample_id": sample_id, + "tile_type": tile_type, + "tile_region": tile_region, + "tile_source": source, + "top": int(top), + "left": int(left), + "tile_size": int(tile_size), + } + ) + + def build_train_tiles( train_rows: List[Dict[str, str]], store: ColdFeatureStore, @@ -656,19 +811,32 @@ def build_train_tiles( neg_pos_ratio: float, positive_tile_placement: str, rng: random.Random, + allowed_mask: np.ndarray | None = None, + region_masks: Dict[str, np.ndarray] | None = None, + region_positive_tile_caps: Dict[str, int] | None = None, + region_positive_tile_repeats: Dict[str, int] | None = None, + region_negative_to_positive_ratio: Dict[str, float] | None = None, + region_min_negative_tiles_per_sample: Dict[str, int] | None = None, ) -> List[Dict[str, object]]: tile_rows: List[Dict[str, object]] = [] + region_masks = region_masks or {} + region_positive_tile_caps = region_positive_tile_caps or {} + region_positive_tile_repeats = region_positive_tile_repeats or {} + region_negative_to_positive_ratio = region_negative_to_positive_ratio or {} + region_min_negative_tiles_per_sample = region_min_negative_tiles_per_sample or {} for row in train_rows: sample_id = str(row["sample_id"]) mask = store.get(sample_id)["y"][0] + positive_mask = mask if allowed_mask is None else mask * allowed_mask.astype(mask.dtype) pos_origins = positive_tile_origins( - mask=mask, + mask=positive_mask, tile_h=tile_size, tile_w=tile_size, max_tiles=max_positive_tiles_per_sample, rng=rng, placement=positive_tile_placement, ) + pos_origins = filter_origins_fully_allowed(pos_origins, allowed_mask, tile_size, tile_size) neg_count = max(min_negative_tiles_per_sample, int(math.ceil(len(pos_origins) * neg_pos_ratio))) neg_origins = negative_tile_origins( mask=mask, @@ -677,30 +845,77 @@ def build_train_tiles( desired=neg_count, rng=rng, forbidden=pos_origins, + allowed_mask=allowed_mask, ) - for top, left in pos_origins: - tile_rows.append( - { - "sample_id": sample_id, - "tile_type": "positive", - "top": int(top), - "left": int(left), - "tile_size": int(tile_size), - } + add_tile_rows(tile_rows, pos_origins, sample_id=sample_id, tile_type="positive", tile_size=tile_size) + add_tile_rows(tile_rows, neg_origins, sample_id=sample_id, tile_type="negative", tile_size=tile_size) + + forbidden = list(pos_origins) + list(neg_origins) + for region_name, cap in region_positive_tile_caps.items(): + if int(cap) <= 0 or region_name not in region_masks: + continue + region_mask = region_masks[region_name].astype(bool) + if allowed_mask is not None: + region_train_mask = region_mask & allowed_mask.astype(bool) + else: + region_train_mask = region_mask + if not region_train_mask.any(): + continue + region_positive_mask = mask * region_train_mask.astype(mask.dtype) + region_pos_origins = positive_tile_origins( + mask=region_positive_mask, + tile_h=tile_size, + tile_w=tile_size, + max_tiles=int(cap), + rng=rng, + placement=positive_tile_placement, ) - for top, left in neg_origins: - tile_rows.append( - { - "sample_id": sample_id, - "tile_type": "negative", - "top": int(top), - "left": int(left), - "tile_size": int(tile_size), - } + region_pos_origins = filter_origins_fully_allowed(region_pos_origins, allowed_mask, tile_size, tile_size) + repeat = int(region_positive_tile_repeats.get(region_name, 1)) + add_tile_rows( + tile_rows, + region_pos_origins, + sample_id=sample_id, + tile_type="positive", + tile_size=tile_size, + tile_region=region_name, + source=f"region_positive:{region_name}", + repeat=repeat, + ) + forbidden.extend(region_pos_origins) + region_neg_ratio = float(region_negative_to_positive_ratio.get(region_name, neg_pos_ratio)) + region_min_neg = int(region_min_negative_tiles_per_sample.get(region_name, min_negative_tiles_per_sample)) + region_neg_count = max(region_min_neg, int(math.ceil(len(region_pos_origins) * region_neg_ratio))) + region_neg_origins = negative_tile_origins( + mask=mask, + tile_h=tile_size, + tile_w=tile_size, + desired=region_neg_count, + rng=rng, + forbidden=forbidden, + allowed_mask=region_train_mask, ) + add_tile_rows( + tile_rows, + region_neg_origins, + sample_id=sample_id, + tile_type="negative", + tile_size=tile_size, + tile_region=region_name, + source=f"region_negative:{region_name}", + ) + forbidden.extend(region_neg_origins) return tile_rows +def count_tile_rows(tile_rows: List[Dict[str, object]], key: str) -> Dict[str, int]: + counts: Dict[str, int] = {} + for row in tile_rows: + value = str(row.get(key, "") or "base") + counts[value] = counts.get(value, 0) + 1 + return dict(sorted(counts.items())) + + class TrainTileDataset(Dataset): def __init__(self, tile_rows: List[Dict[str, object]], store: ColdFeatureStore, augment_flip: bool, seed: int): self.rows = tile_rows @@ -754,6 +969,7 @@ def evaluate( reference_positive_rate: float, criterion: nn.Module, amp: bool, + region_masks: Dict[str, np.ndarray] | None = None, ) -> Dict[str, object]: model.eval() total_loss = 0.0 @@ -783,6 +999,7 @@ def evaluate( fss_radii=fss_radii, n_bins=n_bins, reference_positive_rate=reference_positive_rate, + region_masks=region_masks, ) metrics.update({"loss": total_loss / max(total_items, 1), "num_samples": int(prob_maps.shape[0])}) return metrics @@ -824,7 +1041,16 @@ def main() -> None: val_rows = read_rows(index_root / "splits" / "val.csv") test_rows = read_rows(index_root / "splits" / "test.csv") normalization_stats = load_or_compute_input_normalization_stats(config, train_rows, metric_dir) - store = ColdFeatureStore(train_rows + val_rows + test_rows, normalization_stats=normalization_stats) + zero_input_channels = [int(v) for v in config.get("zero_input_channels", [])] + store = ColdFeatureStore( + train_rows + val_rows + test_rows, + normalization_stats=normalization_stats, + zero_input_channels=zero_input_channels, + input_semantics=config.get("input_semantics") if isinstance(config.get("input_semantics"), dict) else None, + ) + region_masks = load_builtin_region_masks(config, train_rows[0]) + train_allowed_mask = select_train_allowed_mask(config, region_masks) + eval_region_masks = select_eval_region_masks(config, region_masks) tile_rows = build_train_tiles( train_rows=train_rows, @@ -835,6 +1061,12 @@ def main() -> None: neg_pos_ratio=float(config.get("negative_to_positive_ratio", 2.0)), positive_tile_placement=str(config.get("positive_tile_placement", "center")), rng=rng, + allowed_mask=train_allowed_mask, + region_masks=region_masks, + region_positive_tile_caps={str(k): int(v) for k, v in dict(config.get("region_positive_tile_caps", {})).items()}, + region_positive_tile_repeats={str(k): int(v) for k, v in dict(config.get("region_positive_tile_repeats", {})).items()}, + region_negative_to_positive_ratio={str(k): float(v) for k, v in dict(config.get("region_negative_to_positive_ratio", {})).items()}, + region_min_negative_tiles_per_sample={str(k): int(v) for k, v in dict(config.get("region_min_negative_tiles_per_sample", {})).items()}, ) train_ds = TrainTileDataset(tile_rows=tile_rows, store=store, augment_flip=bool(config.get("augment_flip", True)), seed=int(config.get("seed", 7))) val_ds = FullMapDataset(val_rows, store) @@ -893,6 +1125,8 @@ def main() -> None: "num_train_tiles": len(tile_rows), "positive_train_tiles": int(sum(1 for row in tile_rows if row["tile_type"] == "positive")), "negative_train_tiles": int(sum(1 for row in tile_rows if row["tile_type"] == "negative")), + "train_tile_counts_by_region": count_tile_rows(tile_rows, "tile_region"), + "train_tile_counts_by_source": count_tile_rows(tile_rows, "tile_source"), "input_channels": in_ch, "positive_tile_placement": str(config.get("positive_tile_placement", "center")), "train_positive_rate": train_positive_rate, @@ -904,6 +1138,15 @@ def main() -> None: "train_target_mode": str(config.get("train_target_mode", "hard")), "train_target_radius": int(config.get("train_target_radius", 0)), "input_normalization": normalization_stats if normalization_stats else {"enabled": False}, + "zero_input_channels": zero_input_channels, + "input_semantics": store.semantic_summary(), + "region_positive_tile_caps": dict(config.get("region_positive_tile_caps", {})), + "region_positive_tile_repeats": dict(config.get("region_positive_tile_repeats", {})), + "region_negative_to_positive_ratio": dict(config.get("region_negative_to_positive_ratio", {})), + "held_out_region_name": str(config.get("held_out_region_name", "")), + "train_region_name": str(config.get("train_region_name", "")), + "region_mask_cells": {name: int(mask.sum()) for name, mask in region_masks.items()}, + "train_allowed_mask_cells": int(train_allowed_mask.sum()) if train_allowed_mask is not None else None, } (metric_dir / "tile_summary.json").write_text(json.dumps(summary_seed, indent=2), encoding="utf-8") @@ -952,6 +1195,7 @@ def main() -> None: reference_positive_rate=metric_reference_positive_rate, criterion=criterion, amp=bool(config.get("amp", True)), + region_masks=eval_region_masks, ) threshold_metrics = metrics["threshold_metrics"][default_threshold_key] record = { @@ -976,6 +1220,7 @@ def main() -> None: "val_threshold_metrics": metrics["threshold_metrics"], "val_topk_area_metrics": metrics["topk_area_metrics"], "val_fss": metrics["fss"], + "val_region_metrics": metrics.get("region_metrics", {}), "minutes": (time.time() - epoch_start) / 60.0, } history.append(record) @@ -1001,6 +1246,7 @@ def main() -> None: reference_positive_rate=metric_reference_positive_rate, criterion=criterion, amp=bool(config.get("amp", True)), + region_masks=eval_region_masks, ) test_best = evaluate( model=model, @@ -1013,6 +1259,7 @@ def main() -> None: reference_positive_rate=metric_reference_positive_rate, criterion=criterion, amp=bool(config.get("amp", True)), + region_masks=eval_region_masks, ) summary = { "run_name": args.run_name, @@ -1023,6 +1270,8 @@ def main() -> None: "num_train_tiles": len(tile_rows), "positive_train_tiles": int(sum(1 for row in tile_rows if row["tile_type"] == "positive")), "negative_train_tiles": int(sum(1 for row in tile_rows if row["tile_type"] == "negative")), + "train_tile_counts_by_region": count_tile_rows(tile_rows, "tile_region"), + "train_tile_counts_by_source": count_tile_rows(tile_rows, "tile_source"), "positive_tile_placement": str(config.get("positive_tile_placement", "center")), "train_positive_rate": train_positive_rate, "metric_reference_positive_rate": metric_reference_positive_rate, @@ -1033,6 +1282,11 @@ def main() -> None: "aux_positive_rate": aux_positive_rate, "init_checkpoint": str(init_checkpoint) if init_checkpoint else "", "input_normalization": normalization_stats if normalization_stats else {"enabled": False}, + "zero_input_channels": zero_input_channels, + "input_semantics": store.semantic_summary(), + "region_positive_tile_caps": dict(config.get("region_positive_tile_caps", {})), + "region_positive_tile_repeats": dict(config.get("region_positive_tile_repeats", {})), + "region_negative_to_positive_ratio": dict(config.get("region_negative_to_positive_ratio", {})), "best_val_pr_auc": float(val_best["pr_auc"]), "best_val_auroc": float(val_best["auroc"]), "best_test_pr_auc": float(test_best["pr_auc"]), @@ -1042,6 +1296,8 @@ def main() -> None: "best_test_topk_area_metrics": test_best["topk_area_metrics"], "best_test_threshold_metrics": test_best["threshold_metrics"], "best_test_fss": test_best["fss"], + "best_val_region_metrics": val_best.get("region_metrics", {}), + "best_test_region_metrics": test_best.get("region_metrics", {}), "epochs": int(config.get("epochs", 30)), } (metric_dir / "run_summary.json").write_text(json.dumps(summary, indent=2), encoding="utf-8")