Update license year, refine README content, and enhance requirements for better clarity and organization

.gitignore

@@ -131,7 +131,7 @@ cython_debug/
 #  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
-#.idea/
+.idea/
 
 # Ruff stuff:
 .ruff_cache/
@@ -139,7 +139,7 @@ cython_debug/
 # PyPI configuration file
 .pypirc
 
-.vscode/sftp.json
+.vscode/
 
 .DS_Store
 
@@ -151,12 +151,7 @@ wandb/
 
 *.csv
 
-notebooks_tests/
-
-/.idea/
-/.vscode/
-
 *.code-workspace
-.idea
-.xml
-.iml
+
+.claude/
+misc/

.idea/FOXES.iml

@@ -1,18 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<module type="PYTHON_MODULE" version="4">
-  <component name="NewModuleRootManager">
-    <content url="file://$MODULE_DIR$">
-      <sourceFolder url="file://$MODULE_DIR$" isTestSource="false" />
-      <excludePattern pattern="*/T9/*" />
-    </content>
-    <content url="file:///Volumes/T9">
-      <excludeFolder url="file:///Volumes/T9/FOXES_Data" />
-    </content>
-    <orderEntry type="jdk" jdkName="foxes" jdkType="Python SDK" />
-    <orderEntry type="sourceFolder" forTests="false" />
-  </component>
-  <component name="PyDocumentationSettings">
-    <option name="format" value="NUMPY" />
-    <option name="myDocStringFormat" value="NumPy" />
-  </component>
-</module>

.idea/inspectionProfiles/profiles_settings.xml

@@ -1,6 +0,0 @@
-<component name="InspectionProjectProfileManager">
-  <settings>
-    <option name="USE_PROJECT_PROFILE" value="false" />
-    <version value="1.0" />
-  </settings>
-</component>

.idea/misc.xml

@@ -1,7 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="Black">
-    <option name="sdkName" value="Python 3.11 (FOXES)" />
-  </component>
-  <component name="ProjectRootManager" version="2" project-jdk-name="foxes" project-jdk-type="Python SDK" />
-</project>

.idea/shelf/Uncommitted_changes_before_Checkout_at_7_16_25,_8_08_AM_[Changes]/shelved.patch

@@ -1,23 +0,0 @@
-Index: flaring/vision_transformers/vision_transformer.py
-IDEA additional info:
-Subsystem: com.intellij.openapi.diff.impl.patch.BaseRevisionTextPatchEP
-<+>\nimport torch\nimport torch.nn as nn\nimport pytorch_lightning as pl\nfrom timm import create_model\n\nclass VisionTransformerFlaringModel(pl.LightningModule):\n    def __init__(self, d_input=6, d_output=1, eve_norm=(0, 1), lr=1e-4, model_name='vit_base_patch16_224', pretrained=True):\n        super().__init__()\n        self.save_hyperparameters()\n        self.eve_norm = eve_norm\n        self.lr = lr\n        self.vit = create_model(\n            model_name,\n            pretrained=pretrained,\n            in_chans=d_input,\n            num_classes=0,\n            global_pool='avg'\n        )\n        self.regression_head = nn.Linear(self.vit.embed_dim, d_output)\n        self.loss_func = nn.HuberLoss()\n\n    def forward(self, x):\n        x = x.permute(0, 3, 1, 2)\n        features = self.vit(x)\n        output = self.regression_head(features)\n        return output.squeeze(-1)\n\n    def training_step(self, batch, batch_idx):\n        (aia_img, _), sxr_target = batch\n        sxr_pred = self(aia_img)\n        loss = self.loss_func(sxr_pred, sxr_target)\n        self.log('train_loss', loss, prog_bar=True)\n        return loss\n\n    def validation_step(self, batch, batch_idx):\n        (aia_img, _), sxr_target = batch\n        sxr_pred = self(aia_img)\n        loss = self.loss_func(sxr_pred, sxr_target)\n        self.log('valid_loss', loss, prog_bar=True)\n        return loss\n\n    def test_step(self, batch, batch_idx):\n        (aia_img, _), sxr_target = batch\n        sxr_pred = self(aia_img)\n        loss = self.loss_func(sxr_pred, sxr_target)\n        self.log('test_loss', loss, prog_bar=True)\n        return loss\n\n    def configure_optimizers(self):\n        optimizer = torch.optim.AdamW(self.parameters(), lr=self.lr, weight_decay=0.01)\n        scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=10)\n        return {\n            'optimizer': optimizer,\n            'lr_scheduler': {\n                'scheduler': scheduler,\n                'monitor': 'valid_loss',\n                'interval': 'epoch',\n                'frequency': 1\n            }\n        }\n\n    def predict_step(self, batch, batch_idx, dataloader_idx=0):\n        (aia_img, _), _ = batch\n        sxr_pred = self(aia_img)\n        sxr_pred = sxr_pred * self.eve_norm[1] + self.eve_norm[0]\n        sxr_pred = 10 ** sxr_pred - 1e-8\n        return sxr_pred\n
-===================================================================
-diff --git a/flaring/vision_transformers/vision_transformer.py b/flaring/vision_transformers/vision_transformer.py
---- a/flaring/vision_transformers/vision_transformer.py	(revision bb90fb79a516910ac93442850698317720adf6d5)
-+++ b/flaring/vision_transformers/vision_transformer.py	(date 1752678502848)
-@@ -5,7 +5,7 @@
- from timm import create_model
- 
- class VisionTransformerFlaringModel(pl.LightningModule):
--    def __init__(self, d_input=6, d_output=1, eve_norm=(0, 1), lr=1e-4, model_name='vit_base_patch16_224', pretrained=True):
-+    def __init__(self, d_input=6, d_output=1, eve_norm=(0, 1), lr=1e-4, model_name='vit_small_patch16_224', pretrained=True):
-         super().__init__()
-         self.save_hyperparameters()
-         self.eve_norm = eve_norm
-@@ -66,3 +66,5 @@
-         sxr_pred = sxr_pred * self.eve_norm[1] + self.eve_norm[0]
-         sxr_pred = 10 ** sxr_pred - 1e-8
-         return sxr_pred
-+
-+

.idea/vcs.xml

@@ -1,6 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="VcsDirectoryMappings">
-    <mapping directory="" vcs="Git" />
-  </component>
-</project>

.idea/workspace.xml

@@ -1,677 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="AutoImportSettings">
-    <option name="autoReloadType" value="SELECTIVE" />
-  </component>
-  <component name="ChangeListManager">
-    <list default="true" id="e883d810-3b4e-470d-b72e-9ed3eee941a0" name="Changes" comment="Enhance plotting themes for poster analysis, refine regression plot scaling, and add data cleanup utilities&#10;&#10;- Updated flare_analysis_poster.py to use white text and spines for improved visibility on dark backgrounds, adjusted legend and grid styling, and improved annotation contrast.&#10;- Refined regression plot scaling in evaluation.py: enforced realistic log-scale bounds, added buffer for axis limits, and improved color mapping for 2D histograms.&#10;- Added cleanup_data.py and cleanup_data_mp.py scripts for identifying and moving extraneous .npy files not present in train/val CSVs, with multiprocessing support.&#10;- Introduced processing_utils.py with functions for flare classification, parallel file processing, and data summarization.&#10;- Adjusted local_config.yaml time range and updated FOXES.iml data folder structure.">
-      <change afterPath="$PROJECT_DIR$/notebook_tests/PlottingResults.ipynb" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/forecasting/inference/evaluation.py" beforeDir="false" afterPath="$PROJECT_DIR$/forecasting/inference/evaluation.py" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/notebook_tests/Additional_Results_Analysis.ipynb" beforeDir="false" afterPath="$PROJECT_DIR$/notebook_tests/Additional_Results_Analysis.ipynb" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/notebook_tests/Additional_Results_Investigation_Download.ipynb" beforeDir="false" afterPath="$PROJECT_DIR$/notebook_tests/Additional_Results_Investigation_Download.ipynb" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/notebook_tests/processing_utils.py" beforeDir="false" afterPath="$PROJECT_DIR$/notebook_tests/processing_utils.py" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/notebook_tests/timelineplot.ipynb" beforeDir="false" afterPath="$PROJECT_DIR$/notebook_tests/timelineplot.ipynb" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/notebook_tests/weirdsxr.ipynb" beforeDir="false" afterPath="$PROJECT_DIR$/notebook_tests/weirdsxr.ipynb" afterDir="false" />
-    </list>
-    <option name="SHOW_DIALOG" value="false" />
-    <option name="HIGHLIGHT_CONFLICTS" value="true" />
-    <option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
-    <option name="LAST_RESOLUTION" value="IGNORE" />
-  </component>
-  <component name="CopilotPersistence">
-    <persistenceIdMap>
-      <entry key="_/Users/griffingoodwin/Documents/gitrepos/FOXES" value="38WeoPd73qtvhPoxz6TM6eycX2x" />
-      <entry key="_/Volumes/T9" value="38a78qcQSOmwaRego2V2zivIuLc" />
-      <entry key="_/Volumes/T9/FOXES_Data" value="300pTKGqQDgkIrymxlGIrm6ZrO3" />
-    </persistenceIdMap>
-  </component>
-  <component name="EmbeddingIndexingInfo">
-    <option name="cachedIndexableFilesCount" value="109" />
-    <option name="fileBasedEmbeddingIndicesEnabled" value="true" />
-  </component>
-  <component name="FileTemplateManagerImpl">
-    <option name="RECENT_TEMPLATES">
-      <list>
-        <option value="Python Script" />
-        <option value="Jupyter Notebook" />
-      </list>
-    </option>
-  </component>
-  <component name="Git.Settings">
-    <option name="RECENT_BRANCH_BY_REPOSITORY">
-      <map>
-        <entry key="$PROJECT_DIR$" value="flaring" />
-      </map>
-    </option>
-    <option name="RECENT_GIT_ROOT_PATH" value="$PROJECT_DIR$" />
-  </component>
-  <component name="GitHubPullRequestSearchHistory">{
-  &quot;lastFilter&quot;: {}
-}</component>
-  <component name="GithubPullRequestsUISettings">{
-  &quot;selectedUrlAndAccountId&quot;: {
-    &quot;url&quot;: &quot;git@github.com:griffin-goodwin/FOXES.git&quot;,
-    &quot;accountId&quot;: &quot;dd5815ef-d79a-4f68-9de1-d483f99a50dd&quot;
-  }
-}</component>
-  <component name="McpProjectServerCommands">
-    <commands />
-    <urls />
-  </component>
-  <component name="NamedScopeManager">
-    <scope name="hide ._" pattern="!*.___*" />
-  </component>
-  <component name="ProjectColorInfo">{
-  &quot;associatedIndex&quot;: 8
-}</component>
-  <component name="ProjectId" id="300pTKGqQDgkIrymxlGIrm6ZrO3" />
-  <component name="ProjectViewState">
-    <option name="hideEmptyMiddlePackages" value="true" />
-    <option name="showLibraryContents" value="true" />
-    <option name="sortKey" value="BY_TIME_DESCENDING" />
-  </component>
-  <component name="PropertiesComponent">{
-  &quot;keyToString&quot;: {
-    &quot;ModuleVcsDetector.initialDetectionPerformed&quot;: &quot;true&quot;,
-    &quot;RunOnceActivity.MCP Project settings loaded&quot;: &quot;true&quot;,
-    &quot;RunOnceActivity.ShowReadmeOnStart&quot;: &quot;true&quot;,
-    &quot;RunOnceActivity.TerminalTabsStorage.copyFrom.TerminalArrangementManager.252&quot;: &quot;true&quot;,
-    &quot;RunOnceActivity.git.unshallow&quot;: &quot;true&quot;,
-    &quot;RunOnceActivity.typescript.service.memoryLimit.init&quot;: &quot;true&quot;,
-    &quot;WebServerToolWindowPanel.toolwindow.highlight.mappings&quot;: &quot;true&quot;,
-    &quot;WebServerToolWindowPanel.toolwindow.highlight.symlinks&quot;: &quot;true&quot;,
-    &quot;WebServerToolWindowPanel.toolwindow.show.date&quot;: &quot;false&quot;,
-    &quot;WebServerToolWindowPanel.toolwindow.show.permissions&quot;: &quot;false&quot;,
-    &quot;WebServerToolWindowPanel.toolwindow.show.size&quot;: &quot;false&quot;,
-    &quot;ai.playground.ignore.import.keys.banner.in.settings&quot;: &quot;true&quot;,
-    &quot;com.google.cloudcode.ide_session_index&quot;: &quot;20251029_0000&quot;,
-    &quot;com.intellij.ml.llm.matterhorn.ej.ui.settings.DefaultModelSelectionForGA.v1&quot;: &quot;true&quot;,
-    &quot;database.data.extractors.current.export.id&quot;: &quot;Comma-separated (CSV)_id&quot;,
-    &quot;git-widget-placeholder&quot;: &quot;main&quot;,
-    &quot;junie.onboarding.icon.badge.shown&quot;: &quot;true&quot;,
-    &quot;last_opened_file_path&quot;: &quot;/Users/griffingoodwin/Documents/gitrepos/FOXES&quot;,
-    &quot;node.js.detected.package.eslint&quot;: &quot;true&quot;,
-    &quot;node.js.selected.package.eslint&quot;: &quot;(autodetect)&quot;,
-    &quot;node.js.selected.package.tslint&quot;: &quot;(autodetect)&quot;,
-    &quot;nodejs_package_manager_path&quot;: &quot;npm&quot;,
-    &quot;settings.editor.selected.configurable&quot;: &quot;jupyter.general.settings&quot;,
-    &quot;to.speed.mode.migration.done&quot;: &quot;true&quot;,
-    &quot;vue.rearranger.settings.migration&quot;: &quot;true&quot;
-  }
-}</component>
-  <component name="RecapSpentCounter">
-    <option name="endsOfQuotaMs" value="1768932019187" />
-    <option name="spentUsd" value="0.032168448" />
-  </component>
-  <component name="RecapUselessUpdatesCounter">
-    <option name="suspendCountdown" value="0" />
-  </component>
-  <component name="RecentsManager">
-    <key name="MoveFile.RECENT_KEYS">
-      <recent name="$PROJECT_DIR$/flaring/forecasting/training" />
-      <recent name="$PROJECT_DIR$/flaring/forecasting/models" />
-      <recent name="$PROJECT_DIR$/flaring/forecasting/data_loaders" />
-      <recent name="$PROJECT_DIR$/flaring/Forecasting/inference" />
-      <recent name="$PROJECT_DIR$" />
-    </key>
-  </component>
-  <component name="SharedIndexes">
-    <attachedChunks>
-      <set>
-        <option value="bundled-js-predefined-d6986cc7102b-9b0f141eb926-JavaScript-PY-253.29346.142" />
-        <option value="bundled-python-sdk-f2b7a9f6281b-6e1f45a539f7-com.jetbrains.pycharm.pro.sharedIndexes.bundled-PY-253.29346.142" />
-      </set>
-    </attachedChunks>
-  </component>
-  <component name="SpellCheckerSettings" RuntimeDictionaries="0" Folders="0" CustomDictionaries="0" DefaultDictionary="application-level" UseSingleDictionary="true" transferred="true" />
-  <component name="SvnConfiguration">
-    <configuration>$USER_HOME$/.subversion</configuration>
-    <option name="sshConnectionType" value="PRIVATE_KEY" />
-    <option name="sshUserName" value="griffin-goodwin" />
-    <option name="sshPrivateKeyPath" value="$USER_HOME$/.ssh" />
-  </component>
-  <component name="TaskManager">
-    <task active="true" id="Default" summary="Default task">
-      <changelist id="e883d810-3b4e-470d-b72e-9ed3eee941a0" name="Changes" comment="fixed wandb name scheme" />
-      <created>1752774351420</created>
-      <option name="number" value="Default" />
-      <option name="presentableId" value="Default" />
-      <updated>1752774351420</updated>
-      <workItem from="1752774352707" duration="28648000" />
-      <workItem from="1752962518515" duration="2550000" />
-      <workItem from="1753306014729" duration="13973000" />
-      <workItem from="1753374480667" duration="45913000" />
-      <workItem from="1753545524483" duration="2816000" />
-      <workItem from="1753630961668" duration="9897000" />
-      <workItem from="1753929248247" duration="23397000" />
-      <workItem from="1754000190506" duration="10165000" />
-      <workItem from="1754146439867" duration="7692000" />
-      <workItem from="1754282226274" duration="5949000" />
-      <workItem from="1754443347602" duration="1827000" />
-      <workItem from="1754482805314" duration="230000" />
-      <workItem from="1754483043965" duration="195000" />
-      <workItem from="1754483254382" duration="55570000" />
-      <workItem from="1754661837523" duration="32924000" />
-      <workItem from="1754949280985" duration="1853000" />
-      <workItem from="1755012015116" duration="134000" />
-      <workItem from="1755012234233" duration="16222000" />
-      <workItem from="1755110862054" duration="4930000" />
-      <workItem from="1755182186851" duration="7000" />
-      <workItem from="1755182816790" duration="16256000" />
-      <workItem from="1755274543631" duration="1138000" />
-      <workItem from="1755289237970" duration="2241000" />
-      <workItem from="1755526973852" duration="8608000" />
-      <workItem from="1755801805292" duration="58749000" />
-      <workItem from="1756171392559" duration="40222000" />
-      <workItem from="1756308588067" duration="10459000" />
-      <workItem from="1756386184391" duration="64689000" />
-      <workItem from="1756818485332" duration="17000" />
-      <workItem from="1756821188374" duration="7721000" />
-      <workItem from="1765907612776" duration="685000" />
-      <workItem from="1765908307908" duration="552000" />
-      <workItem from="1765909073111" duration="961000" />
-      <workItem from="1765910046969" duration="346000" />
-      <workItem from="1765910410274" duration="468000" />
-      <workItem from="1765915810103" duration="1093000" />
-      <workItem from="1765916926305" duration="367000" />
-      <workItem from="1765917308304" duration="283000" />
-      <workItem from="1765919768282" duration="248000" />
-      <workItem from="1765920350623" duration="2783000" />
-      <workItem from="1765923830409" duration="95000" />
-      <workItem from="1765923932850" duration="4525000" />
-      <workItem from="1766071632234" duration="557000" />
-      <workItem from="1766072202434" duration="3172000" />
-      <workItem from="1767642746610" duration="7711000" />
-      <workItem from="1768318275660" duration="7100000" />
-      <workItem from="1768511581940" duration="1375000" />
-      <workItem from="1768590208522" duration="10356000" />
-      <workItem from="1768855583083" duration="7118000" />
-      <workItem from="1768919929930" duration="31494000" />
-      <workItem from="1769025671291" duration="5485000" />
-      <workItem from="1769032850655" duration="18248000" />
-    </task>
-    <task id="LOCAL-00007" summary="Implement Vision Transformer model (custom) and update configuration for model selection + changed data loader outputs (it was partially redundant)">
-      <option name="closed" value="true" />
-      <created>1753201034025</created>
-      <option name="number" value="00007" />
-      <option name="presentableId" value="LOCAL-00007" />
-      <option name="project" value="LOCAL" />
-      <updated>1753201034025</updated>
-    </task>
-    <task id="LOCAL-00008" summary="Working on attention callbacks">
-      <option name="closed" value="true" />
-      <created>1753203631078</created>
-      <option name="number" value="00008" />
-      <option name="presentableId" value="LOCAL-00008" />
-      <option name="project" value="LOCAL" />
-      <updated>1753203631078</updated>
-    </task>
-    <task id="LOCAL-00009" summary="Refactor attention map callback to log attention plots and add multi-head visualization">
-      <option name="closed" value="true" />
-      <created>1753205974581</created>
-      <option name="number" value="00009" />
-      <option name="presentableId" value="LOCAL-00009" />
-      <option name="project" value="LOCAL" />
-      <updated>1753205974581</updated>
-    </task>
-    <task id="LOCAL-00010" summary="refactor model configuration and callbacks; update data paths and loss functions">
-      <option name="closed" value="true" />
-      <created>1753216904168</created>
-      <option name="number" value="00010" />
-      <option name="presentableId" value="LOCAL-00010" />
-      <option name="project" value="LOCAL" />
-      <updated>1753216904168</updated>
-    </task>
-    <task id="LOCAL-00011" summary="refactor attention visualization; update callback to log attention maps and adjust model configuration">
-      <option name="closed" value="true" />
-      <created>1753221151466</created>
-      <option name="number" value="00011" />
-      <option name="presentableId" value="LOCAL-00011" />
-      <option name="project" value="LOCAL" />
-      <updated>1753221151466</updated>
-    </task>
-    <task id="LOCAL-00012" summary="refactor model configuration; update vit parameters and checkpoint callback monitor">
-      <option name="closed" value="true" />
-      <created>1753223740556</created>
-      <option name="number" value="00012" />
-      <option name="presentableId" value="LOCAL-00012" />
-      <option name="project" value="LOCAL" />
-      <updated>1753223740556</updated>
-    </task>
-    <task id="LOCAL-00013" summary="refactor model configuration and loss functions; update batch size, patch size, and logging metrics">
-      <option name="closed" value="true" />
-      <created>1753280393483</created>
-      <option name="number" value="00013" />
-      <option name="presentableId" value="LOCAL-00013" />
-      <option name="project" value="LOCAL" />
-      <updated>1753280393483</updated>
-    </task>
-    <task id="LOCAL-00014" summary="refactor file structure; rename and reorganize model, callback, and configuration files; added configuration file for inference">
-      <option name="closed" value="true" />
-      <created>1753285792536</created>
-      <option name="number" value="00014" />
-      <option name="presentableId" value="LOCAL-00014" />
-      <option name="project" value="LOCAL" />
-      <updated>1753285792536</updated>
-    </task>
-    <task id="LOCAL-00015" summary="added plotly graphs for inference">
-      <option name="closed" value="true" />
-      <created>1753290195559</created>
-      <option name="number" value="00015" />
-      <option name="presentableId" value="LOCAL-00015" />
-      <option name="project" value="LOCAL" />
-      <updated>1753290195559</updated>
-    </task>
-    <task id="LOCAL-00016" summary="add FastViT model for solar flare prediction; update configuration files and optimize inference batch processing">
-      <option name="closed" value="true" />
-      <created>1753303152301</created>
-      <option name="number" value="00016" />
-      <option name="presentableId" value="LOCAL-00016" />
-      <option name="project" value="LOCAL" />
-      <updated>1753303152301</updated>
-    </task>
-    <task id="LOCAL-00017" summary="refactor model configuration and training scripts; update weight paths, batch size, and learning rate scheduler; add plotting functionality for attention maps">
-      <option name="closed" value="true" />
-      <created>1753380808931</created>
-      <option name="number" value="00017" />
-      <option name="presentableId" value="LOCAL-00017" />
-      <option name="project" value="LOCAL" />
-      <updated>1753380808931</updated>
-    </task>
-    <task id="LOCAL-00018" summary="refactor training and inference scripts; update model configuration, batch size, and weight paths; enhance data loading with prefetching; add support for loading checkpoints, added new weight to loss to mitigate few m / x class points">
-      <option name="closed" value="true" />
-      <created>1753454404812</created>
-      <option name="number" value="00018" />
-      <option name="presentableId" value="LOCAL-00018" />
-      <option name="project" value="LOCAL" />
-      <updated>1753454404812</updated>
-    </task>
-    <task id="LOCAL-00019" summary="update plotting script to change output video name and adjust timestamp generation interval;">
-      <option name="closed" value="true" />
-      <created>1753469024793</created>
-      <option name="number" value="00019" />
-      <option name="presentableId" value="LOCAL-00019" />
-      <option name="project" value="LOCAL" />
-      <updated>1753469024793</updated>
-    </task>
-    <task id="LOCAL-00020" summary="remove unused resizing and normalization code from SDOAIA_dataloader">
-      <option name="closed" value="true" />
-      <created>1753570672548</created>
-      <option name="number" value="00020" />
-      <option name="presentableId" value="LOCAL-00020" />
-      <option name="project" value="LOCAL" />
-      <updated>1753570672548</updated>
-    </task>
-    <task id="LOCAL-00021" summary="add visualization notebook and refactor model configuration; update batch size, patch size, and dropout rate; implement dynamic loss for SXR regression">
-      <option name="closed" value="true" />
-      <created>1753730360460</created>
-      <option name="number" value="00021" />
-      <option name="presentableId" value="LOCAL-00021" />
-      <option name="project" value="LOCAL" />
-      <updated>1753730360460</updated>
-    </task>
-    <task id="LOCAL-00022" summary="update model configuration and training scripts; adjust learning rate, epochs, and data paths; refactor trainer initialization and logging">
-      <option name="closed" value="true" />
-      <created>1753821485670</created>
-      <option name="number" value="00022" />
-      <option name="presentableId" value="LOCAL-00022" />
-      <option name="project" value="LOCAL" />
-      <updated>1753821485670</updated>
-    </task>
-    <task id="LOCAL-00023" summary="update model configuration and training scripts; change selected model to 'ViT Custom', increase epochs to 250, and adjust output paths; refactor model loading in inference to support hybrid models and attention weight saving">
-      <option name="closed" value="true" />
-      <created>1753883159604</created>
-      <option name="number" value="00023" />
-      <option name="presentableId" value="LOCAL-00023" />
-      <option name="project" value="LOCAL" />
-      <updated>1753883159604</updated>
-    </task>
-    <task id="LOCAL-00024" summary="added evaluation script">
-      <option name="closed" value="true" />
-      <created>1753901835422</created>
-      <option name="number" value="00024" />
-      <option name="presentableId" value="LOCAL-00024" />
-      <option name="project" value="LOCAL" />
-      <updated>1753901835422</updated>
-    </task>
-    <task id="LOCAL-00025" summary="update model configuration and evaluation scripts; adjust checkpoint paths, output file names, and weight paths; enhance evaluation metrics for flare classes and improve visualization aesthetics">
-      <option name="closed" value="true" />
-      <created>1754083359321</created>
-      <option name="number" value="00025" />
-      <option name="presentableId" value="LOCAL-00025" />
-      <option name="project" value="LOCAL" />
-      <updated>1754083359321</updated>
-    </task>
-    <task id="LOCAL-00026" summary="updated model and data loaders to take in different wavelengths!">
-      <option name="closed" value="true" />
-      <created>1754149724483</created>
-      <option name="number" value="00026" />
-      <option name="presentableId" value="LOCAL-00026" />
-      <option name="project" value="LOCAL" />
-      <updated>1754149724484</updated>
-    </task>
-    <task id="LOCAL-00027" summary="refactor model evaluation and configuration to include sxr norm; update model name to 'ViT',">
-      <option name="closed" value="true" />
-      <created>1754154137848</created>
-      <option name="number" value="00027" />
-      <option name="presentableId" value="LOCAL-00027" />
-      <option name="project" value="LOCAL" />
-      <updated>1754154137848</updated>
-    </task>
-    <task id="LOCAL-00028" summary="Changed weights for ViT">
-      <option name="closed" value="true" />
-      <created>1754319038616</created>
-      <option name="number" value="00028" />
-      <option name="presentableId" value="LOCAL-00028" />
-      <option name="project" value="LOCAL" />
-      <updated>1754319038616</updated>
-    </task>
-    <task id="LOCAL-00029" summary="Enhance AIA_GOESDataset to support cadence and reference time">
-      <option name="closed" value="true" />
-      <created>1754412270354</created>
-      <option name="number" value="00029" />
-      <option name="presentableId" value="LOCAL-00029" />
-      <option name="project" value="LOCAL" />
-      <updated>1754412270354</updated>
-    </task>
-    <task id="LOCAL-00030" summary="Add oversampling functionality to AIA_GOESDataset and update configuration">
-      <option name="closed" value="true" />
-      <created>1754417745022</created>
-      <option name="number" value="00030" />
-      <option name="presentableId" value="LOCAL-00030" />
-      <option name="project" value="LOCAL" />
-      <updated>1754417745022</updated>
-    </task>
-    <task id="LOCAL-00031" summary="Update AIA wavelengths and model configuration in config.yaml; adjust oversampling setting in dataloader">
-      <option name="closed" value="true" />
-      <created>1754426834672</created>
-      <option name="number" value="00031" />
-      <option name="presentableId" value="LOCAL-00031" />
-      <option name="project" value="LOCAL" />
-      <updated>1754426834672</updated>
-    </task>
-    <task id="LOCAL-00032" summary="added some checks to make sure model training and validation is successful">
-      <option name="closed" value="true" />
-      <created>1754431311188</created>
-      <option name="number" value="00032" />
-      <option name="presentableId" value="LOCAL-00032" />
-      <option name="project" value="LOCAL" />
-      <updated>1754431311188</updated>
-    </task>
-    <task id="LOCAL-00033" summary="updated the weighted loss to return proper multiplication values... also reduced weighting slightly">
-      <option name="closed" value="true" />
-      <created>1754482944827</created>
-      <option name="number" value="00033" />
-      <option name="presentableId" value="LOCAL-00033" />
-      <option name="project" value="LOCAL" />
-      <updated>1754482944827</updated>
-    </task>
-    <task id="LOCAL-00034" summary="updated the weighted loss">
-      <option name="closed" value="true" />
-      <created>1754483729804</created>
-      <option name="number" value="00034" />
-      <option name="presentableId" value="LOCAL-00034" />
-      <option name="project" value="LOCAL" />
-      <updated>1754483729804</updated>
-    </task>
-    <task id="LOCAL-00035" summary="added some additional metrics to analyze the validation loss per class">
-      <option name="closed" value="true" />
-      <created>1754487789125</created>
-      <option name="number" value="00035" />
-      <option name="presentableId" value="LOCAL-00035" />
-      <option name="project" value="LOCAL" />
-      <updated>1754487789125</updated>
-    </task>
-    <task id="LOCAL-00036" summary="refactor evaluation and inference code; add support for uncertainty in predictions and implement MC Dropout evaluation">
-      <option name="closed" value="true" />
-      <created>1754500617537</created>
-      <option name="number" value="00036" />
-      <option name="presentableId" value="LOCAL-00036" />
-      <option name="project" value="LOCAL" />
-      <updated>1754500617537</updated>
-    </task>
-    <task id="LOCAL-00037" summary="enhance evaluation plots; added flare class axes">
-      <option name="closed" value="true" />
-      <created>1754506057840</created>
-      <option name="number" value="00037" />
-      <option name="presentableId" value="LOCAL-00037" />
-      <option name="project" value="LOCAL" />
-      <updated>1754506057840</updated>
-    </task>
-    <task id="LOCAL-00038" summary="refactor evaluation and inference code; add uncertainty handling and update model parameters">
-      <option name="closed" value="true" />
-      <created>1754666197648</created>
-      <option name="number" value="00038" />
-      <option name="presentableId" value="LOCAL-00038" />
-      <option name="project" value="LOCAL" />
-      <updated>1754666197648</updated>
-    </task>
-    <task id="LOCAL-00039" summary="small fixes to data loader wavelengths">
-      <option name="closed" value="true" />
-      <created>1754677152392</created>
-      <option name="number" value="00039" />
-      <option name="presentableId" value="LOCAL-00039" />
-      <option name="project" value="LOCAL" />
-      <updated>1754677152392</updated>
-    </task>
-    <task id="LOCAL-00040" summary="update configuration for data loading to enable oversampling;">
-      <option name="closed" value="true" />
-      <created>1754678561074</created>
-      <option name="number" value="00040" />
-      <option name="presentableId" value="LOCAL-00040" />
-      <option name="project" value="LOCAL" />
-      <updated>1754678561074</updated>
-    </task>
-    <task id="LOCAL-00041" summary="update configuration for data loading to enable oversampling;">
-      <option name="closed" value="true" />
-      <created>1754678590926</created>
-      <option name="number" value="00041" />
-      <option name="presentableId" value="LOCAL-00041" />
-      <option name="project" value="LOCAL" />
-      <updated>1754678590926</updated>
-    </task>
-    <task id="LOCAL-00042" summary="update evaluation and inference configuration for 4-wavelengths dataset">
-      <option name="closed" value="true" />
-      <created>1754927662181</created>
-      <option name="number" value="00042" />
-      <option name="presentableId" value="LOCAL-00042" />
-      <option name="project" value="LOCAL" />
-      <updated>1754927662181</updated>
-    </task>
-    <task id="LOCAL-00043" summary="update evaluation and inference configuration for stereo dataset; adjust wavelengths and model parameters">
-      <option name="closed" value="true" />
-      <created>1755294111029</created>
-      <option name="number" value="00043" />
-      <option name="presentableId" value="LOCAL-00043" />
-      <option name="project" value="LOCAL" />
-      <updated>1755294111029</updated>
-    </task>
-    <task id="LOCAL-00044" summary="implement wavelength dropout in training, just as a test... it's commented out">
-      <option name="closed" value="true" />
-      <created>1755802313840</created>
-      <option name="number" value="00044" />
-      <option name="presentableId" value="LOCAL-00044" />
-      <option name="project" value="LOCAL" />
-      <updated>1755802313840</updated>
-    </task>
-    <task id="LOCAL-00045" summary="added support for a patch based model">
-      <option name="closed" value="true" />
-      <created>1756677376487</created>
-      <option name="number" value="00045" />
-      <option name="presentableId" value="LOCAL-00045" />
-      <option name="project" value="LOCAL" />
-      <updated>1756677376487</updated>
-    </task>
-    <task id="LOCAL-00046" summary="fixed some comments">
-      <option name="closed" value="true" />
-      <created>1756678669786</created>
-      <option name="number" value="00046" />
-      <option name="presentableId" value="LOCAL-00046" />
-      <option name="project" value="LOCAL" />
-      <updated>1756678669786</updated>
-    </task>
-    <task id="LOCAL-00047" summary="weight decay">
-      <option name="closed" value="true" />
-      <created>1756678864389</created>
-      <option name="number" value="00047" />
-      <option name="presentableId" value="LOCAL-00047" />
-      <option name="project" value="LOCAL" />
-      <updated>1756678864389</updated>
-    </task>
-    <task id="LOCAL-00048" summary="Add in FOXES project description and instructions markdown doc.">
-      <option name="closed" value="true" />
-      <created>1761206727112</created>
-      <option name="number" value="00048" />
-      <option name="presentableId" value="LOCAL-00048" />
-      <option name="project" value="LOCAL" />
-      <updated>1761206727112</updated>
-    </task>
-    <task id="LOCAL-00049" summary="Add in FOXES project description and instructions markdown.">
-      <option name="closed" value="true" />
-      <created>1761225368432</created>
-      <option name="number" value="00049" />
-      <option name="presentableId" value="LOCAL-00049" />
-      <option name="project" value="LOCAL" />
-      <updated>1761225368432</updated>
-    </task>
-    <task id="LOCAL-00050" summary="Add in FOXES project description and instructions markdown.">
-      <option name="closed" value="true" />
-      <created>1761225942194</created>
-      <option name="number" value="00050" />
-      <option name="presentableId" value="LOCAL-00050" />
-      <option name="project" value="LOCAL" />
-      <updated>1761225942195</updated>
-    </task>
-    <task id="LOCAL-00051" summary="Add in ViT ML Workflow illustration">
-      <option name="closed" value="true" />
-      <created>1761526222829</created>
-      <option name="number" value="00051" />
-      <option name="presentableId" value="LOCAL-00051" />
-      <option name="project" value="LOCAL" />
-      <updated>1761526222829</updated>
-    </task>
-    <task id="LOCAL-00052" summary="updates">
-      <option name="closed" value="true" />
-      <created>1765909732629</created>
-      <option name="number" value="00052" />
-      <option name="presentableId" value="LOCAL-00052" />
-      <option name="project" value="LOCAL" />
-      <updated>1765909732630</updated>
-    </task>
-    <task id="LOCAL-00053" summary="Implement dynamic theme-based plotting for regression output and tweak default parameters&#10;&#10;- Added theme-based (black/white) formatting for regression plots, impacting text, gridlines, legends, and backgrounds.&#10;- Adjusted layout to handle missing AIA data with specialized SXR-only frames.&#10;- Updated evaluation time ranges and plot intervals for better granularity.&#10;- Changed default regression plot background to black in CLI arguments.">
-      <option name="closed" value="true" />
-      <created>1766005071312</created>
-      <option name="number" value="00053" />
-      <option name="presentableId" value="LOCAL-00053" />
-      <option name="project" value="LOCAL" />
-      <updated>1766005071312</updated>
-    </task>
-    <task id="LOCAL-00054" summary="Simplify and restructure README: streamlined project description, updated team details, clarified repository structure, removed redundant sections, and added Docker setup instructions.">
-      <option name="closed" value="true" />
-      <created>1766078172890</created>
-      <option name="number" value="00054" />
-      <option name="presentableId" value="LOCAL-00054" />
-      <option name="project" value="LOCAL" />
-      <updated>1766078172890</updated>
-    </task>
-    <task id="LOCAL-00055" summary="Enhance plotting themes for poster analysis, refine regression plot scaling, and add data cleanup utilities&#10;&#10;- Updated flare_analysis_poster.py to use white text and spines for improved visibility on dark backgrounds, adjusted legend and grid styling, and improved annotation contrast.&#10;- Refined regression plot scaling in evaluation.py: enforced realistic log-scale bounds, added buffer for axis limits, and improved color mapping for 2D histograms.&#10;- Added cleanup_data.py and cleanup_data_mp.py scripts for identifying and moving extraneous .npy files not present in train/val CSVs, with multiprocessing support.&#10;- Introduced processing_utils.py with functions for flare classification, parallel file processing, and data summarization.&#10;- Adjusted local_config.yaml time range and updated FOXES.iml data folder structure.">
-      <option name="closed" value="true" />
-      <created>1768859793406</created>
-      <option name="number" value="00055" />
-      <option name="presentableId" value="LOCAL-00055" />
-      <option name="project" value="LOCAL" />
-      <updated>1768859793406</updated>
-    </task>
-    <option name="localTasksCounter" value="56" />
-    <servers />
-  </component>
-  <component name="TypeScriptGeneratedFilesManager">
-    <option name="version" value="3" />
-  </component>
-  <component name="Vcs.Log.Tabs.Properties">
-    <option name="RECENT_FILTERS">
-      <map>
-        <entry key="User">
-          <value>
-            <list>
-              <RecentGroup>
-                <option name="FILTER_VALUES">
-                  <option value="*" />
-                </option>
-              </RecentGroup>
-            </list>
-          </value>
-        </entry>
-      </map>
-    </option>
-    <option name="TAB_STATES">
-      <map>
-        <entry key="MAIN">
-          <value>
-            <State>
-              <option name="FILTERS">
-                <map>
-                  <entry key="user">
-                    <value>
-                      <list>
-                        <option value="*" />
-                      </list>
-                    </value>
-                  </entry>
-                </map>
-              </option>
-            </State>
-          </value>
-        </entry>
-      </map>
-    </option>
-  </component>
-  <component name="VcsManagerConfiguration">
-    <MESSAGE value="Enhance AIA_GOESDataset to support cadence and reference time" />
-    <MESSAGE value="Add oversampling functionality to AIA_GOESDataset and update configuration" />
-    <MESSAGE value="Update AIA wavelengths and model configuration in config.yaml; adjust oversampling setting in dataloader" />
-    <MESSAGE value="added some checks to make sure model training and validation is successful" />
-    <MESSAGE value="updated the weighted loss to return proper multiplication values... also reduced weighting slightly" />
-    <MESSAGE value="updated the weighted loss" />
-    <MESSAGE value="added some additional metrics to analyze the validation loss per class" />
-    <MESSAGE value="refactor evaluation and inference code; add support for uncertainty in predictions and implement MC Dropout evaluation" />
-    <MESSAGE value="enhance evaluation plots; added flare class axes" />
-    <MESSAGE value="refactor evaluation and inference code; add uncertainty handling and update model parameters" />
-    <MESSAGE value="small fixes to data loader wavelengths" />
-    <MESSAGE value="update configuration for data loading to enable oversampling;" />
-    <MESSAGE value="update evaluation and inference configuration for 4-wavelengths dataset" />
-    <MESSAGE value="update evaluation and inference configuration for stereo dataset; adjust wavelengths and model parameters" />
-    <MESSAGE value="implement wavelength dropout in training, just as a test... it's commented out" />
-    <MESSAGE value="added support for a patch based model" />
-    <MESSAGE value="fixed some comments" />
-    <MESSAGE value="weight decay" />
-    <MESSAGE value="Add in FOXES project description and instructions markdown doc." />
-    <MESSAGE value="Add in FOXES project description and instructions markdown." />
-    <MESSAGE value="Add in ViT ML Workflow illustration" />
-    <MESSAGE value="updates" />
-    <MESSAGE value="Implement dynamic theme-based plotting for regression output and tweak default parameters&#10;&#10;- Added theme-based (black/white) formatting for regression plots, impacting text, gridlines, legends, and backgrounds.&#10;- Adjusted layout to handle missing AIA data with specialized SXR-only frames.&#10;- Updated evaluation time ranges and plot intervals for better granularity.&#10;- Changed default regression plot background to black in CLI arguments." />
-    <MESSAGE value="Simplify and restructure README: streamlined project description, updated team details, clarified repository structure, removed redundant sections, and added Docker setup instructions." />
-    <MESSAGE value="Enhance plotting themes for poster analysis, refine regression plot scaling, and add data cleanup utilities&#10;&#10;- Updated flare_analysis_poster.py to use white text and spines for improved visibility on dark backgrounds, adjusted legend and grid styling, and improved annotation contrast.&#10;- Refined regression plot scaling in evaluation.py: enforced realistic log-scale bounds, added buffer for axis limits, and improved color mapping for 2D histograms.&#10;- Added cleanup_data.py and cleanup_data_mp.py scripts for identifying and moving extraneous .npy files not present in train/val CSVs, with multiprocessing support.&#10;- Introduced processing_utils.py with functions for flare classification, parallel file processing, and data summarization.&#10;- Adjusted local_config.yaml time range and updated FOXES.iml data folder structure." />
-    <option name="LAST_COMMIT_MESSAGE" value="Enhance plotting themes for poster analysis, refine regression plot scaling, and add data cleanup utilities&#10;&#10;- Updated flare_analysis_poster.py to use white text and spines for improved visibility on dark backgrounds, adjusted legend and grid styling, and improved annotation contrast.&#10;- Refined regression plot scaling in evaluation.py: enforced realistic log-scale bounds, added buffer for axis limits, and improved color mapping for 2D histograms.&#10;- Added cleanup_data.py and cleanup_data_mp.py scripts for identifying and moving extraneous .npy files not present in train/val CSVs, with multiprocessing support.&#10;- Introduced processing_utils.py with functions for flare classification, parallel file processing, and data summarization.&#10;- Adjusted local_config.yaml time range and updated FOXES.iml data folder structure." />
-  </component>
-  <component name="XDebuggerManager">
-    <breakpoint-manager>
-      <breakpoints>
-        <line-breakpoint enabled="true" suspend="THREAD" type="jupyter-line">
-          <url>file://$PROJECT_DIR$/flaring/notebook_tests/plotting_test_results.ipynb</url>
-          <line>1</line>
-          <option name="timeStamp" value="7" />
-        </line-breakpoint>
-      </breakpoints>
-    </breakpoint-manager>
-  </component>
-  <component name="github-copilot-workspace">
-    <instructionFileLocations>
-      <option value=".github/instructions" />
-    </instructionFileLocations>
-    <promptFileLocations>
-      <option value=".github/prompts" />
-    </promptFileLocations>
-  </component>
-</project>

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2025 Frontier Development Lab
+Copyright (c) 2026 Frontier Development Lab
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -1,13 +1,11 @@
 # FOXES: A Framework For Operational X-ray Emission Synthesis
 
-This repository contains the code and resources for **FOXES**, a project developed as part of the _**Frontier Development Lab**_'s Heliolab Multimodal Flare Prediction Model Project.
+This repository contains the code and resources for **FOXES**, a project developed as part of the _**Frontier Development Lab**_'s Heliolab Multimodal Flare Prediction Project.
 
 ### Abstract
-Understanding solar flares is critical for predicting space weather, as their activity shapes how the Sun influences Earth and its environment. The development of reliable forecasting methodologies of these events depends on robust flare catalogs, but current methods are limited to flare classification using integrated soft X-ray emission that are available only from Earth’s perspective. This reduces accuracy in pinpointing the location and strength of farside flares and their connection to geoeffective events. 
+The solar soft X-ray (SXR) irradiance is a long-standing proxy of solar activity, used for the classification of flare strength. As a result, the flare class, along with the SXR light curve, are routinely used as the primary input to many forecasting methods, from coronal mass ejection speed to energetic particle output. However, the SXR irradiance lacks spatial information leading to dubious classification during periods of high activity, and is  applicable only for observations from Earth orbit, hindering forecasting for other places in the heliosphere. This work introduces the Framework for Operational X-ray Emission Synthesis (FOXES), a Vision Transformer-based approach for translating Extreme Ultraviolet (EUV) spatially-resolved observations into SXR irradiance predictions. The model produces two outputs: (1) a global SXR flux prediction and (2) per-patch flux contributions, which offer a spatially resolved interpretation of where the model attributes SXR emission. This paves the way for EUV-based flare detection to be extended beyond Earth's line of sight, allowing for a more comprehensive and reliable flare catalog to support robust, scalable, and real-time forecasting, extending our monitoring into a true multiviewpoint system.
 
-In this work, we introduce a **Vision Transformer (ViT)**-based approach that translates Extreme Ultraviolet (EUV) observations into soft x-ray flux while also setting the groundwork for estimating flare locations in the future. The model achieves accurate flux predictions across flare classes using quantitative metrics. This paves the way for EUV-based flare detection to be extended beyond Earth’s line of sight, which allows for a more comprehensive and complete solar flare catalog. 
-
-**Team**: Griffin Goodwin, Alison March, Jayant Biradar, Christopher Schirninger, Robert Jarolim, Angelos Vourlidas, Lorien Pratt
+**Team**: Griffin Goodwin, Alison March, Jayant Biradar, Christopher Schirninger, Robert Jarolim, Angelos Vourlidas, Viacheslav Sadykov, Lorien Pratt
 
 ## Repository Structure
 
@@ -20,8 +18,10 @@ FOXES
 │   ├── inference       # Inference and evaluation scripts
 │   ├── models          # Vision Transformer model definitions
 │   └── training        # Training scripts and callbacks
+├── misc                # Personal utility scripts (gitignored)
 ├── notebook_tests      # Visualization and testing notebooks
 ├── Dockerfile          # Docker configuration for environment reproducibility
+├── foxes.yml           # Conda environment file
 └── requirements.txt    # Python dependencies
 ```
 
@@ -33,15 +33,19 @@ git clone https://github.com/griffin-goodwin/FOXES.git
 cd FOXES
 ```
 
-### 2) Create an environment (conda or mamba)
+### 2) Create an environment
+
+**Option A — pip:**
 ```bash
-mamba create -n foxes python=3.11 -y    # or: conda create -n foxes python=3.11 -y
-mamba activate foxes                   # or: conda activate foxes
+conda create -n foxes python=3.11 -y
+conda activate foxes
+pip install -r requirements.txt
 ```
 
-### 3) Install Python dependencies
+**Option B — conda (full environment):**
 ```bash
-pip install -r requirements.txt
+conda env create -f foxes.yml
+conda activate foxes
 ```
 
 ### 4) Docker Setup (Optional)
@@ -56,7 +60,7 @@ If you use this code or data in your work, please cite:
     institution     = {Frontier Development Lab (FDL), NASA Goddard Space Flight Center},
     repository-code = {https://github.com/griffin-goodwin/FOXES},
     version         = {v1.0},
-    year            = {2025}
+    year            = {2026}
 }
 ```
 

cleanup_data.py

@@ -1,51 +0,0 @@
-import pandas as pd
-import os
-import shutil
-from pathlib import Path
-
-# Paths
-base_data_path = Path("/Volumes/T9/FOXES_Data")
-train_csv = base_data_path / "train_data_combined.csv"
-val_csv = base_data_path / "val_data_combined.csv"
-
-# Load combined filenames
-print("Loading CSVs...")
-df_train = pd.read_csv(train_csv)
-df_val = pd.read_csv(val_csv)
-
-valid_filenames = set(df_train['filename'].tolist()) | set(df_val['filename'].tolist())
-print(f"Total valid files: {len(valid_filenames)}")
-
-# Directories to process
-dirs_to_clean = [
-    base_data_path / "SXR" / "train",
-    base_data_path / "SXR" / "val",
-    base_data_path / "AIA" / "train",
-    base_data_path / "AIA" / "val"
-]
-
-for directory in dirs_to_clean:
-    if not directory.exists():
-        print(f"Directory {directory} does not exist, skipping.")
-        continue
-    
-    print(f"\nProcessing {directory}...")
-    extras_dir = directory / "extras"
-    extras_dir.mkdir(exist_ok=True)
-    
-    # Get all .npy files in the directory
-    files = [f for f in os.listdir(directory) if f.endswith(".npy") and os.path.isfile(directory / f)]
-    
-    moved_count = 0
-    for filename in files:
-        if filename not in valid_filenames:
-            src = directory / filename
-            dst = extras_dir / filename
-            # shutil.move(src, dst) # Dry run first
-            moved_count += 1
-            if moved_count <= 5:
-                print(f"  [DRY RUN] Would move: {filename}")
-    
-    print(f"  [DRY RUN] Would move {moved_count} files to {extras_dir}")
-
-print("\nDone with dry run.")

cleanup_data_mp.py

@@ -1,88 +0,0 @@
-import pandas as pd
-import os
-import shutil
-from pathlib import Path
-from multiprocessing import Pool, cpu_count
-from functools import partial
-from tqdm import tqdm
-
-def check_and_move_file(filename, directory, extras_dir, valid_filenames, dry_run=True):
-    """Worker function to check and move a single file."""
-    if filename not in valid_filenames:
-        src = directory / filename
-        dst = extras_dir / filename
-        try:
-            if not dry_run:
-                shutil.move(src, dst)
-            return 1 # Increment moved count
-        except Exception as e:
-            print(f"Error moving {filename}: {e}")
-            return 0
-    return 0
-
-def process_directory(directory, extras_sub_dir, valid_filenames, dry_run=True):
-    """Processes a single directory using a pool of workers."""
-    if not directory.exists():
-        print(f"Directory {directory} does not exist, skipping.")
-        return 0
-    
-    print(f"\nProcessing {directory}...")
-    extras_sub_dir.mkdir(parents=True, exist_ok=True)
-    
-    # Get all .npy files in the directory
-    files = [f.name for f in os.scandir(directory) if f.name.endswith(".npy") and f.is_file()]
-    print(f"Found {len(files)} files in {directory.name}")
-
-    # Partial function with fixed arguments
-    worker = partial(check_and_move_file, 
-                    directory=directory, 
-                    extras_dir=extras_sub_dir, 
-                    valid_filenames=valid_filenames, 
-                    dry_run=dry_run)
-    
-    # Use multiprocessing to check and move files
-    num_workers = cpu_count()
-    chunksize = max(1, len(files) // (num_workers * 4))
-    with Pool(num_workers) as pool:
-        results = list(tqdm(pool.imap(worker, files, chunksize=chunksize), total=len(files), desc=f"Cleaning {directory.name}"))
-    
-    moved_count = sum(results)
-    status = "[DRY RUN] Would move" if dry_run else "Moved"
-    print(f"{status} {moved_count} files to {extras_sub_dir}")
-    return moved_count
-
-if __name__ == "__main__":
-    # Paths
-    base_data_path = Path("/Volumes/T9/FOXES_Data")
-    train_csv = base_data_path / "train_data_combined.csv"
-    val_csv = base_data_path / "val_data_combined.csv"
-    extras_base_dir = base_data_path / "extras"
-
-    # Set dry_run to False to actually move files
-    DRY_RUN = False
-
-    # Load combined filenames
-    print("Loading CSVs...")
-    df_train = pd.read_csv(train_csv)
-    df_val = pd.read_csv(val_csv)
-
-    valid_filenames = set(df_train['filename'].tolist()) | set(df_val['filename'].tolist())
-    print(f"Total valid files in CSVs: {len(valid_filenames)}")
-
-    # Directories to process and their corresponding extras sub-directories
-    tasks = [
-        (base_data_path / "SXR" / "train", extras_base_dir / "SXR"),
-        (base_data_path / "SXR" / "val", extras_base_dir / "SXR"),
-        (base_data_path / "AIA" / "train", extras_base_dir / "AIA"),
-        (base_data_path / "AIA" / "val", extras_base_dir / "AIA")
-    ]
-
-    total_moved = 0
-    for directory, extras_sub_dir in tasks:
-        total_moved += process_directory(directory, extras_sub_dir, valid_filenames, dry_run=DRY_RUN)
-
-    if DRY_RUN:
-        print(f"\nDry run complete. Total files that would be moved: {total_moved}")
-        print("To actually move files, set DRY_RUN = False in the script.")
-    else:
-        print(f"\nProcessing complete. Total files moved: {total_moved}")

foxes.yml

@@ -0,0 +1,273 @@
+name: foxes
+channels:
+  - defaults
+  - https://repo.anaconda.com/pkgs/main
+  - https://repo.anaconda.com/pkgs/r
+dependencies:
+  - bzip2=1.0.8=h80987f9_6
+  - ca-certificates=2025.12.2=hca03da5_0
+  - expat=2.7.3=h982b769_0
+  - libcxx=20.1.8=hd7fd590_1
+  - libffi=3.4.4=hca03da5_1
+  - libmpdec=4.0.0=h80987f9_0
+  - libzlib=1.3.1=h5f15de7_0
+  - lz4-c=1.9.4=h313beb8_1
+  - ncurses=6.5=hee39554_0
+  - openssl=3.0.18=h9b4081a_0
+  - python=3.14.0=h1858270_101_cp314
+  - python_abi=3.14=2_cp314
+  - readline=8.3=h0b18652_0
+  - sqlite=3.51.0=hab6afd1_0
+  - tk=8.6.15=hcd8a7d5_0
+  - xz=5.6.4=h80987f9_1
+  - zlib=1.3.1=h5f15de7_0
+  - zstd=1.5.7=h817c040_0
+  - pip:
+      - aiapy==0.10.2
+      - aioftp==0.27.2
+      - aiohappyeyeballs==2.6.1
+      - aiohttp==3.13.2
+      - aiosignal==1.4.0
+      - albucore==0.0.24
+      - albumentations==2.0.8
+      - annotated-types==0.7.0
+      - anyio==4.12.0
+      - appdirs==1.4.4
+      - appnope==0.1.4
+      - argon2-cffi==25.1.0
+      - argon2-cffi-bindings==25.1.0
+      - argparse-dataclass==2.0.0
+      - arrow==1.4.0
+      - asdf==5.1.0
+      - asdf-astropy==0.9.0
+      - asdf-coordinates-schemas==0.4.0
+      - asdf-standard==1.4.0
+      - asdf-transform-schemas==0.6.0
+      - astropy==7.2.0
+      - astropy-healpix==1.1.2
+      - astropy-iers-data==0.2025.12.1.0.45.12
+      - asttokens==3.0.1
+      - async-lru==2.0.5
+      - attrs==25.4.0
+      - babel==2.17.0
+      - beautifulsoup4==4.14.3
+      - bleach==6.3.0
+      - cachetools==6.2.2
+      - cdflib==1.3.7
+      - certifi==2025.11.12
+      - cffi==2.0.0
+      - cftime==1.6.5
+      - charset-normalizer==3.4.4
+      - click==8.3.1
+      - cloudpickle==3.1.2
+      - colormaps==0.4.2
+      - comm==0.2.3
+      - conda-inject==1.3.2
+      - configargparse==1.7.1
+      - connection-pool==0.0.3
+      - contourpy==1.3.3
+      - cycler==0.12.1
+      - dask==2025.11.0
+      - dask-image==2025.11.0
+      - debugpy==1.8.17
+      - decorator==5.2.1
+      - defusedxml==0.7.1
+      - docutils==0.22.3
+      - donfig==0.8.1.post1
+      - dpath==2.2.0
+      - drms==0.9.0
+      - executing==2.2.1
+      - fastjsonschema==2.21.2
+      - filelock==3.20.0
+      - fonttools==4.61.0
+      - fqdn==1.5.1
+      - frozenlist==1.8.0
+      - fsspec==2025.12.0
+      - gitdb==4.0.12
+      - gitpython==3.1.45
+      - glymur==0.14.4
+      - google==3.0.0
+      - google-api-core==2.28.1
+      - google-auth==2.43.0
+      - google-cloud-core==2.5.0
+      - google-cloud-storage==3.6.0
+      - google-resumable-media==2.8.0
+      - googleapis-common-protos==1.72.0
+      - h11==0.16.0
+      - h5netcdf==1.7.3
+      - h5py==3.15.1
+      - httpcore==1.0.9
+      - httpx==0.28.1
+      - humanfriendly==10.0
+      - idna==3.11
+      - imageio==2.37.2
+      - imageio-ffmpeg==0.6.0
+      - immutables==0.21
+      - ipykernel==7.1.0
+      - ipython==9.8.0
+      - ipython-pygments-lexers==1.1.1
+      - ipywidgets==8.1.8
+      - isodate==0.7.2
+      - isoduration==20.11.0
+      - itipy==0.1.1
+      - jedi==0.19.2
+      - jinja2==3.1.6
+      - jmespath==1.0.1
+      - joblib==1.5.2
+      - json5==0.12.1
+      - jsonpointer==3.0.0
+      - jsonschema==4.25.1
+      - jsonschema-specifications==2025.9.1
+      - jupyter==1.1.1
+      - jupyter-client==8.6.3
+      - jupyter-console==6.6.3
+      - jupyter-core==5.9.1
+      - jupyter-events==0.12.0
+      - jupyter-lsp==2.3.0
+      - jupyter-server==2.17.0
+      - jupyter-server-terminals==0.5.3
+      - jupyterlab==4.5.0
+      - jupyterlab-pygments==0.3.0
+      - jupyterlab-server==2.28.0
+      - jupyterlab-widgets==3.0.16
+      - kiwisolver==1.4.9
+      - lark==1.3.1
+      - lazy-loader==0.4
+      - lightning==2.6.0
+      - lightning-utilities==0.15.2
+      - locket==1.0.0
+      - lxml==6.0.2
+      - markupsafe==3.0.3
+      - matplotlib==3.10.7
+      - matplotlib-inline==0.2.1
+      - mistune==3.1.4
+      - mpl-animators==1.2.4
+      - mpmath==1.3.0
+      - multidict==6.7.0
+      - nbclient==0.10.2
+      - nbconvert==7.16.6
+      - nbformat==5.10.4
+      - nest-asyncio==1.6.0
+      - netcdf4==1.7.3
+      - networkx==3.6
+      - notebook==7.5.0
+      - notebook-shim==0.2.4
+      - numcodecs==0.16.5
+      - numpy==2.4.0rc1
+      - numpy-typing-compat==20251206.2.4
+      - opencv-python==4.12.0.88
+      - opencv-python-headless==4.12.0.88
+      - optype==0.15.0
+      - packaging==25.0
+      - pandas==2.3.3
+      - pandas-stubs==2.3.3.251201
+      - pandocfilters==1.5.1
+      - parfive==2.2.0
+      - parso==0.8.5
+      - partd==1.4.2
+      - pexpect==4.9.0
+      - pillow==12.0.0
+      - pims==0.7
+      - pip==24.3.1
+      - platformdirs==4.5.0
+      - prometheus-client==0.23.1
+      - prompt-toolkit==3.0.52
+      - propcache==0.4.1
+      - proto-plus==1.26.1
+      - protobuf==6.33.1
+      - psutil==7.1.3
+      - ptyprocess==0.7.0
+      - pulp==3.3.0
+      - pure-eval==0.2.3
+      - pyarrow==22.0.0
+      - pyasn1==0.6.1
+      - pyasn1-modules==0.4.2
+      - pyavm==0.9.8
+      - pycparser==2.23
+      - pydantic==2.12.5
+      - pydantic-core==2.41.5
+      - pyerfa==2.0.1.5
+      - pygments==2.19.2
+      - pyparsing==3.2.5
+      - python-dateutil==2.9.0.post0
+      - python-json-logger==4.0.0
+      - pytorch-fid==0.3.0
+      - pytorch-lightning==2.6.0
+      - pytz==2025.2
+      - pyyaml==6.0.3
+      - pyzmq==27.1.0
+      - referencing==0.37.0
+      - reproject==0.19.0
+      - requests==2.32.5
+      - requests-file==3.0.1
+      - requests-toolbelt==1.0.0
+      - reretry==0.11.8
+      - rfc3339-validator==0.1.4
+      - rfc3986-validator==0.1.1
+      - rfc3987-syntax==1.1.0
+      - rpds-py==0.30.0
+      - rsa==4.9.1
+      - scikit-image==0.25.2
+      - scikit-learn==1.7.2
+      - scipy==1.16.3
+      - scipy-stubs==1.16.3.3
+      - seaborn==0.13.2
+      - semantic-version==2.10.0
+      - send2trash==1.8.3
+      - sentry-sdk==2.47.0
+      - setuptools==80.9.0
+      - simsimd==6.5.3
+      - six==1.17.0
+      - slicerator==1.1.0
+      - smart-open==7.5.0
+      - smmap==5.0.2
+      - snakemake==9.14.1
+      - snakemake-interface-common==1.22.0
+      - snakemake-interface-executor-plugins==9.3.9
+      - snakemake-interface-logger-plugins==2.0.0
+      - snakemake-interface-report-plugins==1.3.0
+      - snakemake-interface-scheduler-plugins==2.0.2
+      - snakemake-interface-storage-plugins==4.3.2
+      - soupsieve==2.8
+      - spicepy==1.0.5
+      - spiceypy==8.0.0
+      - stack-data==0.6.3
+      - stringzilla==4.4.0
+      - sunpy==7.0.3
+      - sunpy-soar==1.11.1
+      - sympy==1.14.0
+      - tabulate==0.9.0
+      - terminado==0.18.1
+      - threadpoolctl==3.6.0
+      - throttler==1.2.2
+      - tifffile==2025.10.16
+      - tinycss2==1.4.0
+      - toolz==1.1.0
+      - torch==2.9.1
+      - torchaudio==2.9.1
+      - torchmetrics==1.8.2
+      - torchvision==0.24.1
+      - tornado==6.5.2
+      - tqdm==4.67.1
+      - traitlets==5.14.3
+      - types-pytz==2025.2.0.20251108
+      - typing-extensions==4.15.0
+      - typing-inspection==0.4.2
+      - tzdata==2025.2
+      - uri-template==1.3.0
+      - urllib3==2.5.0
+      - validators==0.35.0
+      - wandb==0.23.1
+      - wcwidth==0.2.14
+      - webcolors==25.10.0
+      - webencodings==0.5.1
+      - websocket-client==1.9.0
+      - widgetsnbextension==4.0.15
+      - wrapt==2.0.1
+      - xarray==2025.11.0
+      - xitorch==0.3.0
+      - yarl==1.22.0
+      - yte==1.9.4
+      - zarr==3.1.5
+      - zeep==4.3.2
+prefix: /opt/anaconda3/envs/foxes

requirements.txt

@@ -1,30 +1,38 @@
+# Solar physics and astronomy
 sunpy[all]
+sunpy-soar
+astropy
+drms
 itipy
-snakemake
-wandb
-pytorch-lightning
-pandas
-netCDF4
-matplotlib
-scikit-learn
-scikit-image
+
+# Deep learning
 torch
 torchvision
-torchaudio
-validators
-albumentations
-opencv-python
-spicepy
-xitorch
-sympy
+pytorch-lightning
+
+# Data processing
 numpy
 scipy
-PyYAML
-imageio
-tqdm
-seaborn
+pandas
 xarray
-astropy
-Pillow
+h5py
+
+# Machine learning
+scikit-learn
+
+# Visualization
+matplotlib
+seaborn
+plotly
 colormaps
-imageio-ffmpeg
+
+# Image processing
+opencv-python
+Pillow
+imageio
+imageio-ffmpeg
+
+# Utilities
+tqdm
+wandb
+PyYAML