Sync to HF Space

.env.example

@@ -0,0 +1,8 @@
+SVJ_CODE_ROOT=/work/gkrzmanc/jetclustering/code
+SVJ_DATA_ROOT=/work/gkrzmanc/jetclustering/data
+#SVJ_PREPROCESSED_DATA_ROOT=/pnfs/psi.ch/cms/trivcat/store/user/gkrzmanc/jetclustering/preprocessed_data
+SVJ_PREPROCESSED_DATA_ROOT=/work/gkrzmanc/jetclustering/preprocessed_data
+SVJ_RESULTS_ROOT=/work/gkrzmanc/jetclustering/results
+SVJ_RESULTS_ROOT_FALLBACK=/pnfs/psi.ch/cms/trivcat/store/user/gkrzmanc/jetclustering/results
+SVJ_WANDB_ENTITY=fcc_ml
+WANDB_API_KEY=1a2b3c

.github/workflows/tests.yml

@@ -0,0 +1,41 @@
+name: Tests
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install numba==0.58.1
+          pip install torch==2.5.0 torchvision==0.20.0 torchaudio==2.5.0 --index-url https://download.pytorch.org/whl/cpu
+          pip install torch_geometric
+          pip install pyg_lib torch_scatter torch_sparse torch_cluster torch_spline_conv -f https://data.pyg.org/whl/torch-2.5.0+cpu.html
+          pip install xformers==0.0.29.post1 --no-deps
+          pip install pytorch-lightning yacs torchmetrics performer-pytorch tensorboardX ogb wandb seaborn dgl
+          pip install scipy pandas scikit-learn matplotlib tqdm PyYAML
+          pip install awkward0 uproot awkward vector lz4 xxhash tables tensorboard plotly
+          pip install fastjet gradio huggingface_hub hdbscan ruff
+          pip install --no-deps git+https://github.com/gregorkrz/lorentz-gatr.git
+          pip install pytest
+
+      - name: Lint with ruff
+        run: |
+          ruff check --select E9,F63,F7 --output-format=github tests/
+
+      - name: Run tests
+        run: |
+          pytest tests/ -v --tb=short

.gitignore

@@ -0,0 +1,4 @@
+models/
+demo_datasets/
+.idea/
+.env

.idea/.gitignore

@@ -0,0 +1,8 @@
+# Default ignored files
+/shelf/
+/workspace.xml
+# Editor-based HTTP Client requests
+/httpRequests/
+# Datasource local storage ignored files
+/dataSources/
+/dataSources.local.xml

AGENTS.md

@@ -0,0 +1,50 @@
+# AGENTS.md
+
+## Cursor Cloud specific instructions
+
+### Overview
+This is a Python 3.10 ML framework for IRC-safe jet clustering using Lorentz Geometric Algebra Transformers (L-GATr). The main runnable service is a **Gradio demo web app** (`app.py`) on port 7860 that performs jet clustering inference using pre-trained models.
+
+### Python version
+The project requires **Python 3.10** (pinned by numba 0.58.1 and the Dockerfile). The VM needs `python3.10` installed via deadsnakes PPA. A virtualenv at `/workspace/.venv` is used.
+
+### Key dependencies and install order
+Dependencies must be installed in a specific order to avoid version conflicts:
+1. `numba==0.58.1` (pins numpy < 1.27)
+2. PyTorch 2.5.0 **CPU** from `https://download.pytorch.org/whl/cpu`
+3. `torch_geometric` and PyG extensions (`pyg_lib`, `torch_scatter`, `torch_sparse`, `torch_cluster`, `torch_spline_conv`) from `https://data.pyg.org/whl/torch-2.5.0+cpu.html`
+4. `xformers==0.0.29.post1` (must match torch 2.5.x; do NOT let lgatr upgrade torch)
+5. L-GATr from `https://github.com/gregorkrz/lorentz-gatr` — install with `pip install --no-deps` to prevent it from upgrading torch
+6. Remaining packages: `pytorch-lightning`, `fastjet`, `gradio`, `huggingface_hub`, `hdbscan`, `ruff`, etc.
+
+### Gotcha: torch version conflicts
+Installing `lgatr` from git will attempt to upgrade PyTorch to the latest CUDA version. Always install lgatr with `--no-deps` and manually install its dependencies first. After any dependency changes, verify `python -c "import torch; print(torch.__version__)"` shows `2.5.0+cpu`.
+
+### Gotcha: xformers on CPU
+xformers CUDA extensions won't load on CPU — the warning is expected. The demo uses `cpu_demo=True` which sets the attention mask to `None`, bypassing xformers attention. The import `from xformers.ops.fmha import BlockDiagonalMask` requires xformers >= 0.0.29.
+
+### Gotcha: pytorch_cmspepr
+`pytorch_cmspepr` requires CUDA to build and is NOT needed for the demo app. It's only used by `GravNetConv` layers for training.
+
+### Running the demo app
+```bash
+source /workspace/.venv/bin/activate
+cd /workspace
+GRADIO_SERVER_NAME=0.0.0.0 python app.py
+```
+The app starts on port 7860. Pre-trained models must exist in `models/` and demo datasets in `demo_datasets/` (downloaded from HuggingFace Hub).
+
+### Models and datasets
+Downloaded from HuggingFace:
+- Models: `huggingface_hub.snapshot_download(repo_id='gregorkrzmanc/jetclustering', local_dir='models/')`
+- Demo datasets: `huggingface_hub.snapshot_download(repo_id='gregorkrzmanc/jetclustering_demo', local_dir='demo_datasets/', repo_type='dataset')`
+
+### Linting
+```bash
+source /workspace/.venv/bin/activate
+ruff check
+```
+The codebase has existing lint issues (research code); `ruff` is listed in `requirements.txt`.
+
+### Testing
+No automated test suite exists. The demo app can be tested via the Gradio API or browser UI. See README.md for project structure and usage.

Dockerfile

@@ -0,0 +1,113 @@
+## gkrz/lgatr:v3
+# docker build -t gkrz/lgatr:v4 .
+FROM nvidia/cuda:11.8.0-runtime-ubuntu22.04
+
+
+# Set up a new user named "user" with user ID 1000
+RUN useradd -m -u 1000 user
+
+
+# Set home to the user's home directory
+ENV HOME=/home/user \
+	PATH=/home/user/.local/bin:$PATH
+
+# Set the working directory to the user's home directory
+WORKDIR $HOME/app
+
+
+SHELL ["/bin/bash", "-c"]
+
+RUN apt update && \
+    DEBIAN_FRONTEND=noninteractive apt install --yes --no-install-recommends \
+        build-essential \
+        cmake \
+        ffmpeg \
+        git \
+        python-is-python3 \
+        python3-dev \
+        python3-pip \
+        && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN python3.10 --version
+RUN python3 --version
+RUN python --version
+
+RUN python3 -m pip install --no-cache-dir --upgrade pip
+#python3 -m pip install --no-cache-dir --upgrade --requirement requirements.txt
+RUN python3 -m pip install numba==0.58.1
+# packages without conda
+# RUN python3 -m pip install --no-cache-dir torch==1.13.1+cu117 torchvision==0.14.1+cu117 torchaudio==0.13.1 --extra-index-url https://download.pytorch.org/whl/cu117
+RUN python3 -m pip install torch==2.5.0 torchvision torchaudio
+RUN python3 -m pip install torch_geometric
+#RUN python3 -m pip install  torch_scatter torch_sparse
+#RUN python3 -m pip install torch-scatter -f https://data.pyg.org/whl/torch-2.5.1.html
+#RUN python3 -m pip install torch-sparse -f https://data.pyg.org/whl/torch-2.5.1.html
+#RUN python3 -m pip install torch-cluster -f https://data.pyg.org/whl/torch-2.5.1.html
+RUN python3 -m pip install pyg_lib torch_scatter torch_sparse torch_cluster torch_spline_conv xformers==0.0.29.post1 -f https://data.pyg.org/whl/torch-2.5.0+cpu.html
+
+RUN python3 -m pip install pytorch-lightning yacs torchmetrics
+RUN python3 -m pip install performer-pytorch
+RUN python3 -m pip install tensorboardX
+RUN python3 -m pip install ogb
+RUN python3 -m pip install wandb
+RUN python3 -m pip install seaborn
+RUN python3 -m pip install  dgl
+RUN python3 -m pip install numpy
+RUN python3 -m pip install scipy
+RUN python3 -m pip install pandas
+RUN python3 -m pip install scikit-learn
+RUN python3 -m pip install matplotlib
+RUN python3 -m pip install tqdm
+RUN python3 -m pip install PyYAML
+RUN python3 -m pip install awkward0
+RUN python3 -m pip install uproot
+RUN python3 -m pip install awkward
+RUN python3 -m pip install vector
+RUN python3 -m pip install lz4
+RUN python3 -m pip install xxhash
+RUN python3 -m pip install tables
+RUN python3 -m pip install tensorboard
+RUN python3 -m pip install plotly
+RUN python3 -m pip install fastjet
+RUN python3 -m pip install gradio
+RUN python3 -m pip install huggingface_hub
+RUN python3 -m pip install hdbscan
+#RUN python3 -m pip install lgatr # This doesn't work
+RUN python3 -c "\
+from huggingface_hub import snapshot_download; \
+snapshot_download(repo_id='gregorkrzmanc/jetclustering', local_dir='models/'); \
+snapshot_download(repo_id='gregorkrzmanc/jetclustering_demo', local_dir='demo_datasets/', repo_type='dataset')"
+# remove pip cache
+RUN python3 -m pip cache purge
+
+# COPY docker/ext_packages /docker/ext_packages
+# RUN python3 /docker/ext_packages/install_upstream_python_packages.py
+RUN mkdir -p $HOME/opt/pepr
+
+# Install GATr
+#RUN cd /opt/pepr && git clone https://github.com/Qualcomm-AI-research/geometric-algebra-transformer.git geometric-algebra-transformer1
+#RUN cd /opt/pepr/geometric-algebra-transformer1/ && python3 -m pip install .
+
+
+# Install L-GATr - for some reason this only works if executed from the already-built container
+RUN cd $HOME/opt/pepr && git clone https://github.com/gregorkrz/lorentz-gatr lgatr
+RUN cd $HOME/opt/pepr/lgatr/ && python3 -m pip install .
+RUN ls /usr/local/lib/python3.10/dist-packages/lgatr
+RUN ls /usr/local/lib/python3.10/dist-packages/lgatr/layers
+# Install torch_cmspepr
+
+RUN cd $HOME/opt/pepr && git clone https://github.com/cms-pepr/pytorch_cmspepr
+RUN cd $HOME/opt/pepr/pytorch_cmspepr/ && python3 -m pip install --no-build-isolation .
+
+COPY --chown=user .  $HOME/app
+
+USER root
+RUN chmod -R 777 $HOME
+USER user
+
+# entrypoint run app.py with python
+EXPOSE 7860
+ENV GRADIO_SERVER_NAME="0.0.0.0"
+CMD ["python", "app.py"]
+

Dockerfile_training

@@ -0,0 +1,77 @@
+# gkrz/lgatr:v3
+# docker build -t gkrz/lgatr:v4 .
+FROM nvidia/cuda:11.8.0-runtime-ubuntu22.04
+
+SHELL ["/bin/bash", "-c"]
+
+USER root
+
+RUN apt update && \
+    DEBIAN_FRONTEND=noninteractive apt install --yes --no-install-recommends \
+        build-essential \
+        cmake \
+        ffmpeg \
+        git \
+        python-is-python3 \
+        python3-dev \
+        python3-pip \
+        && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN python3.10 --version
+RUN python3 --version
+RUN python --version
+
+RUN python3 -m pip install --no-cache-dir --upgrade pip
+#python3 -m pip install --no-cache-dir --upgrade --requirement requirements.txt
+RUN python3 -m pip install numba==0.58.1
+# packages without conda
+# RUN python3 -m pip install --no-cache-dir torch==1.13.1+cu117 torchvision==0.14.1+cu117 torchaudio==0.13.1 --extra-index-url https://download.pytorch.org/whl/cu117
+RUN python3 -m pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu118
+RUN python3 -m pip install torch_geometric
+RUN python3 -m pip install pyg_lib torch_scatter torch_sparse torch_cluster torch_spline_conv -f https://data.pyg.org/whl/torch-2.5.1+cu118.html
+RUN python3 -m pip install pytorch-lightning yacs torchmetrics
+RUN python3 -m pip install performer-pytorch
+RUN python3 -m pip install tensorboardX
+RUN python3 -m pip install ogb
+RUN python3 -m pip install wandb
+RUN python3 -m pip install seaborn
+RUN python3 -m pip install  dgl -f https://data.dgl.ai/wheels/cu118/repo.html
+RUN python3 -m pip install numpy
+RUN python3 -m pip install scipy
+RUN python3 -m pip install pandas
+RUN python3 -m pip install scikit-learn
+RUN python3 -m pip install matplotlib
+RUN python3 -m pip install tqdm
+RUN python3 -m pip install PyYAML
+RUN python3 -m pip install awkward0
+RUN python3 -m pip install uproot
+RUN python3 -m pip install awkward
+RUN python3 -m pip install vector
+RUN python3 -m pip install lz4
+RUN python3 -m pip install xxhash
+RUN python3 -m pip install tables
+RUN python3 -m pip install tensorboard
+RUN python3 -m pip install plotly
+RUN python3 -m pip install xformers --index-url https://download.pytorch.org/whl/cu118
+RUN python3 -m pip install fastjet
+
+# remove pip cache
+RUN python3 -m pip cache purge
+
+# COPY docker/ext_packages /docker/ext_packages
+# RUN python3 /docker/ext_packages/install_upstream_python_packages.py
+RUN mkdir -p /opt/pepr
+
+# Install GATr
+RUN cd /opt/pepr && git clone https://github.com/Qualcomm-AI-research/geometric-algebra-transformer.git geometric-algebra-transformer1
+RUN cd /opt/pepr/geometric-algebra-transformer1/ && python3 -m pip install .
+
+# Install L-GATr - for some reason this only works if executed from the already-built container
+#RUN cd /opt/pepr && git clone https://github.com/gregorkrz/lorentz-gatr lgatr
+#RUN cd /opt/pepr/lgatr/ && python3 -m pip install .
+
+# Install torch_cmspepr
+
+RUN cd /opt/pepr && git clone https://github.com/cms-pepr/pytorch_cmspepr
+RUN cd /opt/pepr/pytorch_cmspepr/ && python3 -m pip install --no-build-isolation .

README.md

@@ -0,0 +1,352 @@
+---
+title: JetClustering
+emoji: ⚛️
+colorFrom: "red"
+colorTo: "blue"
+sdk: docker
+app_file: app.py
+pinned: false
+---
+
+# Learning IRC-Safe Jet Clustering with Geometric Algebra Transformers
+
+Authors: Gregor Kržmanc, Roberto Seidita, Annapaola de Cosa
+
+Paper at ML4PS Workshop at NeurIPS: https://ml4physicalsciences.github.io/2025/files/NeurIPS_ML4PS_2025_59.pdf
+
+
+A machine learning framework for jet clustering in CMS events using Geometric Algebra Transformers. This repository provides tools for preprocessing, training, and evaluating jet clustering models on Delphes simulation data.
+
+## 🚀 Quick Start
+
+**Live Demo**: Try the interactive demo at [https://huggingface.co/spaces/gregorkrzmanc/jetclustering](https://huggingface.co/spaces/gregorkrzmanc/jetclustering)
+
+The demo allows you to:
+- Upload particle-level data for an event (CSV of pt, eta, phi, mass, charge)
+- Select different model variants and training datasets
+- Visualize clustering results compared to Anti-kt jets
+- View detailed jet information in JSON format
+
+> **Note**: The live demo runs on the free HuggingFace tier, and it's extremely slow (1-5 minutes per event). For faster local execution, see the [Local Demo Setup](#local-demo-setup) section below.
+
+## 📋 Table of Contents
+
+- [Overview](#overview)
+- [Prerequisites](#prerequisites)
+- [Installation](#installation)
+- [Data Preparation](#data-preparation)
+- [Training](#training)
+- [Evaluation](#evaluation)
+- [Visualization](#visualization)
+- [Pre-trained Models](#pre-trained-models)
+- [Project Structure](#project-structure)
+
+## Overview
+
+The repo has evolved from the [MLPF repository](https://github.com/selvaggi/mlpf) (we partially reuse the dataloader).
+
+The framework supports:
+- Multiple loss functions (GP, GP_IRC_S, GP_IRC_SN)
+- Various training datasets (QCD, SVJ events with different parameters)
+- Automated evaluation pipelines
+- Comprehensive visualization tools
+
+## Prerequisites
+
+- Python 3.8+
+- CUDA-capable GPU (for training)
+- Singularity/Apptainer (for containerized training)
+- Access to CERN/PSI computing infrastructure (for full workflow)
+
+## Installation
+
+### Environment Setup
+
+This project uses a Docker container with pre-compiled packages. The container image is `gkrz/lgatr:v3`.
+
+#### Option 1: Use Pre-built Container
+
+```bash
+export APPTAINER_TMPDIR=/work/gkrzmanc/singularity_tmp
+export APPTAINER_CACHEDIR=/work/gkrzmanc/singularity_cache
+singularity shell -B /work/gkrzmanc/ -B /pnfs/psi.ch/cms/trivcat/store/user/gkrzmanc -B /t3home -H /t3home/gkrzmanc --nv docker://gkrz/lgatr:v3
+```
+
+#### Option 2: Build from Dockerfile
+
+Build the training container from scratch:
+
+```bash
+docker build -f Dockerfile_training -t gkrz/lgatr:v3 .
+```
+
+**Important**: Ensure consistent `APPTAINER_CACHEDIR` and `APPTAINER_TMPDIR` settings across sessions.
+
+### Environment Variables
+
+1. **Set up environment variables** by sourcing `env.sh`:
+
+```bash
+source env.sh
+```
+
+Or use the `.env` file for IDE integration (e.g., PyCharm).
+
+2. **Configure paths**: Edit `env.sh` to set your local paths:
+
+```bash
+export SVJ_CODE_ROOT="/path/to/jetclustering/code"
+export SVJ_DATA_ROOT="/path/to/jetclustering/data"
+export SVJ_RESULTS_ROOT="/path/to/jetclustering/results"
+export SVJ_PREPROCESSED_DATA_ROOT="/path/to/jetclustering/preprocessed_data"
+export SVJ_RESULTS_ROOT_FALLBACK="/path/to/fallback/results"  # Optional: for SE storage
+export SVJ_WANDB_ENTITY="your_wandb_entity"
+```
+
+**Path Configuration Notes**:
+- Use relative paths for portability across machines (lxplus, T3 work, T3 SE)
+- Absolute paths starting with `/` are also supported
+- `SVJ_RESULTS_ROOT_FALLBACK` is used when files aren't available in the primary results directory
+
+### Local Demo Setup
+
+For faster local execution, use Docker Compose:
+
+```yaml
+version: '3.8'
+
+services:
+  jetclustering_demo:
+    image: gkrz/jetclustering_demo_cpu:v0
+    ports:
+      - "7860:7860"
+```
+
+Save as `docker-compose.yml` and run:
+
+```bash
+docker-compose up
+```
+
+## Data Preparation
+
+### Generating Delphes Data
+
+See the [jetclustering_sim repository](https://github.com/gregorkrz/jetclustering_sim) for instructions on generating Delphes simulation data.
+
+### Preprocessing Delphes Data
+
+Preprocess your Delphes data using the provided SLURM jobs:
+
+```bash
+# For QCD training data
+sbatch jobs/preprocess_v3_Delphes_QCDtrain.slurm
+
+# For QCD evaluation data
+sbatch jobs/preprocess_v3_Delphes_QCDEval.slurm
+
+# For pile-up (PU) data
+sbatch jobs/preprocess_v3_Delphes_PU_PFfix_Train.slurm
+sbatch jobs/preprocess_v3_Delphes_PU_PFfix.slurm
+```
+
+**Important**: Update your local `env.sh` file before running preprocessing jobs!
+
+### Download Preprocessed Datasets
+
+Preprocessed datasets are available at:
+- **Hugging Face Datasets**: [https://huggingface.co/datasets/gregorkrzmanc/jetclustering](https://huggingface.co/datasets/gregorkrzmanc/jetclustering)
+
+Download and place them in the `preprocessed_data/` folder.
+
+### Storage Management
+
+To copy results to Storage Element (SE) and free up local storage:
+
+```bash
+rsync -avz -e "ssh" /work/gkrzmanc/jetclustering/results/ /pnfs/psi.ch/cms/trivcat/store/user/gkrzmanc/jetclustering/results
+```
+
+The system automatically falls back to `SVJ_RESULTS_ROOT_FALLBACK` when files aren't found in the primary location.
+
+## Training
+
+### Base Model Training
+
+The base clustering model is trained on m=900 GeV, r_inv=0.3 for 50k steps:
+
+```bash
+# Training scripts are located in:
+jobs/base_training/
+```
+
+### Extended Training
+
+For models trained with additional steps (GP, GP_IRC_S, GP_IRC_SN variants with +25k steps):
+
+```bash
+jobs/base_training_different_datasets/
+```
+
+These scripts load the base model using `--load-model-weights` and continue training.
+
+**Important Configuration Note**: 
+- Switch between `GP_IRC_SN` and `GP_IRC_S` by modifying line `if i % 2: # Every second one:` in `dataset/dataset.py`
+- Set to `if i % 2:` for GP_IRC_SN
+- Set to `if not (i % 2):` for GP_IRC_S
+
+### Training on Different Datasets
+
+Scripts for training on various dataset combinations:
+
+```bash
+jobs/base_training_different_datasets/aug/  # Augmented datasets
+```
+
+## Evaluation
+
+### Automated Evaluation Pipeline
+
+The evaluation process consists of two stages:
+
+#### Stage 1: GPU Evaluation
+
+Generate evaluation jobs for a specific checkpoint:
+
+```bash
+python -m scripts.generate_test_jobs \
+    -template t3 \
+    -run Transformer_training_40k_5_64_4_2025_01_22_15_55_39 \
+    -step 10000 \
+    -tag params_study
+```
+
+**Parameters**:
+- `-template`: Job template (e.g., `t3`, `vega`)
+- `-run`: Training run identifier
+- `-step`: Checkpoint step (counts from training start)
+- `-tag`: Study identifier for grouping evaluations
+- `-os`: Path to objectness score checkpoint (optional, not used in final paper)
+- `-pl`: Evaluate on parton-level particles
+- `-gl`: Evaluate on gen-level particles
+- `--steps-from-zero`: Disable automatic checkpoint detection from previous runs
+
+**Checkpoint Resolution**:
+- The script automatically detects if training was restarted from a checkpoint
+- It loads the appropriate checkpoint from previous runs if needed
+- Use `--steps-from-zero` to disable this behavior
+
+**Helper Script**: Use `notebooks/gen_test_job_cmd_gen.py` to generate evaluation commands interactively.
+
+#### Stage 2: CPU Evaluation and Analysis
+
+After GPU evaluation completes, run analysis and plotting:
+
+```bash
+python -m scripts.test_plot_jobs \
+    --tag params_study \
+    --input <input_dataset>
+```
+
+**Additional Flags**:
+- `--submit-AKX`: Spawn Anti-kt evaluation jobs
+- `-pl`: For parton-level evaluation
+- `-gl`: For gen-level evaluation
+- `-ow`: Overwrite existing results
+- `-pt <cutoff>`: Run pT cutoff studies (e.g., `-pt 90`)
+
+**Evaluation Workflow**:
+1. Run GPU evaluation for each dataset
+2. Run CPU evaluation/analysis (4 times per dataset: 3 for AK variants + 1 for GPU results)
+3. Results include `run_config.pkl` for later metric analysis
+
+### pT Cutoff Studies
+
+To study performance at different pT cutoffs:
+
+```bash
+python -m scripts.test_plot_jobs --tag params_study --input <dataset> -pt 90
+```
+
+This creates results with suffix `_pt_90.0`. Generate plots comparing metrics vs. pT cutoff:
+
+```bash
+python -m scripts/metrics_plots_vs_pt_cutoff.py
+```
+
+## Visualization
+
+### Generating Evaluation Plots
+
+Produce comprehensive evaluation plots:
+
+```bash
+python -m scripts.plot_eval_count_matched_quarks --input <input_directory>
+```
+
+**Input Directory**: Points to the directory produced by `test_plot_jobs` (named after the tag).
+
+**Configuration**: Modify the dictionary around line 320 in the script to map training run IDs to standardized names (e.g., `LGATr_GP_IRC_SN`).
+
+### Metric Analysis
+
+Use scripts in `scripts/` to generate joint plots of:
+- F1 score
+- Precision
+- Recall
+- Other performance metrics
+
+The `run_config.pkl` files generated during evaluation can be used to create plots comparing:
+- Metrics vs. number of parameters
+- Metrics vs. model architecture
+- Metrics vs. training duration
+
+## Pre-trained Models
+
+### Model Weights
+
+Pre-trained model weights are available at:
+- **Hugging Face Model Hub**: [https://huggingface.co/gregorkrzmanc/jetclustering/tree/main](https://huggingface.co/gregorkrzmanc/jetclustering/tree/main)
+
+### Weights & Biases Runs
+
+Training runs and metrics are logged at:
+- **WandB Project**: [https://wandb.ai/fcc_ml/svj_clustering](https://wandb.ai/fcc_ml/svj_clustering)
+
+**Setup**: Add your WandB API key to `env.sh`:
+
+```bash
+export WANDB_API_KEY="your_api_key_here"
+```
+
+## Project Structure
+
+```
+jetclustering/
+├── app.py                      # Gradio demo interface
+├── Dockerfile                  # Demo container
+├── Dockerfile_training         # Training container
+├── env.sh                      # Environment variables
+├── requirements.txt            # Python dependencies
+├── config_files/               # Model and dataset configurations
+├── jobs/                       # SLURM job scripts
+│   ├── base_training/          # Base model training
+│   ├── base_training_different_datasets/  # Extended training
+│   └── preprocess_*.slurm      # Preprocessing jobs
+├── notebooks/                  # Jupyter notebooks and helper scripts
+├── scripts/                    # Evaluation and plotting scripts
+│   ├── generate_test_jobs.py   # Generate evaluation jobs
+│   ├── test_plot_jobs.py       # Run analysis and plotting
+│   └── plot_eval_count_matched_quarks.py  # Main plotting script
+└── src/                        # Source code
+    ├── data/                   # Data loading utilities
+    ├── dataset/                # Dataset classes
+    ├── evaluation/             # Evaluation metrics
+    ├── jetfinder/              # Jet finding algorithms
+    ├── layers/                 # Neural network layers
+    ├── models/                 # Model architectures
+    ├── plotting/               # Visualization utilities
+    ├── preprocessing/          # Data preprocessing
+    ├── train.py                # Training script
+    └── utils/                  # Utility functions
+```

app.py

@@ -0,0 +1,77 @@
+import gradio as gr
+import matplotlib.pyplot as plt
+import numpy as np
+import os
+
+from src.model_wrapper_gradio import inference
+
+# === Dummy file-based prefill function ===
+def prefill_event(subdataset, event_idx):
+    base_path = f"demo_datasets/{subdataset}/{event_idx}"
+    try:
+        with open(f"{base_path}.txt", "r") as f:
+            particles_data = f.read()
+    except FileNotFoundError:
+        particles_data = "pt eta phi mass charge\n"
+
+    try:
+        with open(f"{base_path}_quarks.txt", "r") as f:
+            quarks_data = f.read()
+    except FileNotFoundError:
+        quarks_data = "pt eta phi\n"
+
+    return particles_data, quarks_data
+
+
+#from huggingface_hub import snapshot_download
+#snapshot_download(repo_id="gregorkrzmanc/jetclustering", local_dir="models/")
+#snapshot_download(repo_id="gregorkrzmanc/jetclustering_demo", local_dir="demo_datasets/", repo_type="dataset")
+
+# === Interface layout ===
+def gradio_ui():
+    with gr.Blocks() as demo:
+        gr.Markdown("## Jet Clustering Demo")
+        # now put a short text explaining that the demo is very slow and if you want to run it on your machine, you can use the following docker-compose file:
+        # version: '3.8'
+        #
+        # services:
+        #   jetclustering_demo:
+        #     image: gkrz/jetclustering_demo_cpu:v0
+        #     ports:
+        #       - "7860:7860"
+        gr.Markdown("The live demo is very slow (usually takes 1-5 minutes for a single event). If you want to run it on your machine, you can use the following docker-compose file:\n\n```yaml\nversion: '3.8'\n\nservices:\n  jetclustering_demo:\n    image: gkrz/jetclustering_demo_cpu:v0\n    ports:\n      - '7860:7860'\n```")
+        with gr.Row():
+            loss_dropdown = gr.Dropdown(choices=["GP_IRC_SN", "GP_IRC_S", "GP", "base"], label="Loss Function", value="GP_IRC_SN")
+            train_dataset_dropdown = gr.Dropdown(choices=["QCD", "900_03", "900_03+700_07", "700_07", "900_03+700_07+QCD"], label="Training Dataset", value="QCD")
+
+        with gr.Row():
+            subdataset_dropdown = gr.Dropdown(choices=[x for x in os.listdir("demo_datasets") if not x.startswith(".")], label="Subdataset", value="QCD")
+            event_idx_dropdown = gr.Dropdown(choices=list(range(20)), label="Event Index", value=15)
+        prefill_btn = gr.Button("Load Event from Dataset")
+
+        particles_text = gr.Textbox(label="Particles CSV (pt eta phi mass charge)", lines=6, interactive=True)
+        quarks_text = gr.Textbox(label="Quarks CSV (pt eta phi) - optional", lines=3, interactive=True)
+
+        process_btn = gr.Button("Run Jet Clustering")
+        gr.Markdown("### Outputs")
+        gr.Markdown("The jets with transverse momentum above 100 GeV are circled (green for AK8, blue for the model). The dark quarks are marked with red triangles. The particles are colored based on their jet (only jets with pT > 30 GeV are colored). The json objects contain the jets with pT > 30 GeV.")
+
+        image_output = gr.Plot(label="Output")
+        model_jets_output = gr.JSON(label="Model Jets")
+        antikt_jets_output = gr.JSON(label="Anti-kt Jets")
+        model_coords_output = gr.JSON(label="Model Coordinates")
+        prefill_btn.click(fn=prefill_event,
+                          inputs=[subdataset_dropdown, event_idx_dropdown],
+                          outputs=[particles_text, quarks_text])
+
+
+        process_btn.click(fn=inference,
+                          inputs=[loss_dropdown, train_dataset_dropdown, particles_text, quarks_text],
+                          outputs=[model_jets_output, antikt_jets_output, image_output, model_coords_output])
+
+    return demo
+
+
+demo = gradio_ui()
+demo.launch()
+

config_files/config_jets.yaml

@@ -0,0 +1,172 @@
+treename: mmtree/Events;1
+selection:
+   ### use `&`, `|`, `~` for logical operations on numpy arrays
+   ### can use functions from `math`, `np` (numpy), and `awkward` in the expression
+   #(jet_tightId==1) & (jet_no<2) & (fj_pt>200) & (fj_pt<2500) & (((sample_isQCD==0) & (fj_isQCD==0)) | ((sample_isQCD==1) & (fj_isQCD==1))) & (event_no%7!=0)
+   #(recojet_e>=5)
+
+test_time_selection:
+   ### selection to apply at test time (i.e., when running w/ --predict)
+   #(jet_tightId==1) & (jet_no<2) & (fj_pt>200) & (fj_pt<2500) & (((sample_isQCD==0) & (fj_isQCD==0)) | ((sample_isQCD==1) & (fj_isQCD==1))) & (event_no%7==0)
+   #(recojet_e<5)
+
+new_variables:
+   ### [format] name: formula
+   ### can use functions from `math`, `np` (numpy), and `awkward` in the expression
+   #pfcand_mask: awkward.JaggedArray.ones_like(pfcand_etarel)
+   #sv_mask: awkward.JaggedArray.ones_like(sv_etarel)
+   #pfcand_mask: awkward.JaggedArray.ones_like(pfcand_e)
+
+preprocess:
+  ### method: [manual, auto] - whether to use manually specified parameters for variable standardization
+  ### [note]: `[var]_mask` will not be transformed even if `method=auto`
+
+inputs:
+  n_fat_jets:
+    pad_mode: wrap
+    length: 1
+    vars:
+      - [nFatJet, null]
+#      - [nJetId, null]
+  fat_jets:
+    pad_mode: wrap
+    length: 50
+    vars:
+       - [FatJet_pt, null]
+       - [FatJet_eta, null]
+       - [FatJet_phi, null]
+       - [FatJet_mass, null]
+  n_jets:
+    pad_mode: wrap
+    length: 1
+    vars:
+      - [ nJet, null ]
+  jets:
+    pad_mode: wrap
+    length: 50
+    vars:
+      - [ Jet_pt, null ]
+      - [ Jet_eta, null ]
+      - [ Jet_phi, null ]
+      - [ Jet_mass, null ]
+  n_genjets:
+    pad_mode: wrap
+    length: 1
+    vars:
+      - [n_genjet, null]
+  genjets:
+    pad_mode: wrap
+    length: 50
+    vars:
+       - [GenFatJet_pt, null]
+       - [GenFatJet_eta, null]
+       - [GenFatJet_phi, null]
+       - [GenFatJet_mass, null]
+  n_pfcands:
+    pad_mode: wrap
+    length: 1
+    vars:
+      - [ nPFCands, null ]
+  pfcands:
+    pad_mode: wrap
+    length: 750
+    vars:
+       - [PFCands_pt, null]
+       - [PFCands_eta, null]
+       - [PFCands_phi, null]
+       - [PFCands_mass, null]
+       - [PFCands_charge, null]
+       - [PFCands_pdgId, null]
+
+  pfcands_jet_mapping:
+    pad_mode: wrap
+    length: 750
+    vars:
+      - [ FatJetPFCands_jetIdx, null ]
+      - [ FatJetPFCands_pFCandsIdx, null ]
+  #n_offline_pfcands:
+  #  pad_mode: wrap
+  #  length: 1
+  #  vars:
+  #    - [ nOfflinePFCands, null ]
+  #offline_pfcands:
+  #  pad_mode: wrap
+  #  length: 750
+  #  vars:
+  #    - [ OfflinePFCands_pt, null ]
+  #    - [ OfflinePFCands_eta, null ]
+  #    - [ OfflinePFCands_phi, null ]
+  #    - [ OfflinePFCands_mass, null ]
+  #    - [ OfflinePFCands_charge, null ]
+  #    - [ OfflinePFCands_pdgId, null ]
+  #offline_pfcands_jet_mapping:
+  #  pad_mode: wrap
+  #  length: 750
+  #  vars:
+  #    - [ OfflineFatJetPFCands_jetIdx, null ]
+  #    - [ OfflineFatJetPFCands_pFCandsIdx, null ]
+  MET:
+    pad_mode: wrap
+    length: 1
+    vars:
+      - [ MET_pt, null ]
+      - [ MET_phi, null ]
+      - [ scouting_trig, null]
+      - [ offline_trig, null]
+      - [ veto_trig, null ]
+  n_electrons:
+    pad_mode: wrap
+    length: 1
+    vars:
+      - [ nElectron, null ]
+  n_photons:
+    pad_mode: wrap
+    length: 1
+    vars:
+      - [ nPhotons, null ]
+  n_muons:
+    pad_mode: wrap
+    length: 1
+    vars:
+      - [ nMuons, null ]
+  electrons:
+    pad_mode: wrap
+    length: 10
+    vars:
+      - [ Electron_pt, null ]
+      - [ Electron_eta, null ]
+      - [ Electron_phi, null ]
+      - [ Electron_charge, null ]
+  muons:
+    pad_mode: wrap
+    length: 10
+    vars:
+      - [ Muon_pt, null ]
+      - [ Muon_eta, null ]
+      - [ Muon_phi, null ]
+      - [ Muon_charge, null ]
+  photons:
+    pad_mode: wrap
+    length: 10
+    vars:
+      - [ Photon_pt, null ]
+      - [ Photon_eta, null ]
+      - [ Photon_phi, null ]
+  matrix_element_gen_particles:
+    pad_mode: wrap
+    length: 2
+    vars:
+      - [MatrixElementGenParticle_pt, null]
+      - [MatrixElementGenParticle_eta, null]
+      - [MatrixElementGenParticle_phi, null]
+      - [MatrixElementGenParticle_mass, null]
+      - [MatrixElementGenParticle_pdgId, null]
+labels:
+
+observers:
+   #- recojet_e
+   #- recojet_theta
+   #- recojet_phi
+   #- recojet_m
+   #- n_pfcand
+

config_files/config_jets_1.yaml

@@ -0,0 +1,191 @@
+treename: null
+selection:
+   ### use `&`, `|`, `~` for logical operations on numpy arrays
+   ### can use functions from `math`, `np` (numpy), and `awkward` in the expression
+   #(jet_tightId==1) & (jet_no<2) & (fj_pt>200) & (fj_pt<2500) & (((sample_isQCD==0) & (fj_isQCD==0)) | ((sample_isQCD==1) & (fj_isQCD==1))) & (event_no%7!=0)
+   #(recojet_e>=5)
+
+test_time_selection:
+   ### selection to apply at test time (i.e., when running w/ --predict)
+   #(jet_tightId==1) & (jet_no<2) & (fj_pt>200) & (fj_pt<2500) & (((sample_isQCD==0) & (fj_isQCD==0)) | ((sample_isQCD==1) & (fj_isQCD==1))) & (event_no%7==0)
+   #(recojet_e<5)
+
+new_variables:
+   ### [format] name: formula
+   ### can use functions from `math`, `np` (numpy), and `awkward` in the expression
+   #pfcand_mask: awkward.JaggedArray.ones_like(pfcand_etarel)
+   #sv_mask: awkward.JaggedArray.ones_like(sv_etarel)
+   #pfcand_mask: awkward.JaggedArray.ones_like(pfcand_e)
+
+preprocess:
+  ### method: [manual, auto] - whether to use manually specified parameters for variable standardization
+  ### [note]: `[var]_mask` will not be transformed even if `method=auto`
+
+inputs:
+  n_fat_jets:
+    pad_mode: wrap
+    length: 1
+    vars:
+      - [nFatJet, null]
+#      - [nJetId, null]
+  fat_jets:
+    pad_mode: wrap
+    length: 50
+    vars:
+       - [FatJet_pt, null]
+       - [FatJet_eta, null]
+       - [FatJet_phi, null]
+       - [FatJet_mass, null]
+  n_jets:
+    pad_mode: wrap
+    length: 1
+    vars:
+      - [ nJet, null ]
+  jets:
+    pad_mode: wrap
+    length: 50
+    vars:
+      - [ Jet_pt, null ]
+      - [ Jet_eta, null ]
+      - [ Jet_phi, null ]
+      - [ Jet_mass, null ]
+  n_genjets:
+    pad_mode: wrap
+    length: 1
+    vars:
+      - [n_genjet, null]
+  genjets:
+    pad_mode: wrap
+    length: 50
+    vars:
+       - [GenFatJet_pt, null]
+       - [GenFatJet_eta, null]
+       - [GenFatJet_phi, null]
+       - [GenFatJet_mass, null]
+  n_pfcands:
+    pad_mode: wrap
+    length: 1
+    vars:
+      - [ nPFCands, null ]
+  pfcands:
+    pad_mode: wrap
+    length: 750
+    vars:
+       - [PFCands_pt, null]
+       - [PFCands_eta, null]
+       - [PFCands_phi, null]
+       - [PFCands_mass, null]
+       - [PFCands_charge, null]
+       - [PFCands_pdgId, null]
+
+  pfcands_jet_mapping:
+    pad_mode: wrap
+    length: 750
+    vars:
+      - [ FatJetPFCands_jetIdx, null ]
+      - [ FatJetPFCands_pFCandsIdx, null ]
+  #n_offline_pfcands:
+  #  pad_mode: wrap
+  #  length: 1
+  #  vars:
+  #    - [ nOfflinePFCands, null ]
+  #offline_pfcands:
+  #  pad_mode: wrap
+  #  length: 750
+  #  vars:
+  #    - [ OfflinePFCands_pt, null ]
+  #    - [ OfflinePFCands_eta, null ]
+  #    - [ OfflinePFCands_phi, null ]
+  #    - [ OfflinePFCands_mass, null ]
+  #    - [ OfflinePFCands_charge, null ]
+  #    - [ OfflinePFCands_pdgId, null ]
+  #offline_pfcands_jet_mapping:
+  #  pad_mode: wrap
+  #  length: 750
+  #  vars:
+  #    - [ OfflineFatJetPFCands_jetIdx, null ]
+  #    - [ OfflineFatJetPFCands_pFCandsIdx, null ]
+  MET:
+    pad_mode: wrap
+    length: 1
+    vars:
+      - [ MET_pt, null ]
+      - [ MET_phi, null ]
+      - [ scouting_trig, null]
+      - [ offline_trig, null]
+      - [ veto_trig, null ]
+  n_electrons:
+    pad_mode: wrap
+    length: 1
+    vars:
+      - [ nElectron, null ]
+  n_photons:
+    pad_mode: wrap
+    length: 1
+    vars:
+      - [ nPhotons, null ]
+  n_muons:
+    pad_mode: wrap
+    length: 1
+    vars:
+      - [ nMuons, null ]
+  electrons:
+    pad_mode: wrap
+    length: 10
+    vars:
+      - [ Electron_pt, null ]
+      - [ Electron_eta, null ]
+      - [ Electron_phi, null ]
+      - [ Electron_charge, null ]
+  muons:
+    pad_mode: wrap
+    length: 10
+    vars:
+      - [ Muon_pt, null ]
+      - [ Muon_eta, null ]
+      - [ Muon_phi, null ]
+      - [ Muon_charge, null ]
+  photons:
+    pad_mode: wrap
+    length: 10
+    vars:
+      - [ Photon_pt, null ]
+      - [ Photon_eta, null ]
+      - [ Photon_phi, null ]
+  matrix_element_gen_particles:
+    pad_mode: wrap
+    length: 2
+    vars:
+      - [MatrixElementGenParticle_pt, null]
+      - [MatrixElementGenParticle_eta, null]
+      - [MatrixElementGenParticle_phi, null]
+      - [MatrixElementGenParticle_mass, null]
+      - [MatrixElementGenParticle_pdgId, null]
+  final_gen_particles:
+      pad_mode: wrap
+      length: 2000
+      vars:
+        - [FinalGenParticle_pt, null]
+        - [FinalGenParticle_eta, null]
+        - [FinalGenParticle_phi, null]
+        - [FinalGenParticle_mass, null]
+        - [FinalGenParticle_pdgId, null]
+        - [FinalGenParticle_status, null]
+  final_parton_level_particles:
+        pad_mode: wrap
+        length: 400
+        vars:
+            - [FinalPartonLevelParticle_pt, null]
+            - [FinalPartonLevelParticle_eta, null]
+            - [FinalPartonLevelParticle_phi, null]
+            - [FinalPartonLevelParticle_mass, null]
+            - [FinalPartonLevelParticle_pdgId, null]
+            - [FinalPartonLevelParticle_status, null]
+
+observers:
+   #- recojet_e
+   #- recojet_theta
+   #- recojet_phi
+   #- recojet_m
+   #- n_pfcand
+

config_files/config_jets_1_delphes.yaml

@@ -0,0 +1,86 @@
+treename: Delphes;1
+selection:
+   ### use `&`, `|`, `~` for logical operations on numpy arrays
+   ### can use functions from `math`, `np` (numpy), and `awkward` in the expression
+   #(jet_tightId==1) & (jet_no<2) & (fj_pt>200) & (fj_pt<2500) & (((sample_isQCD==0) & (fj_isQCD==0)) | ((sample_isQCD==1) & (fj_isQCD==1))) & (event_no%7!=0)
+   #(recojet_e>=5)
+
+test_time_selection:
+   ### selection to apply at test time (i.e., when running w/ --predict)
+   #(jet_tightId==1) & (jet_no<2) & (fj_pt>200) & (fj_pt<2500) & (((sample_isQCD==0) & (fj_isQCD==0)) | ((sample_isQCD==1) & (fj_isQCD==1))) & (event_no%7==0)
+   #(recojet_e<5)
+
+new_variables:
+   ### [format] name: formula
+   ### can use functions from `math`, `np` (numpy), and `awkward` in the expression
+   #pfcand_mask: awkward.JaggedArray.ones_like(pfcand_etarel)
+   #sv_mask: awkward.JaggedArray.ones_like(sv_etarel)
+   #pfcand_mask: awkward.JaggedArray.ones_like(pfcand_e)
+
+preprocess:
+  ### method: [manual, auto] - whether to use manually specified parameters for variable standardization
+  ### [note]: `[var]_mask` will not be transformed even if `method=auto`
+
+inputs:
+  n_CH:
+    pad_mode: wrap
+    length: 1
+    vars:
+      - [ EFlowTrack_size, null ]
+  n_NH:
+    pad_mode: wrap
+    length: 1
+    vars:
+      - [ EFlowNeutralHadron_size, null ]
+  n_photon:
+    pad_mode: wrap
+    length: 1
+    vars:
+      - [ EFlowPhoton_size, null ]
+  CH:
+    pad_mode: wrap
+    length: 1500
+    vars:
+       - [EFlowTrack.Eta, null]
+       - [EFlowTrack.Phi, null]
+       - [EFlowTrack.PT, null]
+       - [EFlowTrack.Mass, null]
+       - [EFlowTrack.Charge, null]
+  NH:
+    pad_mode: wrap
+    length: 1500
+    vars:
+       - [EFlowNeutralHadron.Eta, null]
+       - [EFlowNeutralHadron.Phi, null]
+       - [EFlowNeutralHadron.ET, null]
+  EFlowPhoton:
+    pad_mode: wrap
+    length: 1500
+    vars:
+       - [EFlowPhoton.Eta, null]
+       - [EFlowPhoton.Phi, null]
+       - [EFlowPhoton.ET, null]
+  GenParticles:
+      pad_mode: wrap
+      length: 7500
+      vars:
+        - [Particle.Eta, null]
+        - [Particle.Phi, null]
+        - [Particle.PT, null]
+        - [Particle.Charge, null]
+        - [Particle.Mass, null]
+        - [Particle.PID, null]
+        - [Particle.Status, null]
+  NParticles:
+      pad_mode: wrap
+      length: 1
+      vars:
+          - [Particle_size, null]
+observers:
+   #- recojet_e
+   #- recojet_theta
+   #- recojet_phi
+   #- recojet_m
+   #- n_pfcand
+
+

config_files/config_jets_2_delphes.yaml

@@ -0,0 +1,63 @@
+treename: Delphes;1
+selection:
+   ### use `&`, `|`, `~` for logical operations on numpy arrays
+   ### can use functions from `math`, `np` (numpy), and `awkward` in the expression
+   #(jet_tightId==1) & (jet_no<2) & (fj_pt>200) & (fj_pt<2500) & (((sample_isQCD==0) & (fj_isQCD==0)) | ((sample_isQCD==1) & (fj_isQCD==1))) & (event_no%7!=0)
+   #(recojet_e>=5)
+
+test_time_selection:
+   ### selection to apply at test time (i.e., when running w/ --predict)
+   #(jet_tightId==1) & (jet_no<2) & (fj_pt>200) & (fj_pt<2500) & (((sample_isQCD==0) & (fj_isQCD==0)) | ((sample_isQCD==1) & (fj_isQCD==1))) & (event_no%7==0)
+   #(recojet_e<5)
+
+new_variables:
+   ### [format] name: formula
+   ### can use functions from `math`, `np` (numpy), and `awkward` in the expression
+   #pfcand_mask: awkward.JaggedArray.ones_like(pfcand_etarel)
+   #sv_mask: awkward.JaggedArray.ones_like(sv_etarel)
+   #pfcand_mask: awkward.JaggedArray.ones_like(pfcand_e)
+
+preprocess:
+  ### method: [manual, auto] - whether to use manually specified parameters for variable standardization
+  ### [note]: `[var]_mask` will not be transformed even if `method=auto`
+
+inputs:
+  n_PFCands:
+    pad_mode: wrap
+    length: 1
+    vars:
+      - [ ParticleFlowCandidate_size, null ]
+  PFCands:
+    pad_mode: wrap
+    length: 1500
+    vars:
+       - [ParticleFlowCandidate.Eta, null]
+       - [ParticleFlowCandidate.Phi, null]
+       - [ParticleFlowCandidate.PT, null]
+       - [ParticleFlowCandidate.Mass, null]
+       - [ParticleFlowCandidate.Charge, null]
+       - [ParticleFlowCandidate.PID, null]
+  GenParticles:
+      pad_mode: wrap
+      length: 7500
+      vars:
+        - [Particle.Eta, null]
+        - [Particle.Phi, null]
+        - [Particle.PT, null]
+        - [Particle.Charge, null]
+        - [Particle.Mass, null]
+        - [Particle.PID, null]
+        - [Particle.Status, null]
+  NParticles:
+      pad_mode: wrap
+      length: 1
+      vars:
+          - [Particle_size, null]
+observers:
+   #- recojet_e
+   #- recojet_theta
+   #- recojet_phi
+   #- recojet_m
+   #- n_pfcand
+
+

container_shell.sh

@@ -0,0 +1,4 @@
+export APPTAINER_TMPDIR=/work/gkrzmanc/singularity_tmp
+export APPTAINER_CACHEDIR=/work/gkrzmanc/singularity_cache
+singularity shell  -B /work/gkrzmanc/ --nv docker://dologarcia/gatr:v0
+

docker-compose.yaml

@@ -0,0 +1,7 @@
+version: '3'
+
+services:
+  app:
+    image: gkrz/jetclustering_demo:v0
+    ports:
+      - "7860:7860"

env.sh

@@ -0,0 +1,17 @@
+# For CERN machines
+#export SVJ_CODE_ROOT="/eos/home-g/gkrzmanc/jetclustering/code"
+#export SVJ_DATA_ROOT="/eos/home-g/gkrzmanc/jetclustering/data"
+#export SVJ_RESULTS_ROOT="/eos/home-g/gkrzmanc/jetclustering/results"
+#export SVJ_PREPROCESSED_DATA_ROOT="/eos/home-g/gkrzmanc/jetclustering/preprocessed_data"
+
+
+# For PSI T3
+export SVJ_CODE_ROOT="/work/gkrzmanc/jetclustering/code"
+#export SVJ_DATA_ROOT="/work/gkrzmanc/jetclustering/data"
+export SVJ_DATA_ROOT="/pnfs/psi.ch/cms/trivcat/store/user/gkrzmanc/jetclustering/data"
+export SVJ_RESULTS_ROOT="/work/gkrzmanc/jetclustering/results"
+export SVJ_PREPROCESSED_DATA_ROOT="/work/gkrzmanc/jetclustering/preprocessed_data"
+#export SVJ_PREPROCESSED_DATA_ROOT="/pnfs/psi.ch/cms/trivcat/store/user/gkrzmanc/jetclustering/preprocessed_data"
+export SVJ_WANDB_ENTITY="fcc_ml"
+export SVJ_RESULTS_ROOT_FALLBACK="/pnfs/psi.ch/cms/trivcat/store/user/gkrzmanc/jetclustering/results"
+

jobs/BigTraining_2_spatial_part_only_t3.slurm

@@ -0,0 +1,15 @@
+#!/bin/bash
+#SBATCH --partition=qgpu               # Specify the partition
+#SBATCH --account=gpu_gres            # Specify the account
+#SBATCH --mem=3000                   # Request 10GB of memory
+#SBATCH --time=00:10:00
+#SBATCH --job-name=SVJtr3  # Name the job
+#SBATCH --output=jobs/BigTraining_output.log       # Redirect stdout to a log file
+#SBATCH --error=jobs/BigTraining_error.log         # Redirect stderr to a log file
+#SBATCH --gres=gpu:1
+source env.sh
+export APPTAINER_TMPDIR=/work/gkrzmanc/singularity_tmp
+export APPTAINER_CACHEDIR=/work/gkrzmanc/singularity_cache
+nvidia-smi
+
+srun singularity exec -B /t3home/gkrzmanc/  -B /work/gkrzmanc/  --nv docker://gkrz/lgatr:v3  python -m src.train -train scouting_PFNano_signals2/SVJ_hadronic_std3/s-channel_mMed-1100_mDark-20_rinv-0.3_alpha-peak_13TeV-pythia8_n-2000 scouting_PFNano_signals2/SVJ_hadronic_std3/s-channel_mMed-1500_mDark-20_rinv-0.3_alpha-peak_13TeV-pythia8_n-2000 scouting_PFNano_signals2/SVJ_hadronic_std3/s-channel_mMed-1100_mDark-20_rinv-0.5_alpha-peak_13TeV-pythia8_n-2000 scouting_PFNano_signals2/SVJ_hadronic_std3/s-channel_mMed-1500_mDark-20_rinv-0.5_alpha-peak_13TeV-pythia8_n-2000 scouting_PFNano_signals2/SVJ_hadronic_std3/s-channel_mMed-1100_mDark-20_rinv-0.7_alpha-peak_13TeV-pythia8_n-2000 scouting_PFNano_signals2/SVJ_hadronic_std3/s-channel_mMed-1500_mDark-20_rinv-0.7_alpha-peak_13TeV-pythia8_n-2000 scouting_PFNano_signals2/SVJ_hadronic_std3/s-channel_mMed-1200_mDark-20_rinv-0.3_alpha-peak_13TeV-pythia8_n-2000 scouting_PFNano_signals2/SVJ_hadronic_std3/s-channel_mMed-700_mDark-20_rinv-0.3_alpha-peak_13TeV-pythia8_n-2000 scouting_PFNano_signals2/SVJ_hadronic_std3/s-channel_mMed-1200_mDark-20_rinv-0.5_alpha-peak_13TeV-pythia8_n-2000 scouting_PFNano_signals2/SVJ_hadronic_std3/s-channel_mMed-700_mDark-20_rinv-0.5_alpha-peak_13TeV-pythia8_n-2000 scouting_PFNano_signals2/SVJ_hadronic_std3/s-channel_mMed-1200_mDark-20_rinv-0.7_alpha-peak_13TeV-pythia8_n-2000 scouting_PFNano_signals2/SVJ_hadronic_std3/s-channel_mMed-700_mDark-20_rinv-0.7_alpha-peak_13TeV-pythia8_n-2000 scouting_PFNano_signals2/SVJ_hadronic_std3/s-channel_mMed-1300_mDark-20_rinv-0.3_alpha-peak_13TeV-pythia8_n-2000 scouting_PFNano_signals2/SVJ_hadronic_std3/s-channel_mMed-800_mDark-20_rinv-0.3_alpha-peak_13TeV-pythia8_n-2000 scouting_PFNano_signals2/SVJ_hadronic_std3/s-channel_mMed-1300_mDark-20_rinv-0.5_alpha-peak_13TeV-pythia8_n-2000 scouting_PFNano_signals2/SVJ_hadronic_std3/s-channel_mMed-800_mDark-20_rinv-0.5_alpha-peak_13TeV-pythia8_n-2000 scouting_PFNano_signals2/SVJ_hadronic_std3/s-channel_mMed-1300_mDark-20_rinv-0.7_alpha-peak_13TeV-pythia8_n-2000 scouting_PFNano_signals2/SVJ_hadronic_std3/s-channel_mMed-800_mDark-20_rinv-0.7_alpha-peak_13TeV-pythia8_n-2000 scouting_PFNano_signals2/SVJ_hadronic_std3/s-channel_mMed-1400_mDark-20_rinv-0.3_alpha-peak_13TeV-pythia8_n-2000 scouting_PFNano_signals2/SVJ_hadronic_std3/s-channel_mMed-900_mDark-20_rinv-0.3_alpha-peak_13TeV-pythia8_n-2000 scouting_PFNano_signals2/SVJ_hadronic_std3/s-channel_mMed-1400_mDark-20_rinv-0.5_alpha-peak_13TeV-pythia8_n-2000 scouting_PFNano_signals2/SVJ_hadronic_std3/s-channel_mMed-900_mDark-20_rinv-0.5_alpha-peak_13TeV-pythia8_n-2000 scouting_PFNano_signals2/SVJ_hadronic_std3/s-channel_mMed-1400_mDark-20_rinv-0.7_alpha-peak_13TeV-pythia8_n-2000 scouting_PFNano_signals2/SVJ_hadronic_std3/s-channel_mMed-900_mDark-20_rinv-0.7_alpha-peak_13TeV-pythia8_n-2000  -val scouting_PFNano_signals2/SVJ_hadronic_std2/s-channel_mMed-1000_mDark-20_rinv-0.3 scouting_PFNano_signals2/SVJ_hadronic_std2/s-channel_mMed-1500_mDark-20_rinv-0.5 scouting_PFNano_signals2/SVJ_hadronic_std2/s-channel_mMed-700_mDark-20_rinv-0.7 scouting_PFNano_signals2/SVJ_hadronic_std2/s-channel_mMed-900_mDark-20_rinv-0.3 scouting_PFNano_signals2/SVJ_hadronic_std2/s-channel_mMed-1000_mDark-20_rinv-0.5 scouting_PFNano_signals2/SVJ_hadronic_std2/s-channel_mMed-1500_mDark-20_rinv-0.7 scouting_PFNano_signals2/SVJ_hadronic_std2/s-channel_mMed-800_mDark-20_rinv-0.3 scouting_PFNano_signals2/SVJ_hadronic_std2/s-channel_mMed-900_mDark-20_rinv-0.5 scouting_PFNano_signals2/SVJ_hadronic_std2/s-channel_mMed-1000_mDark-20_rinv-0.7 scouting_PFNano_signals2/SVJ_hadronic_std2/s-channel_mMed-700_mDark-20_rinv-0.3 scouting_PFNano_signals2/SVJ_hadronic_std2/s-channel_mMed-800_mDark-20_rinv-0.5 scouting_PFNano_signals2/SVJ_hadronic_std2/s-channel_mMed-900_mDark-20_rinv-0.7 scouting_PFNano_signals2/SVJ_hadronic_std2/s-channel_mMed-1500_mDark-20_rinv-0.3 scouting_PFNano_signals2/SVJ_hadronic_std2/s-channel_mMed-700_mDark-20_rinv-0.5 scouting_PFNano_signals2/SVJ_hadronic_std2/s-channel_mMed-800_mDark-20_rinv-0.7  -net src/models/LGATr/lgatr.py -bs 64 --gpus 0 --run-name Train_LGATr_SB_spatial_part_only --val-dataset-size 4000 --num-epochs 1000 --attr-loss-weight 0.1 --coord-loss-weight 0.1 --beta-type pt+bc --spatial-part-only --validation-steps 2000 --num-workers 0

jobs/BigTraining_2_spatial_part_only_vega.slurm

@@ -0,0 +1,21 @@
+#!/bin/bash
+#SBATCH --job-name="SVJtrAll"
+#SBATCH --time=48:00:00
+#SBATCH --nodes=1
+#SBATCH --gres=gpu:1
+#SBATCH --ntasks-per-core=1
+#SBATCH --ntasks-per-node=1
+#SBATCH --cpus-per-task=2
+#SBATCH --partition=gpu
+#SBATCH --mem=25GB
+#SBATCH --output=jobs/big_training_2_output1.log
+#SBATCH --error=jobs/big_training_2_error1.log
+
+
+source env.sh
+export APPTAINER_TMPDIR=/work/gkrzmanc/singularity_tmp
+export APPTAINER_CACHEDIR=/work/gkrzmanc/singularity_cache
+nvidia-smi
+srun singularity exec -B /ceph/hpc/home/krzmancg --nv docker://gkrz/lgatr:v3  python -m src.train -train scouting_PFNano_signals2/SVJ_hadronic_std3/s-channel_mMed-1100_mDark-20_rinv-0.3_alpha-peak_13TeV-pythia8_n-2000 scouting_PFNano_signals2/SVJ_hadronic_std3/s-channel_mMed-1500_mDark-20_rinv-0.3_alpha-peak_13TeV-pythia8_n-2000 scouting_PFNano_signals2/SVJ_hadronic_std3/s-channel_mMed-1100_mDark-20_rinv-0.5_alpha-peak_13TeV-pythia8_n-2000 scouting_PFNano_signals2/SVJ_hadronic_std3/s-channel_mMed-1500_mDark-20_rinv-0.5_alpha-peak_13TeV-pythia8_n-2000 scouting_PFNano_signals2/SVJ_hadronic_std3/s-channel_mMed-1100_mDark-20_rinv-0.7_alpha-peak_13TeV-pythia8_n-2000 scouting_PFNano_signals2/SVJ_hadronic_std3/s-channel_mMed-1500_mDark-20_rinv-0.7_alpha-peak_13TeV-pythia8_n-2000 scouting_PFNano_signals2/SVJ_hadronic_std3/s-channel_mMed-1200_mDark-20_rinv-0.3_alpha-peak_13TeV-pythia8_n-2000 scouting_PFNano_signals2/SVJ_hadronic_std3/s-channel_mMed-700_mDark-20_rinv-0.3_alpha-peak_13TeV-pythia8_n-2000 scouting_PFNano_signals2/SVJ_hadronic_std3/s-channel_mMed-1200_mDark-20_rinv-0.5_alpha-peak_13TeV-pythia8_n-2000 scouting_PFNano_signals2/SVJ_hadronic_std3/s-channel_mMed-700_mDark-20_rinv-0.5_alpha-peak_13TeV-pythia8_n-2000 scouting_PFNano_signals2/SVJ_hadronic_std3/s-channel_mMed-1200_mDark-20_rinv-0.7_alpha-peak_13TeV-pythia8_n-2000 scouting_PFNano_signals2/SVJ_hadronic_std3/s-channel_mMed-700_mDark-20_rinv-0.7_alpha-peak_13TeV-pythia8_n-2000 scouting_PFNano_signals2/SVJ_hadronic_std3/s-channel_mMed-1300_mDark-20_rinv-0.3_alpha-peak_13TeV-pythia8_n-2000 scouting_PFNano_signals2/SVJ_hadronic_std3/s-channel_mMed-800_mDark-20_rinv-0.3_alpha-peak_13TeV-pythia8_n-2000 scouting_PFNano_signals2/SVJ_hadronic_std3/s-channel_mMed-1300_mDark-20_rinv-0.5_alpha-peak_13TeV-pythia8_n-2000 scouting_PFNano_signals2/SVJ_hadronic_std3/s-channel_mMed-800_mDark-20_rinv-0.5_alpha-peak_13TeV-pythia8_n-2000 scouting_PFNano_signals2/SVJ_hadronic_std3/s-channel_mMed-1300_mDark-20_rinv-0.7_alpha-peak_13TeV-pythia8_n-2000 scouting_PFNano_signals2/SVJ_hadronic_std3/s-channel_mMed-800_mDark-20_rinv-0.7_alpha-peak_13TeV-pythia8_n-2000 scouting_PFNano_signals2/SVJ_hadronic_std3/s-channel_mMed-1400_mDark-20_rinv-0.3_alpha-peak_13TeV-pythia8_n-2000 scouting_PFNano_signals2/SVJ_hadronic_std3/s-channel_mMed-900_mDark-20_rinv-0.3_alpha-peak_13TeV-pythia8_n-2000 scouting_PFNano_signals2/SVJ_hadronic_std3/s-channel_mMed-1400_mDark-20_rinv-0.5_alpha-peak_13TeV-pythia8_n-2000 scouting_PFNano_signals2/SVJ_hadronic_std3/s-channel_mMed-900_mDark-20_rinv-0.5_alpha-peak_13TeV-pythia8_n-2000 scouting_PFNano_signals2/SVJ_hadronic_std3/s-channel_mMed-1400_mDark-20_rinv-0.7_alpha-peak_13TeV-pythia8_n-2000 scouting_PFNano_signals2/SVJ_hadronic_std3/s-channel_mMed-900_mDark-20_rinv-0.7_alpha-peak_13TeV-pythia8_n-2000  -val scouting_PFNano_signals2/SVJ_hadronic_std2/s-channel_mMed-1000_mDark-20_rinv-0.3 scouting_PFNano_signals2/SVJ_hadronic_std2/s-channel_mMed-1500_mDark-20_rinv-0.5 scouting_PFNano_signals2/SVJ_hadronic_std2/s-channel_mMed-700_mDark-20_rinv-0.7 scouting_PFNano_signals2/SVJ_hadronic_std2/s-channel_mMed-900_mDark-20_rinv-0.3 scouting_PFNano_signals2/SVJ_hadronic_std2/s-channel_mMed-1000_mDark-20_rinv-0.5 scouting_PFNano_signals2/SVJ_hadronic_std2/s-channel_mMed-1500_mDark-20_rinv-0.7 scouting_PFNano_signals2/SVJ_hadronic_std2/s-channel_mMed-800_mDark-20_rinv-0.3 scouting_PFNano_signals2/SVJ_hadronic_std2/s-channel_mMed-900_mDark-20_rinv-0.5 scouting_PFNano_signals2/SVJ_hadronic_std2/s-channel_mMed-1000_mDark-20_rinv-0.7 scouting_PFNano_signals2/SVJ_hadronic_std2/s-channel_mMed-700_mDark-20_rinv-0.3 scouting_PFNano_signals2/SVJ_hadronic_std2/s-channel_mMed-800_mDark-20_rinv-0.5 scouting_PFNano_signals2/SVJ_hadronic_std2/s-channel_mMed-900_mDark-20_rinv-0.7 scouting_PFNano_signals2/SVJ_hadronic_std2/s-channel_mMed-1500_mDark-20_rinv-0.3 scouting_PFNano_signals2/SVJ_hadronic_std2/s-channel_mMed-700_mDark-20_rinv-0.5 scouting_PFNano_signals2/SVJ_hadronic_std2/s-channel_mMed-800_mDark-20_rinv-0.7  -net src/models/LGATr/lgatr.py -bs 256 --gpus 0 --run-name Train_LGATr_SB_All_data_CONT --val-dataset-size 4000 --num-epochs 1000 --attr-loss-weight 0.1 --coord-loss-weight 0.1 --beta-type pt+bc --spatial-part-only --validation-steps 1000 --load-model-weights train/Train_LGATr_SB_All_data_2025_01_14_13_19_34/step_5000_epoch_1.ckpt
+
+# sbatch jobs/BigTraining_2_spatial_part_only_vega.slurm

jobs/IRC_training/Delphes_training_t3_NoPID_augment.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+sbatch <<EOT
+#!/bin/bash
+#SBATCH --partition=gpu               # Specify the partition
+#SBATCH --mem=25000                   # Request 10GB of memory
+#SBATCH --time=48:00:00
+#SBATCH --job-name=DelphesTrainSVJ  # Name the job
+#SBATCH --gres=gpu:1
+#SBATCH --account=gpu_gres            # Specify the account
+#SBATCH --output=jobs/vega/Dtr_lgatr_aug.out
+#SBATCH --error=jobs/vega/Dtr_lgatr_aug.err
+
+source env.sh
+export APPTAINER_TMPDIR=/work/gkrzmanc/singularity_tmp
+export APPTAINER_CACHEDIR=/work/gkrzmanc/singularity_cache
+nvidia-smi
+
+srun singularity exec  -B /t3home/gkrzmanc/ -B /work/gkrzmanc/ -B /pnfs/psi.ch/cms/trivcat/store/user/gkrzmanc  --nv docker://gkrz/lgatr:v3 python -m src.train  -train Delphes_020425_train2_PU_PFfix_part0/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part1/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part2/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part3/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part4/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part5/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part6/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak  Delphes_020425_train2_PU_PFfix_part7/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak  Delphes_020425_train2_PU_PFfix_part8/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak  -val Delphes_020425_train2_PU_PFfix_part9/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak -val scouting_PFNano_signals2/SVJ_hadronic_std2/s-channel_mMed-700_mDark-20_rinv-0.7 -net src/models/LGATr/lgatr.py -bs 8 --gpus 0 --run-name LGATr_Aug --val-dataset-size 150 --num-steps 200000 --attr-loss-weight 0.1 --coord-loss-weight 0.1 --beta-type pt+bc --gt-radius 0.8 --num-blocks 10 -mv-ch 16 -s-ch 64 --spatial-part-only --validation-steps 60 --no-pid --augment-soft-particles --load-model-weights train/LGATr_training_NoPID_Delphes_PU_PFfix_10_16_64_0.8_2025_05_03_18_35_53_134/step_70000_epoch_16.ckpt --num-workers 0
+
+exit 0
+EOT
+
+# Args: n_blocks mv_channels s_channels radius (default: 10, 16, 64, 0.8)
+# bash jobs/IRC_training/Delphes_training_t3_NoPID_augment.sh

jobs/IRC_training/Delphes_training_t3_NoPID_augment_IRC.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+sbatch <<EOT
+#!/bin/bash
+#SBATCH --partition=gpu               # Specify the partition
+#SBATCH --mem=25000                   # Request 10GB of memory
+#SBATCH --time=48:00:00
+#SBATCH --job-name=DelphesTrainSVJ  # Name the job
+#SBATCH --gres=gpu:1
+#SBATCH --account=gpu_gres            # Specify the account
+#SBATCH --output=jobs/vega/Dtr_lgatr_aug.out
+#SBATCH --error=jobs/vega/Dtr_lgatr_aug.err
+
+source env.sh
+export APPTAINER_TMPDIR=/work/gkrzmanc/singularity_tmp
+export APPTAINER_CACHEDIR=/work/gkrzmanc/singularity_cache
+nvidia-smi
+srun singularity exec  -B /t3home/gkrzmanc/ -B /work/gkrzmanc/ -B /pnfs/psi.ch/cms/trivcat/store/user/gkrzmanc  --nv docker://gkrz/lgatr:v3 python -m src.train  -train Delphes_020425_train2_PU_PFfix_part0/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part1/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part2/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part3/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part4/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part5/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part6/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak  Delphes_020425_train2_PU_PFfix_part7/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak  Delphes_020425_train2_PU_PFfix_part8/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak  -val Delphes_020425_train2_PU_PFfix_part9/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak  -net src/models/LGATr/lgatr.py -bs 8 --gpus 0 --run-name Delphes_Aug_IRCSplit_CONT --val-dataset-size 150 --num-steps 200000 --attr-loss-weight 0.1 --coord-loss-weight 0.1 --beta-type pt+bc --gt-radius 0.8 --num-blocks 10 -mv-ch 16 -s-ch 64 --spatial-part-only --validation-steps 60 --no-pid --augment-soft-particles --load-model-weights train/Delphes_Aug_IRCSplit_2025_05_06_10_09_00_567/step_8820_epoch_1.ckpt  --num-workers 0 -irc
+
+
+exit 0
+EOT
+
+# bash jobs/IRC_training/Delphes_training_t3_NoPID_augment_IRC.sh
+

jobs/IRC_training/Delphes_training_t3_NoPID_augment_IRC_SN.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+sbatch <<EOT
+#!/bin/bash
+#SBATCH --partition=gpu               # Specify the partition
+#SBATCH --mem=25000                   # Request 10GB of memory
+#SBATCH --time=48:00:00
+#SBATCH --job-name=DelphesTrainSVJ  # Name the job
+#SBATCH --gres=gpu:1
+#SBATCH --account=gpu_gres            # Specify the account
+#SBATCH --output=jobs/vega/Dtr_lgatr_aug.out
+#SBATCH --error=jobs/vega/Dtr_lgatr_aug.err
+
+source env.sh
+export APPTAINER_TMPDIR=/work/gkrzmanc/singularity_tmp
+export APPTAINER_CACHEDIR=/work/gkrzmanc/singularity_cache
+nvidia-smi
+srun singularity exec  -B /t3home/gkrzmanc/ -B /work/gkrzmanc/ -B /pnfs/psi.ch/cms/trivcat/store/user/gkrzmanc  --nv docker://gkrz/lgatr:v3 python -m src.train  -train Delphes_020425_train2_PU_PFfix_part0/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part1/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part2/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part3/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part4/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part5/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part6/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak  Delphes_020425_train2_PU_PFfix_part7/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak  Delphes_020425_train2_PU_PFfix_part8/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak  -val Delphes_020425_train2_PU_PFfix_part9/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak  -net src/models/LGATr/lgatr.py -bs 8 --gpus 0 --run-name Delphes_Aug_IRC_Split_and_Noise_CONT --val-dataset-size 150 --num-steps 200000 --attr-loss-weight 0.1 --coord-loss-weight 0.1 --beta-type pt+bc --gt-radius 0.8 --num-blocks 10 -mv-ch 16 -s-ch 64 --spatial-part-only --validation-steps 60 --no-pid --augment-soft-particles --num-workers 0 -irc --load-model-weights train/LGATr_training_NoPID_Delphes_PU_PFfix_10_16_64_0.8_2025_05_03_18_35_53_134/step_70000_epoch_16.ckpt
+
+
+exit 0
+EOT
+
+# bash jobs/IRC_training/Delphes_training_t3_NoPID_augment_IRC_SN.sh
+

jobs/IRC_training/start_at_30k/Delphes_training_t3_NoPID_augment.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+sbatch <<EOT
+#!/bin/bash
+#SBATCH --partition=gpu               # Specify the partition
+#SBATCH --mem=25000                   # Request 10GB of memory
+#SBATCH --time=48:00:00
+#SBATCH --job-name=DelphesTrainSVJ  # Name the job
+#SBATCH --gres=gpu:1
+#SBATCH --account=gpu_gres            # Specify the account
+#SBATCH --output=jobs/vega/Dtr_lgatr_aug.out
+#SBATCH --error=jobs/vega/Dtr_lgatr_aug.err
+
+source env.sh
+export APPTAINER_TMPDIR=/work/gkrzmanc/singularity_tmp
+export APPTAINER_CACHEDIR=/work/gkrzmanc/singularity_cache
+nvidia-smi
+
+srun singularity exec  -B /t3home/gkrzmanc/ -B /work/gkrzmanc/ -B /pnfs/psi.ch/cms/trivcat/store/user/gkrzmanc  --nv docker://gkrz/lgatr:v3 python -m src.train  -train Delphes_020425_train2_PU_PFfix_part0/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part1/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part2/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part3/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part4/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part5/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part6/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak  Delphes_020425_train2_PU_PFfix_part7/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak  Delphes_020425_train2_PU_PFfix_part8/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak  -val Delphes_020425_train2_PU_PFfix_part9/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak -val scouting_PFNano_signals2/SVJ_hadronic_std2/s-channel_mMed-700_mDark-20_rinv-0.7 -net src/models/LGATr/lgatr.py -bs 8 --gpus 0 --run-name LGATr_Aug_30k --val-dataset-size 150 --num-steps 200000 --attr-loss-weight 0.1 --coord-loss-weight 0.1 --beta-type pt+bc --gt-radius 0.8 --num-blocks 10 -mv-ch 16 -s-ch 64 --spatial-part-only --validation-steps 60 --no-pid --augment-soft-particles --load-model-weights train/LGATr_training_NoPID_Delphes_PU_PFfix_10_16_64_0.8_2025_05_03_18_35_53_134/step_30000_epoch_7.ckpt  --num-workers 0
+
+exit 0
+EOT
+
+# Args: n_blocks mv_channels s_channels radius (default: 10, 16, 64, 0.8)
+# bash jobs/IRC_training/start_at_30k/Delphes_training_t3_NoPID_augment.sh

jobs/IRC_training/start_at_30k/Delphes_training_t3_NoPID_augment_IRC.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+sbatch <<EOT
+#!/bin/bash
+#SBATCH --partition=gpu               # Specify the partition
+#SBATCH --mem=25000                   # Request 10GB of memory
+#SBATCH --time=48:00:00
+#SBATCH --job-name=DelphesTrainSVJ  # Name the job
+#SBATCH --gres=gpu:1
+#SBATCH --account=gpu_gres            # Specify the account
+#SBATCH --output=jobs/vega/Dtr_lgatr_aug.out
+#SBATCH --error=jobs/vega/Dtr_lgatr_aug.err
+
+source env.sh
+export APPTAINER_TMPDIR=/work/gkrzmanc/singularity_tmp
+export APPTAINER_CACHEDIR=/work/gkrzmanc/singularity_cache
+nvidia-smi
+
+
+srun singularity exec  -B /t3home/gkrzmanc/ -B /work/gkrzmanc/ -B /pnfs/psi.ch/cms/trivcat/store/user/gkrzmanc  --nv docker://gkrz/lgatr:v3 python -m src.train  -train Delphes_020425_train2_PU_PFfix_part0/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part1/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part2/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part3/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part4/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part5/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part6/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak  Delphes_020425_train2_PU_PFfix_part7/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak  Delphes_020425_train2_PU_PFfix_part8/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak  -val Delphes_020425_train2_PU_PFfix_part9/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak  -net src/models/LGATr/lgatr.py -bs 8 --gpus 0 --run-name Delphes_Aug_IRCSplit_30k_from10k --val-dataset-size 150 --num-steps 200000 --attr-loss-weight 0.1 --coord-loss-weight 0.1 --beta-type pt+bc --gt-radius 0.8 --num-blocks 10 -mv-ch 16 -s-ch 64 --spatial-part-only --validation-steps 60 --no-pid --augment-soft-particles --load-model-weights  train/LGATr_training_NoPID_Delphes_PU_PFfix_10_16_64_0.8_2025_05_03_18_35_53_134/step_30000_epoch_7.ckpt  --num-workers 0 -irc
+
+
+exit 0
+EOT
+
+# bash jobs/IRC_training/start_at_30k/Delphes_training_t3_NoPID_augment_IRC.sh
+####

jobs/IRC_training/start_at_50k/Delphes_training_t3_NoPID_augment.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+sbatch <<EOT
+#!/bin/bash
+#SBATCH --partition=gpu               # Specify the partition
+#SBATCH --mem=25000                   # Request 10GB of memory
+#SBATCH --time=48:00:00
+#SBATCH --job-name=DelphesTrainSVJ  # Name the job
+#SBATCH --gres=gpu:1
+#SBATCH --account=gpu_gres            # Specify the account
+#SBATCH --output=jobs/vega/Dtr_lgatr_aug.out
+#SBATCH --error=jobs/vega/Dtr_lgatr_aug.err
+
+source env.sh
+export APPTAINER_TMPDIR=/work/gkrzmanc/singularity_tmp
+export APPTAINER_CACHEDIR=/work/gkrzmanc/singularity_cache
+nvidia-smi
+
+srun singularity exec  -B /t3home/gkrzmanc/ -B /work/gkrzmanc/ -B /pnfs/psi.ch/cms/trivcat/store/user/gkrzmanc  --nv docker://gkrz/lgatr:v3 python -m src.train  -train Delphes_020425_train2_PU_PFfix_part0/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part1/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part2/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part3/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part4/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part5/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part6/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak  Delphes_020425_train2_PU_PFfix_part7/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak  Delphes_020425_train2_PU_PFfix_part8/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak  -val Delphes_020425_train2_PU_PFfix_part9/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak -val scouting_PFNano_signals2/SVJ_hadronic_std2/s-channel_mMed-700_mDark-20_rinv-0.7 -net src/models/LGATr/lgatr.py -bs 8 --gpus 0 --run-name LGATr_Aug_50k --val-dataset-size 150 --num-steps 200000 --attr-loss-weight 0.1 --coord-loss-weight 0.1 --beta-type pt+bc --gt-radius 0.8 --num-blocks 10 -mv-ch 16 -s-ch 64 --spatial-part-only --validation-steps 60 --no-pid --augment-soft-particles --load-model-weights train/LGATr_training_NoPID_Delphes_PU_PFfix_10_16_64_0.8_2025_05_03_18_35_53_134/step_50000_epoch_12.ckpt --num-workers 0
+
+exit 0
+EOT
+
+# Args: n_blocks mv_channels s_channels radius (default: 10, 16, 64, 0.8)
+# bash jobs/IRC_training/start_at_50k/Delphes_training_t3_NoPID_augment.sh

jobs/IRC_training/start_at_50k/Delphes_training_t3_NoPID_augment_IRC.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+sbatch <<EOT
+#!/bin/bash
+#SBATCH --partition=gpu               # Specify the partition
+#SBATCH --mem=25000                   # Request 10GB of memory
+#SBATCH --time=48:00:00
+#SBATCH --job-name=DelphesTrainSVJ  # Name the job
+#SBATCH --gres=gpu:1
+#SBATCH --account=gpu_gres            # Specify the account
+#SBATCH --output=jobs/vega/Dtr_lgatr_aug.out
+#SBATCH --error=jobs/vega/Dtr_lgatr_aug.err
+
+source env.sh
+export APPTAINER_TMPDIR=/work/gkrzmanc/singularity_tmp
+export APPTAINER_CACHEDIR=/work/gkrzmanc/singularity_cache
+nvidia-smi
+#srun singularity exec  -B /t3home/gkrzmanc/ -B /work/gkrzmanc/ -B /pnfs/psi.ch/cms/trivcat/store/user/gkrzmanc  --nv docker://gkrz/lgatr:v3 python -m src.train  -train Delphes_020425_train2_PU_PFfix_part0/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part1/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part2/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part3/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part4/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part5/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part6/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak  Delphes_020425_train2_PU_PFfix_part7/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak  Delphes_020425_train2_PU_PFfix_part8/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak  -val Delphes_020425_train2_PU_PFfix_part9/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak  -net src/models/LGATr/lgatr.py -bs 8 --gpus 0 --run-name Delphes_Aug_IRCSplit_50k --val-dataset-size 150 --num-steps 200000 --attr-loss-weight 0.1 --coord-loss-weight 0.1 --beta-type pt+bc --gt-radius 0.8 --num-blocks 10 -mv-ch 16 -s-ch 64 --spatial-part-only --validation-steps 60 --no-pid --augment-soft-particles --load-model-weights train/LGATr_training_NoPID_Delphes_PU_PFfix_10_16_64_0.8_2025_05_03_18_35_53_134/step_50000_epoch_12.ckpt  --num-workers 0 -irc
+srun singularity exec  -B /t3home/gkrzmanc/ -B /work/gkrzmanc/ -B /pnfs/psi.ch/cms/trivcat/store/user/gkrzmanc  --nv docker://gkrz/lgatr:v3 python -m src.train  -train Delphes_020425_train2_PU_PFfix_part0/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part1/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part2/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part3/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part4/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part5/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part6/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak  Delphes_020425_train2_PU_PFfix_part7/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak  Delphes_020425_train2_PU_PFfix_part8/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak  -val Delphes_020425_train2_PU_PFfix_part9/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak  -net src/models/LGATr/lgatr.py -bs 8 --gpus 0 --run-name Delphes_Aug_IRCSplit_50k_from10k --val-dataset-size 150 --num-steps 200000 --attr-loss-weight 0.1 --coord-loss-weight 0.1 --beta-type pt+bc --gt-radius 0.8 --num-blocks 10 -mv-ch 16 -s-ch 64 --spatial-part-only --validation-steps 60 --no-pid --augment-soft-particles --load-model-weights train/Delphes_Aug_IRCSplit_50k_2025_05_09_15_22_38_956/step_13620_epoch_2.ckpt  --num-workers 0 -irc
+
+-irc
+exit 0
+EOT
+
+# bash jobs/IRC_training/start_at_50k/Delphes_training_t3_NoPID_augment_IRC.sh
+####

jobs/IRC_training/start_at_50k/Delphes_training_t3_NoPID_augment_IRCSN.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+sbatch <<EOT
+#!/bin/bash
+#SBATCH --partition=gpu               # Specify the partition
+#SBATCH --mem=25000                   # Request 10GB of memory
+#SBATCH --time=48:00:00
+#SBATCH --job-name=DelphesTrainSVJ  # Name the job
+#SBATCH --gres=gpu:1
+#SBATCH --account=gpu_gres            # Specify the account
+#SBATCH --output=jobs/vega/Dtr_lgatr_aug.out
+#SBATCH --error=jobs/vega/Dtr_lgatr_aug.err
+
+source env.sh
+export APPTAINER_TMPDIR=/work/gkrzmanc/singularity_tmp
+export APPTAINER_CACHEDIR=/work/gkrzmanc/singularity_cache
+nvidia-smi
+
+srun singularity exec  -B /t3home/gkrzmanc/ -B /work/gkrzmanc/ -B /pnfs/psi.ch/cms/trivcat/store/user/gkrzmanc -H /t3home/gkrzmanc  --nv docker://gkrz/lgatr:v3 python -m src.train  -train Delphes_020425_train2_PU_PFfix_part0/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part1/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part2/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part3/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part4/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part5/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part6/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak  Delphes_020425_train2_PU_PFfix_part7/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak  Delphes_020425_train2_PU_PFfix_part8/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak  -val Delphes_020425_train2_PU_PFfix_part9/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak  -net src/models/LGATr/lgatr.py -bs 8 --gpus 0 --run-name Delphes_Aug_IRCSplit_50k_SN_from3kFT --val-dataset-size 150 --num-steps 200000 --attr-loss-weight 0.1 --coord-loss-weight 0.1 --beta-type pt+bc --gt-radius 0.8 --num-blocks 10 -mv-ch 16 -s-ch 64 --spatial-part-only --validation-steps 60 --no-pid --augment-soft-particles --load-model-weights  train/Delphes_Aug_IRCSplit_50k_SN_2025_05_12_13_57_45_477/step_3060_epoch_1.ckpt  --num-workers 0 -irc
+
+
+exit 0
+EOT
+
+# bash jobs/IRC_training/start_at_50k/Delphes_training_t3_NoPID_augment_IRCSN.sh
+####

jobs/IRC_training/start_at_50k/Delphes_training_t3_NoPID_augment_IRC_noaug.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+sbatch <<EOT
+#!/bin/bash
+#SBATCH --partition=gpu               # Specify the partition
+#SBATCH --mem=25000                   # Request 10GB of memory
+#SBATCH --time=48:00:00
+#SBATCH --job-name=DelphesTrainSVJ  # Name the job
+#SBATCH --gres=gpu:1
+#SBATCH --account=gpu_gres            # Specify the account
+#SBATCH --output=jobs/vega/Dtr_lgatr_augGPU.out
+#SBATCH --error=jobs/vega/Dtr_lgatr_augGPU.err
+
+source env.sh
+export APPTAINER_TMPDIR=/work/gkrzmanc/singularity_tmp
+export APPTAINER_CACHEDIR=/work/gkrzmanc/singularity_cache
+export PATH=/t3home/gkrzmanc/.local/lib/site-packages:$PATH
+nvidia-smi
+env
+echo " ---- end env ---- "
+
+srun singularity exec  -B /work/gkrzmanc/ -B /pnfs/psi.ch/cms/trivcat/store/user/gkrzmanc -B /t3home/gkrzmanc  -H /t3home/gkrzmanc    --nv docker://gkrz/lgatr:v3  python -c "import fastjet"
+echo "Hello"
+srun singularity exec  -B /work/gkrzmanc/ -B /pnfs/psi.ch/cms/trivcat/store/user/gkrzmanc   -B /t3home/gkrzmanc   -H /t3home/gkrzmanc   --nv docker://gkrz/lgatr:v3  python -m src.train  -train Delphes_020425_train2_PU_PFfix_part0/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part1/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part2/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part3/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part4/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part5/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part6/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak  Delphes_020425_train2_PU_PFfix_part7/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak  Delphes_020425_train2_PU_PFfix_part8/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak  -val Delphes_020425_train2_PU_PFfix_part9/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak  -net src/models/LGATr/lgatr.py -bs 8 --gpus 0 --run-name Delphes_NOAug_IRCSplit_50k_cont --val-dataset-size 150 --num-steps 200000 --attr-loss-weight 0.1 --coord-loss-weight 0.1 --beta-type pt+bc --gt-radius 0.8 --num-blocks 10 -mv-ch 16 -s-ch 64 --spatial-part-only --validation-steps 60 --no-pid --load-model-weights  train/Delphes_NOAug_IRCSplit_50k__2025_05_13_09_56_39_345/step_2460_epoch_1.ckpt   --num-workers 0 -irc
+
+exit 0
+EOT
+
+
+
+# bash jobs/IRC_training/start_at_50k/Delphes_training_t3_NoPID_augment_IRC_noaug.sh
+####
+
+
+

jobs/IRC_training/start_at_50k/test.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+sbatch <<EOT
+  #!/bin/bash
+  #SBATCH --partition=short               # Specify the partition
+  #SBATCH --mem=25000                   # Request 10GB of memory
+  #SBATCH --time=00:01:00
+  #SBATCH --job-name=test1  # Name the job
+  #SBATCH --account=t3            # Specify the account
+  #SBATCH --output=jobs/vega/Dtr_lgatr_aug.out
+  #SBATCH --error=jobs/vega/Dtr_lgatr_aug.err
+
+  source env.sh
+  export APPTAINER_TMPDIR=/work/gkrzmanc/singularity_tmp
+  export APPTAINER_CACHEDIR=/work/gkrzmanc/singularity_cache
+
+nvidia-smi
+env
+echo " ---- end env ---- "
+srun singularity exec  -B /t3home/gkrzmanc/ -B /work/gkrzmanc/ -B /pnfs/psi.ch/cms/trivcat/store/user/gkrzmanc -H /t3home/gkrzmanc  --nv docker://gkrz/lgatr:v3 python -m pip install  --no-input fastjet
+
+srun singularity exec  -B /t3home/gkrzmanc/ -B /work/gkrzmanc/ -B /pnfs/psi.ch/cms/trivcat/store/user/gkrzmanc -H /t3home/gkrzmanc    --nv docker://gkrz/lgatr:v3 python -c "import fastjet"
+echo "Hello"
+srun singularity exec  -B /t3home/gkrzmanc/ -B /work/gkrzmanc/ -B /pnfs/psi.ch/cms/trivcat/store/user/gkrzmanc  -H /t3home/gkrzmanc   --nv docker://gkrz/lgatr:v3 python -m src.train  -train Delphes_020425_train2_PU_PFfix_part0/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part1/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part2/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part3/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part4/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part5/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part6/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak  Delphes_020425_train2_PU_PFfix_part7/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak  Delphes_020425_train2_PU_PFfix_part8/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak  -val Delphes_020425_train2_PU_PFfix_part9/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak  -net src/models/LGATr/lgatr.py -bs 8 --gpus 0 --run-name Delphes_NOAug_IRCSplit_50k_cont --val-dataset-size 150 --num-steps 200000 --attr-loss-weight 0.1 --coord-loss-weight 0.1 --beta-type pt+bc --gt-radius 0.8 --num-blocks 10 -mv-ch 16 -s-ch 64 --spatial-part-only --validation-steps 60 --no-pid --load-model-weights  train/Delphes_NOAug_IRCSplit_50k__2025_05_13_09_56_39_345/step_2460_epoch_1.ckpt   --num-workers 0 -irc
+
+exit 0
+EOT
+
+# bash jobs/IRC_training/start_at_50k/test.sh
+####
+
+
+

jobs/base_training/gatr_training_NoPIDDelphes.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+sbatch <<EOT
+#!/bin/bash
+#SBATCH --partition=gpu               # Specify the partition
+#SBATCH --mem=25000                   # Request 10GB of memory
+#SBATCH --time=48:00:00
+#SBATCH --job-name=DelphesSVJTrain  # Name the job
+#SBATCH --gres=gpu:1
+#SBATCH --account=gpu_gres             # Specify the account
+#SBATCH --output=jobs/vega/DTrGATr_out_$1_$2_$3_R$4.log
+#SBATCH --error=jobs/vega/DTrGATr_err_$1_$2_$3_R$4.log
+
+source env.sh
+export APPTAINER_TMPDIR=/work/gkrzmanc/singularity_tmp
+export APPTAINER_CACHEDIR=/work/gkrzmanc/singularity_cache
+nvidia-smi
+
+srun singularity exec  -B /work -B /pnfs -B /t3home --nv docker://gkrz/lgatr:v3 python -m src.train -train Delphes_020425_train2_PU_PFfix_part0/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part1/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part2/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part3/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part4/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part5/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part6/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak  Delphes_020425_train2_PU_PFfix_part7/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak  Delphes_020425_train2_PU_PFfix_part8/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak  -val Delphes_020425_train2_PU_PFfix_part9/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak -net src/models/GATr/Gatr.py -bs 20  --gpus 0 --run-name GATr_training_NoPID_Delphes_PU_CoordFix_CONT_$1_$2_$3_$4 --val-dataset-size 1000 --num-steps 200000 --attr-loss-weight 0.1 --coord-loss-weight 0.1 --beta-type pt+bc --gt-radius $4 --num-blocks $1 -mv-ch $2 -s-ch $3 --spatial-part-only --validation-steps 2000 --no-pid --load-model-weights train/GATr_training_NoPID_Delphes_PU_CoordFix_10_16_64_0.8_2025_05_05_13_06_27_898/step_50000_epoch_12.ckpt
+
+exit 0
+EOT
+
+
+# Args: n_blocks mv_channels s_channels radius (default: 10, 16, 64, 0.8)
+# bash jobs/base_training/gatr_training_NoPIDDelphes.sh 10 16 64 0.8
+

jobs/base_training/lgatr_training_NoPIDDelphes.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+sbatch <<EOT
+#!/bin/bash
+#SBATCH --partition=gpu               # Specify the partition
+#SBATCH --mem=25000                   # Request 10GB of memory
+#SBATCH --time=48:00:00
+#SBATCH --job-name=DelphesSVJTrain  # Name the job
+#SBATCH --gres=gpu:1
+#SBATCH --account=gpu_gres             # Specify the account
+#SBATCH --output=jobs/vega/DTrLGATr_out_$1_$2_$3_R$4.log
+#SBATCH --error=jobs/vega/DTrLGATr_err_$1_$2_$3_R$4.log
+
+
+source env.sh
+export APPTAINER_TMPDIR=/work/gkrzmanc/singularity_tmp
+export APPTAINER_CACHEDIR=/work/gkrzmanc/singularity_cache
+nvidia-smi
+
+srun singularity exec  -B /work -B /pnfs -B /t3home --nv docker://gkrz/lgatr:v3 python -m src.train -train Delphes_020425_train2_PU_PFfix_part0/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part1/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part2/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part3/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part4/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part5/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part6/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak  Delphes_020425_train2_PU_PFfix_part7/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak  Delphes_020425_train2_PU_PFfix_part8/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak  -val Delphes_020425_train2_PU_PFfix_part9/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak -net src/models/LGATr/lgatr.py -bs 20  --gpus 0 --run-name LGATr_training_NoPID_Delphes_PU_PFfix_$1_$2_$3_$4 --val-dataset-size 1000 --num-steps 200000 --attr-loss-weight 0.1 --coord-loss-weight 0.1 --beta-type pt+bc --gt-radius $4 --num-blocks $1 -mv-ch $2 -s-ch $3 --spatial-part-only --validation-steps 2000 --no-pid
+
+exit 0
+EOT
+
+# Args: n_blocks mv_channels s_channels radius (default: 10, 16, 64, 0.8)
+# bash jobs/base_training/lgatr_700_07.sh 10 16 64 0.8
+

jobs/base_training/transformer_training_NoPIDDelphes.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+sbatch <<EOT
+#!/bin/bash
+#SBATCH --partition=gpu               # Specify the partition
+#SBATCH --mem=25000                   # Request 10GB of memory
+#SBATCH --time=48:00:00
+#SBATCH --job-name=DelphesSVJTrain  # Name the job
+#SBATCH --gres=gpu:1
+#SBATCH --account=gpu_gres             # Specify the account
+#SBATCH --output=jobs/vega/DTr_out_$1_$2_$3_R$4.log
+#SBATCH --error=jobs/vega/DTr_err_$1_$2_$3_R$4.log
+
+source env.sh
+export APPTAINER_TMPDIR=/work/gkrzmanc/singularity_tmp
+export APPTAINER_CACHEDIR=/work/gkrzmanc/singularity_cache
+nvidia-smi
+
+srun singularity exec  -B /work -B /pnfs -B /t3home --nv docker://gkrz/lgatr:v3 python -m src.train -train Delphes_020425_train2_PU_PFfix_part0/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part1/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part2/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part3/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part4/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part5/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part6/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak  Delphes_020425_train2_PU_PFfix_part7/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak  Delphes_020425_train2_PU_PFfix_part8/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak  -val Delphes_020425_train2_PU_PFfix_part9/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak -net src/models/transformer/transformer.py -bs 20  --gpus 0 --run-name Transformer_training_NoPID_Delphes_PU_CoordFix_$1_$2_$3_$4 --val-dataset-size 1000 --num-steps 200000 --attr-loss-weight 0.1 --coord-loss-weight 0.1 --beta-type pt+bc --gt-radius $4 --num-blocks $1 -mv-ch $2 -s-ch $3 --spatial-part-only --validation-steps 2000 --no-pid
+
+exit 0
+EOT
+
+# Args: n_blocks mv_channels s_channels radius (default: 10, 16, 64, 0.8)
+# bash jobs/vega/transformer_training_NoPIDDelphes.sh 10 16 64 0.8

jobs/base_training_different_datasets/aug/lgatr_700_07.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+sbatch <<EOT
+#!/bin/bash
+#SBATCH --partition=gpu               # Specify the partition
+#SBATCH --mem=25000                   # Request 10GB of memory
+#SBATCH --time=48:00:00
+#SBATCH --job-name=DelphesSVJTrain  # Name the job
+#SBATCH --gres=gpu:1
+#SBATCH --account=gpu_gres             # Specify the account
+#SBATCH --output=jobs/vega/DTrLGATr_out_$1_$2_$3_R$4.log
+#SBATCH --error=jobs/vega/DTrLGATr_err_$1_$2_$3_R$4.log
+
+
+source env.sh
+export APPTAINER_TMPDIR=/work/gkrzmanc/singularity_tmp
+export APPTAINER_CACHEDIR=/work/gkrzmanc/singularity_cache
+nvidia-smi
+
+srun singularity exec  -B /work -B /pnfs -B /t3home -H /t3home/gkrzmanc --nv docker://gkrz/lgatr:v3 python -m src.train -train Delphes_020425_train3_PU_PFfix_part0/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak Delphes_020425_train3_PU_PFfix_part1/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak Delphes_020425_train3_PU_PFfix_part2/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak Delphes_020425_train3_PU_PFfix_part3/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak Delphes_020425_train3_PU_PFfix_part4/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak Delphes_020425_train3_PU_PFfix_part5/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak Delphes_020425_train3_PU_PFfix_part6/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak  Delphes_020425_train3_PU_PFfix_part7/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak  Delphes_020425_train3_PU_PFfix_part8/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak  -val Delphes_020425_train3_PU_PFfix_part9/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak -net src/models/LGATr/lgatr.py -bs 20  --gpus 0 --run-name GP_LGATr_training_NoPID_Delphes_PU_PFfix_700_07_$1_$2_$3_$4 --val-dataset-size 1000 --num-steps 200000 --attr-loss-weight 0.1 --coord-loss-weight 0.1 --beta-type pt+bc --gt-radius $4 --num-blocks $1 -mv-ch $2 -s-ch $3 --spatial-part-only --validation-steps 2000 --no-pid --augment-soft-particles --load-model-weights train/LGATr_training_NoPID_Delphes_PU_PFfix_700_07_10_16_64_0.8_2025_05_16_19_44_46_795/step_50000_epoch_12.ckpt
+
+exit 0
+EOT
+
+# Args: n_blocks mv_channels s_channels radius (default: 10, 16, 64, 0.8)
+# bash jobs/base_training_different_datasets/aug/lgatr_700_07.sh 10 16 64 0.8
+

jobs/base_training_different_datasets/aug/lgatr_700_07_and_900_03.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+sbatch <<EOT
+#!/bin/bash
+#SBATCH --partition=gpu               # Specify the partition
+#SBATCH --mem=25000                   # Request 10GB of memory
+#SBATCH --time=48:00:00
+#SBATCH --job-name=DelphesSVJTrain  # Name the job
+#SBATCH --gres=gpu:1
+#SBATCH --account=gpu_gres             # Specify the account
+#SBATCH --output=jobs/vega/DTrLGATr_out_$1_$2_$3_R$4.log
+#SBATCH --error=jobs/vega/DTrLGATr_err_$1_$2_$3_R$4.log
+
+
+source env.sh
+export APPTAINER_TMPDIR=/work/gkrzmanc/singularity_tmp
+export APPTAINER_CACHEDIR=/work/gkrzmanc/singularity_cache
+nvidia-smi
+
+srun singularity exec  -B /work -B /pnfs -B /t3home -H /t3home/gkrzmanc --nv docker://gkrz/lgatr:v3 python -m src.train -train Delphes_020425_train3_PU_PFfix_part0/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak Delphes_020425_train3_PU_PFfix_part1/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak Delphes_020425_train3_PU_PFfix_part2/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak Delphes_020425_train3_PU_PFfix_part3/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak Delphes_020425_train3_PU_PFfix_part4/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak Delphes_020425_train3_PU_PFfix_part5/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak Delphes_020425_train3_PU_PFfix_part6/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak  Delphes_020425_train3_PU_PFfix_part7/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak  Delphes_020425_train3_PU_PFfix_part8/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak  Delphes_020425_train2_PU_PFfix_part0/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part1/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part2/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part3/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part4/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part5/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part6/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak  Delphes_020425_train2_PU_PFfix_part7/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak  Delphes_020425_train2_PU_PFfix_part8/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak  -val Delphes_020425_train3_PU_PFfix_part9/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak -net src/models/LGATr/lgatr.py -bs 20  --gpus 0 --run-name GP_LGATr_training_NoPID_Delphes_PU_PFfix_700_07_AND_900_03_$1_$2_$3_$4 --val-dataset-size 1000 --num-steps 200000 --attr-loss-weight 0.1 --coord-loss-weight 0.1 --beta-type pt+bc --gt-radius $4 --num-blocks $1 -mv-ch $2 -s-ch $3 --spatial-part-only --validation-steps 2000 --no-pid --augment-soft-particles --load-model-weights train/LGATr_training_NoPID_Delphes_PU_PFfix_700_07_AND_900_03_10_16_64_0.8_2025_05_16_21_04_26_991/step_50000_epoch_6.ckpt
+
+exit 0
+EOT
+
+# Args: n_blocks mv_channels s_channels radius (default: 10, 16, 64, 0.8)
+# bash jobs/base_training_different_datasets/aug/lgatr_700_07_and_900_03.sh 10 16 64 0.8

jobs/base_training_different_datasets/aug/lgatr_700_07_and_900_03_and_QCD.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+sbatch <<EOT
+#!/bin/bash
+#SBATCH --partition=gpu               # Specify the partition
+#SBATCH --mem=25000                   # Request 10GB of memory
+#SBATCH --time=48:00:00
+#SBATCH --job-name=DelphesSVJTrain  # Name the job
+#SBATCH --gres=gpu:1
+#SBATCH --account=gpu_gres             # Specify the account
+#SBATCH --output=jobs/vega/DTrLGATr_out_$1_$2_$3_R$4.log
+#SBATCH --error=jobs/vega/DTrLGATr_err_$1_$2_$3_R$4.log
+
+
+source env.sh
+export APPTAINER_TMPDIR=/work/gkrzmanc/singularity_tmp
+export APPTAINER_CACHEDIR=/work/gkrzmanc/singularity_cache
+nvidia-smi
+
+srun singularity exec  -B /work -B /pnfs -B /t3home -H /t3home/gkrzmanc --nv docker://gkrz/lgatr:v3 python -m src.train -train Delphes_020425_train3_PU_PFfix_part0/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak Delphes_020425_train3_PU_PFfix_part1/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak Delphes_020425_train3_PU_PFfix_part2/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak Delphes_020425_train3_PU_PFfix_part3/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak Delphes_020425_train3_PU_PFfix_part4/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak Delphes_020425_train3_PU_PFfix_part5/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak Delphes_020425_train3_PU_PFfix_part6/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak  Delphes_020425_train3_PU_PFfix_part7/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak  Delphes_020425_train3_PU_PFfix_part8/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak  Delphes_020425_train2_PU_PFfix_part0/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part1/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part2/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part3/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part4/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part5/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part6/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak  Delphes_020425_train2_PU_PFfix_part7/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak  Delphes_020425_train2_PU_PFfix_part8/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak  QCDtrain_part0/qcd_test_0 QCDtrain_part0/qcd_test_1 QCDtrain_part0/qcd_test_2 QCDtrain_part0/qcd_test_3 QCDtrain_part0/qcd_test_4 QCDtrain_part0/qcd_test_5 QCDtrain_part0/qcd_test_6 QCDtrain_part0/qcd_test_7 QCDtrain_part0/qcd_test_8  -val Delphes_020425_train3_PU_PFfix_part9/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak -net src/models/LGATr/lgatr.py -bs 20  --gpus 0 --run-name GP_LGATr_training_NoPID_Delphes_PU_PFfix_700_07_AND_900_03_AND_QCD_$1_$2_$3_$4 --val-dataset-size 1000 --num-steps 200000 --attr-loss-weight 0.1 --coord-loss-weight 0.1 --beta-type pt+bc --gt-radius $4 --num-blocks $1 -mv-ch $2 -s-ch $3 --spatial-part-only --validation-steps 2000 --no-pid --augment-soft-particles --load-model-weights train/LGATr_training_NoPID_Delphes_PU_PFfix_700_07_AND_900_03_AND_QCD_10_16_64_0.8_2025_05_16_21_04_26_937/step_50000_epoch_4.ckpt
+
+exit 0
+EOT
+
+# Args: n_blocks mv_channels s_channels radius (default: 10, 16, 64, 0.8)
+# bash jobs/base_training_different_datasets/aug/lgatr_700_07_and_900_03_and_QCD.sh 10 16 64 0.8

jobs/base_training_different_datasets/aug/lgatr_QCD.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+sbatch <<EOT
+#!/bin/bash
+#SBATCH --partition=gpu               # Specify the partition
+#SBATCH --mem=25000                   # Request 10GB of memory
+#SBATCH --time=48:00:00
+#SBATCH --job-name=DelphesSVJTrain  # Name the job
+#SBATCH --gres=gpu:1
+#SBATCH --account=gpu_gres             # Specify the account
+#SBATCH --output=jobs/vega/DTrLGATr_out_$1_$2_$3_R$4.log
+#SBATCH --error=jobs/vega/DTrLGATr_err_$1_$2_$3_R$4.log
+
+
+source env.sh
+export APPTAINER_TMPDIR=/work/gkrzmanc/singularity_tmp
+export APPTAINER_CACHEDIR=/work/gkrzmanc/singularity_cache
+nvidia-smi
+
+srun singularity exec  -B /work -B /pnfs -B /t3home -H /t3home/gkrzmanc --nv docker://gkrz/lgatr:v3 python -m src.train -train QCDtrain_part0/qcd_test_0 QCDtrain_part0/qcd_test_1 QCDtrain_part0/qcd_test_2 QCDtrain_part0/qcd_test_3 QCDtrain_part0/qcd_test_4 QCDtrain_part0/qcd_test_5 QCDtrain_part0/qcd_test_6 QCDtrain_part0/qcd_test_7 QCDtrain_part0/qcd_test_8  -val QCDtrain_part0/qcd_test_9 -net src/models/LGATr/lgatr.py -bs 20  --gpus 0 --run-name GP_LGATr_training_NoPID_Delphes_PU_PFfix_QCD_events_$1_$2_$3_$4 --val-dataset-size 1000 --num-steps 200000 --attr-loss-weight 0.1 --coord-loss-weight 0.1 --beta-type pt+bc --gt-radius $4 --num-blocks $1 -mv-ch $2 -s-ch $3 --spatial-part-only --validation-steps 2000 --no-pid --augment-soft-particles  --load-model-weights train/LGATr_training_NoPID_Delphes_PU_PFfix_QCD_events_10_16_64_0.8_2025_05_16_19_46_57_48/step_50000_epoch_12.ckpt
+
+exit 0
+EOT
+
+# Args: n_blocks mv_channels s_channels radius (default: 10, 16, 64, 0.8)
+# bash jobs/base_training_different_datasets/aug/lgatr_QCD.sh 10 16 64 0.8

jobs/base_training_different_datasets/aug_IRC_S/lgatr_700_07.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+sbatch <<EOT
+#!/bin/bash
+#SBATCH --partition=gpu               # Specify the partition
+#SBATCH --mem=25000                   # Request 10GB of memory
+#SBATCH --time=48:00:00
+#SBATCH --job-name=DelphesSVJTrain  # Name the job
+#SBATCH --gres=gpu:1
+#SBATCH --account=gpu_gres             # Specify the account
+#SBATCH --output=jobs/vega/7DTrLGATr_out_$1_$2_$3_R$4.log
+#SBATCH --error=jobs/vega/7DTrLGATr_err_$1_$2_$3_R$4.log
+
+
+source env.sh
+export APPTAINER_TMPDIR=/work/gkrzmanc/singularity_tmp
+export APPTAINER_CACHEDIR=/work/gkrzmanc/singularity_cache
+nvidia-smi
+
+srun singularity exec  -B /work -B /pnfs -B /t3home -H /t3home/gkrzmanc --nv docker://gkrz/lgatr:v3 python -m src.train -train Delphes_020425_train3_PU_PFfix_part0/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak Delphes_020425_train3_PU_PFfix_part1/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak Delphes_020425_train3_PU_PFfix_part2/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak Delphes_020425_train3_PU_PFfix_part3/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak Delphes_020425_train3_PU_PFfix_part4/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak Delphes_020425_train3_PU_PFfix_part5/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak Delphes_020425_train3_PU_PFfix_part6/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak  Delphes_020425_train3_PU_PFfix_part7/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak  Delphes_020425_train3_PU_PFfix_part8/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak  -val Delphes_020425_train3_PU_PFfix_part9/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak -net src/models/LGATr/lgatr.py -bs 8  --gpus 0 --run-name GP_IRC_S_LGATr_training_NoPID_Delphes_PU_PFfix_700_07_$1_$2_$3_$4 --val-dataset-size 10 --num-steps 200000 --attr-loss-weight 0.1 --coord-loss-weight 0.1 --beta-type pt+bc --gt-radius $4 --num-blocks $1 -mv-ch $2 -s-ch $3 --spatial-part-only --validation-steps 2000 --no-pid --augment-soft-particles --load-model-weights train/LGATr_training_NoPID_Delphes_PU_PFfix_700_07_10_16_64_0.8_2025_05_16_19_44_46_795/step_50000_epoch_12.ckpt  -irc
+
+exit 0
+EOT
+
+# Args: n_blocks mv_channels s_channels radius (default: 10, 16, 64, 0.8)
+# bash jobs/base_training_different_datasets/aug_IRC_S/lgatr_700_07.sh 10 16 64 0.8
+
+

jobs/base_training_different_datasets/aug_IRC_S/lgatr_700_07_and_900_03.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+sbatch <<EOT
+#!/bin/bash
+#SBATCH --partition=gpu               # Specify the partition
+#SBATCH --mem=25000                   # Request 10GB of memory
+#SBATCH --time=48:00:00
+#SBATCH --job-name=DelphesSVJTrain  # Name the job
+#SBATCH --gres=gpu:1
+#SBATCH --account=gpu_gres             # Specify the account
+#SBATCH --output=jobs/vega/700and900_DTrLGATr_out_$1_$2_$3_R$4.log
+#SBATCH --error=jobs/vega/700and900_DTrLGATr_err_$1_$2_$3_R$4.log
+
+
+source env.sh
+export APPTAINER_TMPDIR=/work/gkrzmanc/singularity_tmp
+export APPTAINER_CACHEDIR=/work/gkrzmanc/singularity_cache
+nvidia-smi
+
+srun singularity exec  -B /work -B /pnfs -B /t3home -H /t3home/gkrzmanc --nv docker://gkrz/lgatr:v3 python -m src.train -train Delphes_020425_train3_PU_PFfix_part0/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak Delphes_020425_train3_PU_PFfix_part1/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak Delphes_020425_train3_PU_PFfix_part2/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak Delphes_020425_train3_PU_PFfix_part3/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak Delphes_020425_train3_PU_PFfix_part4/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak Delphes_020425_train3_PU_PFfix_part5/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak Delphes_020425_train3_PU_PFfix_part6/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak  Delphes_020425_train3_PU_PFfix_part7/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak  Delphes_020425_train3_PU_PFfix_part8/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak  Delphes_020425_train2_PU_PFfix_part0/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part1/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part2/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part3/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part4/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part5/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part6/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak  Delphes_020425_train2_PU_PFfix_part7/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak  Delphes_020425_train2_PU_PFfix_part8/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak  -val Delphes_020425_train3_PU_PFfix_part9/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak -net src/models/LGATr/lgatr.py -bs 8  --gpus 0 --run-name GP_IRC_S_LGATr_training_NoPID_Delphes_PU_PFfix_700_07_AND_900_03_$1_$2_$3_$4 --val-dataset-size 10 --num-steps 200000 --attr-loss-weight 0.1 --coord-loss-weight 0.1 --beta-type pt+bc --gt-radius $4 --num-blocks $1 -mv-ch $2 -s-ch $3 --spatial-part-only --validation-steps 2000 --no-pid --augment-soft-particles --load-model-weights train/LGATr_training_NoPID_Delphes_PU_PFfix_700_07_AND_900_03_10_16_64_0.8_2025_05_16_21_04_26_991/step_50000_epoch_6.ckpt  -irc
+
+exit 0
+EOT
+
+# Args: n_blocks mv_channels s_channels radius (default: 10, 16, 64, 0.8)
+# bash jobs/base_training_different_datasets/aug_IRC_S/lgatr_700_07_and_900_03.sh 10 16 64 0.8
+

jobs/base_training_different_datasets/aug_IRC_S/lgatr_700_07_and_900_03_and_QCD.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+sbatch <<EOT
+#!/bin/bash
+#SBATCH --partition=gpu               # Specify the partition
+#SBATCH --mem=25000                   # Request 10GB of memory
+#SBATCH --time=48:00:00
+#SBATCH --job-name=DelphesSVJTrain  # Name the job
+#SBATCH --gres=gpu:1
+#SBATCH --account=gpu_gres             # Specify the account
+#SBATCH --output=jobs/vega/Train_SB1_DTrLGATr_out_$1_$2_$3_R$4.log
+#SBATCH --error=jobs/vega/Train_SB1_DTrLGATr_err_$1_$2_$3_R$4.log
+
+
+source env.sh
+export APPTAINER_TMPDIR=/work/gkrzmanc/singularity_tmp
+export APPTAINER_CACHEDIR=/work/gkrzmanc/singularity_cache
+nvidia-smi
+
+srun singularity exec  -B /work -B /pnfs -B /t3home -H /t3home/gkrzmanc --nv docker://gkrz/lgatr:v3 python -m src.train -train Delphes_020425_train3_PU_PFfix_part0/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak Delphes_020425_train3_PU_PFfix_part1/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak Delphes_020425_train3_PU_PFfix_part2/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak Delphes_020425_train3_PU_PFfix_part3/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak Delphes_020425_train3_PU_PFfix_part4/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak Delphes_020425_train3_PU_PFfix_part5/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak Delphes_020425_train3_PU_PFfix_part6/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak  Delphes_020425_train3_PU_PFfix_part7/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak  Delphes_020425_train3_PU_PFfix_part8/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak  Delphes_020425_train2_PU_PFfix_part0/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part1/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part2/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part3/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part4/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part5/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part6/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak  Delphes_020425_train2_PU_PFfix_part7/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak  Delphes_020425_train2_PU_PFfix_part8/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak  QCDtrain_part0/qcd_test_0 QCDtrain_part0/qcd_test_1 QCDtrain_part0/qcd_test_2 QCDtrain_part0/qcd_test_3 QCDtrain_part0/qcd_test_4 QCDtrain_part0/qcd_test_5 QCDtrain_part0/qcd_test_6 QCDtrain_part0/qcd_test_7 QCDtrain_part0/qcd_test_8  -val Delphes_020425_train3_PU_PFfix_part9/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak -net src/models/LGATr/lgatr.py -bs 8  --gpus 0 --run-name GP_IRC_S_LGATr_training_NoPID_Delphes_PU_PFfix_700_07_AND_900_03_AND_QCD_$1_$2_$3_$4 --val-dataset-size 10 --num-steps 200000 --attr-loss-weight 0.1 --coord-loss-weight 0.1 --beta-type pt+bc --gt-radius $4 --num-blocks $1 -mv-ch $2 -s-ch $3 --spatial-part-only --validation-steps 2000 --no-pid --augment-soft-particles --load-model-weights train/LGATr_training_NoPID_Delphes_PU_PFfix_700_07_AND_900_03_AND_QCD_10_16_64_0.8_2025_05_16_21_04_26_937/step_50000_epoch_4.ckpt  -irc
+
+exit 0
+EOT
+
+# Args: n_blocks mv_channels s_channels radius (default: 10, 16, 64, 0.8)
+# bash jobs/base_training_different_datasets/aug_IRC_S/lgatr_700_07_and_900_03_and_QCD.sh 10 16 64 0.8

jobs/base_training_different_datasets/aug_IRC_S/lgatr_QCD.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+sbatch <<EOT
+#!/bin/bash
+#SBATCH --partition=gpu               # Specify the partition
+#SBATCH --mem=25000                   # Request 10GB of memory
+#SBATCH --time=48:00:00
+#SBATCH --job-name=DelphesSVJTrain  # Name the job
+#SBATCH --gres=gpu:1
+#SBATCH --account=gpu_gres             # Specify the account
+#SBATCH --output=jobs/vega/QCDDTrLGATr_out_$1_$2_$3_R$4.log
+#SBATCH --error=jobs/vega/QCDDTrLGATr_err_$1_$2_$3_R$4.log
+
+
+source env.sh
+export APPTAINER_TMPDIR=/work/gkrzmanc/singularity_tmp
+export APPTAINER_CACHEDIR=/work/gkrzmanc/singularity_cache
+nvidia-smi
+
+srun singularity exec  -B /work -B /pnfs -B /t3home -H /t3home/gkrzmanc --nv docker://gkrz/lgatr:v3 python -m src.train -train QCDtrain_part0/qcd_test_0 QCDtrain_part0/qcd_test_1 QCDtrain_part0/qcd_test_2 QCDtrain_part0/qcd_test_3 QCDtrain_part0/qcd_test_4 QCDtrain_part0/qcd_test_5 QCDtrain_part0/qcd_test_6 QCDtrain_part0/qcd_test_7 QCDtrain_part0/qcd_test_8  -val QCDtrain_part0/qcd_test_9 -net src/models/LGATr/lgatr.py -bs 8  --gpus 0 --run-name GP_IRC_S_LGATr_training_NoPID_Delphes_PU_PFfix_QCD_events_$1_$2_$3_$4 --val-dataset-size 10 --num-steps 200000 --attr-loss-weight 0.1 --coord-loss-weight 0.1 --beta-type pt+bc --gt-radius $4 --num-blocks $1 -mv-ch $2 -s-ch $3 --spatial-part-only --validation-steps 2000 --no-pid --augment-soft-particles  --load-model-weights train/LGATr_training_NoPID_Delphes_PU_PFfix_QCD_events_10_16_64_0.8_2025_05_16_19_46_57_48/step_50000_epoch_12.ckpt -irc
+
+exit 0
+EOT
+
+
+# Args: n_blocks mv_channels s_channels radius (default: 10, 16, 64, 0.8)
+# bash jobs/base_training_different_datasets/aug_IRC_S/lgatr_QCD.sh 10 16 64 0.8

jobs/base_training_different_datasets/aug_IRC_SN/lgatr_700_07.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+sbatch <<EOT
+#!/bin/bash
+#SBATCH --partition=gpu               # Specify the partition
+#SBATCH --mem=25000                   # Request 10GB of memory
+#SBATCH --time=48:00:00
+#SBATCH --job-name=DelphesSVJTrain  # Name the job
+#SBATCH --gres=gpu:1
+#SBATCH --account=gpu_gres             # Specify the account
+#SBATCH --output=jobs/vega/7DTrLGATr_out_$1_$2_$3_R$4.log
+#SBATCH --error=jobs/vega/7DTrLGATr_err_$1_$2_$3_R$4.log
+
+
+source env.sh
+export APPTAINER_TMPDIR=/work/gkrzmanc/singularity_tmp
+export APPTAINER_CACHEDIR=/work/gkrzmanc/singularity_cache
+nvidia-smi
+
+srun singularity exec  -B /work -B /pnfs -B /t3home -H /t3home/gkrzmanc --nv docker://gkrz/lgatr:v3 python -m src.train -train Delphes_020425_train3_PU_PFfix_part0/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak Delphes_020425_train3_PU_PFfix_part1/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak Delphes_020425_train3_PU_PFfix_part2/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak Delphes_020425_train3_PU_PFfix_part3/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak Delphes_020425_train3_PU_PFfix_part4/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak Delphes_020425_train3_PU_PFfix_part5/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak Delphes_020425_train3_PU_PFfix_part6/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak  Delphes_020425_train3_PU_PFfix_part7/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak  Delphes_020425_train3_PU_PFfix_part8/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak  -val Delphes_020425_train3_PU_PFfix_part9/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak -net src/models/LGATr/lgatr.py -bs 8  --gpus 0 --run-name GP_IRC_SN_LGATr_training_NoPID_Delphes_PU_PFfix_700_07_$1_$2_$3_$4 --val-dataset-size 10 --num-steps 200000 --attr-loss-weight 0.1 --coord-loss-weight 0.1 --beta-type pt+bc --gt-radius $4 --num-blocks $1 -mv-ch $2 -s-ch $3 --spatial-part-only --validation-steps 2000 --no-pid --augment-soft-particles --load-model-weights train/LGATr_training_NoPID_Delphes_PU_PFfix_700_07_10_16_64_0.8_2025_05_16_19_44_46_795/step_50000_epoch_12.ckpt  -irc
+
+exit 0
+EOT
+
+# Args: n_blocks mv_channels s_channels radius (default: 10, 16, 64, 0.8)
+# bash jobs/base_training_different_datasets/aug_IRC_S/lgatr_700_07.sh 10 16 64 0.8
+
+

jobs/base_training_different_datasets/aug_IRC_SN/lgatr_700_07_and_900_03.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+sbatch <<EOT
+#!/bin/bash
+#SBATCH --partition=gpu               # Specify the partition
+#SBATCH --mem=25000                   # Request 10GB of memory
+#SBATCH --time=48:00:00
+#SBATCH --job-name=DelphesSVJTrain  # Name the job
+#SBATCH --gres=gpu:1
+#SBATCH --account=gpu_gres             # Specify the account
+#SBATCH --output=jobs/vega/700and900_DTrLGATr_out_$1_$2_$3_R$4.log
+#SBATCH --error=jobs/vega/700and900_DTrLGATr_err_$1_$2_$3_R$4.log
+
+
+source env.sh
+export APPTAINER_TMPDIR=/work/gkrzmanc/singularity_tmp
+export APPTAINER_CACHEDIR=/work/gkrzmanc/singularity_cache
+nvidia-smi
+
+srun singularity exec  -B /work -B /pnfs -B /t3home -H /t3home/gkrzmanc --nv docker://gkrz/lgatr:v3 python -m src.train -train Delphes_020425_train3_PU_PFfix_part0/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak Delphes_020425_train3_PU_PFfix_part1/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak Delphes_020425_train3_PU_PFfix_part2/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak Delphes_020425_train3_PU_PFfix_part3/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak Delphes_020425_train3_PU_PFfix_part4/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak Delphes_020425_train3_PU_PFfix_part5/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak Delphes_020425_train3_PU_PFfix_part6/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak  Delphes_020425_train3_PU_PFfix_part7/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak  Delphes_020425_train3_PU_PFfix_part8/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak  Delphes_020425_train2_PU_PFfix_part0/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part1/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part2/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part3/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part4/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part5/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part6/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak  Delphes_020425_train2_PU_PFfix_part7/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak  Delphes_020425_train2_PU_PFfix_part8/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak  -val Delphes_020425_train3_PU_PFfix_part9/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak -net src/models/LGATr/lgatr.py -bs 8  --gpus 0 --run-name GP_IRC_SN_LGATr_training_NoPID_Delphes_PU_PFfix_700_07_AND_900_03_$1_$2_$3_$4 --val-dataset-size 10 --num-steps 200000 --attr-loss-weight 0.1 --coord-loss-weight 0.1 --beta-type pt+bc --gt-radius $4 --num-blocks $1 -mv-ch $2 -s-ch $3 --spatial-part-only --validation-steps 2000 --no-pid --augment-soft-particles --load-model-weights train/LGATr_training_NoPID_Delphes_PU_PFfix_700_07_AND_900_03_10_16_64_0.8_2025_05_16_21_04_26_991/step_50000_epoch_6.ckpt  -irc
+
+exit 0
+EOT
+
+# Args: n_blocks mv_channels s_channels radius (default: 10, 16, 64, 0.8)
+# bash jobs/base_training_different_datasets/aug_IRC_S/lgatr_700_07_and_900_03.sh 10 16 64 0.8
+

jobs/base_training_different_datasets/aug_IRC_SN/lgatr_700_07_and_900_03_and_QCD.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+sbatch <<EOT
+#!/bin/bash
+#SBATCH --partition=gpu               # Specify the partition
+#SBATCH --mem=25000                   # Request 10GB of memory
+#SBATCH --time=48:00:00
+#SBATCH --job-name=DelphesSVJTrain  # Name the job
+#SBATCH --gres=gpu:1
+#SBATCH --account=gpu_gres             # Specify the account
+#SBATCH --output=jobs/vega/Train_SB1_DTrLGATr_out_$1_$2_$3_R$4.log
+#SBATCH --error=jobs/vega/Train_SB1_DTrLGATr_err_$1_$2_$3_R$4.log
+
+
+source env.sh
+export APPTAINER_TMPDIR=/work/gkrzmanc/singularity_tmp
+export APPTAINER_CACHEDIR=/work/gkrzmanc/singularity_cache
+nvidia-smi
+
+srun singularity exec  -B /work -B /pnfs -B /t3home -H /t3home/gkrzmanc --nv docker://gkrz/lgatr:v3 python -m src.train -train Delphes_020425_train3_PU_PFfix_part0/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak Delphes_020425_train3_PU_PFfix_part1/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak Delphes_020425_train3_PU_PFfix_part2/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak Delphes_020425_train3_PU_PFfix_part3/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak Delphes_020425_train3_PU_PFfix_part4/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak Delphes_020425_train3_PU_PFfix_part5/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak Delphes_020425_train3_PU_PFfix_part6/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak  Delphes_020425_train3_PU_PFfix_part7/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak  Delphes_020425_train3_PU_PFfix_part8/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak  Delphes_020425_train2_PU_PFfix_part0/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part1/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part2/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part3/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part4/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part5/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part6/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak  Delphes_020425_train2_PU_PFfix_part7/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak  Delphes_020425_train2_PU_PFfix_part8/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak  QCDtrain_part0/qcd_test_0 QCDtrain_part0/qcd_test_1 QCDtrain_part0/qcd_test_2 QCDtrain_part0/qcd_test_3 QCDtrain_part0/qcd_test_4 QCDtrain_part0/qcd_test_5 QCDtrain_part0/qcd_test_6 QCDtrain_part0/qcd_test_7 QCDtrain_part0/qcd_test_8  -val Delphes_020425_train3_PU_PFfix_part9/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak -net src/models/LGATr/lgatr.py -bs 8  --gpus 0 --run-name GP_IRC_SN_LGATr_training_NoPID_Delphes_PU_PFfix_700_07_AND_900_03_AND_QCD_$1_$2_$3_$4 --val-dataset-size 10 --num-steps 200000 --attr-loss-weight 0.1 --coord-loss-weight 0.1 --beta-type pt+bc --gt-radius $4 --num-blocks $1 -mv-ch $2 -s-ch $3 --spatial-part-only --validation-steps 2000 --no-pid --augment-soft-particles --load-model-weights train/LGATr_training_NoPID_Delphes_PU_PFfix_700_07_AND_900_03_AND_QCD_10_16_64_0.8_2025_05_16_21_04_26_937/step_50000_epoch_4.ckpt  -irc
+
+exit 0
+EOT
+
+# Args: n_blocks mv_channels s_channels radius (default: 10, 16, 64, 0.8)
+# bash jobs/base_training_different_datasets/aug_IRC_S/lgatr_700_07_and_900_03_and_QCD.sh 10 16 64 0.8

jobs/base_training_different_datasets/aug_IRC_SN/lgatr_900_03.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+sbatch <<EOT
+#!/bin/bash
+#SBATCH --partition=gpu               # Specify the partition
+#SBATCH --mem=25000                   # Request 10GB of memory
+#SBATCH --time=48:00:00
+#SBATCH --job-name=DelphesTrainSVJ  # Name the job
+#SBATCH --gres=gpu:1
+#SBATCH --account=gpu_gres            # Specify the account
+#SBATCH --output=jobs/vega/Dtr_lgatr_aug.out
+#SBATCH --error=jobs/vega/Dtr_lgatr_aug.err
+
+source env.sh
+export APPTAINER_TMPDIR=/work/gkrzmanc/singularity_tmp
+export APPTAINER_CACHEDIR=/work/gkrzmanc/singularity_cache
+nvidia-smi
+
+srun singularity exec  -B /t3home/gkrzmanc/ -B /work/gkrzmanc/ -B /pnfs/psi.ch/cms/trivcat/store/user/gkrzmanc -H /t3home/gkrzmanc  --nv docker://gkrz/lgatr:v3 python -m src.train  -train Delphes_020425_train2_PU_PFfix_part0/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part1/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part2/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part3/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part4/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part5/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part6/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak  Delphes_020425_train2_PU_PFfix_part7/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak  Delphes_020425_train2_PU_PFfix_part8/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak  -val Delphes_020425_train2_PU_PFfix_part9/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak  -net src/models/LGATr/lgatr.py -bs 8 --gpus 0 --run-name Delphes_Aug_IRCSplit_50k_SN_from3kFT --val-dataset-size 150 --num-steps 200000 --attr-loss-weight 0.1 --coord-loss-weight 0.1 --beta-type pt+bc --gt-radius 0.8 --num-blocks 10 -mv-ch 16 -s-ch 64 --spatial-part-only --validation-steps 60 --no-pid --augment-soft-particles --load-model-weights  train/Delphes_Aug_IRCSplit_50k_SN_2025_05_12_13_57_45_477/step_3060_epoch_1.ckpt  --num-workers 0 -irc
+
+
+exit 0
+EOT
+
+# bash jobs/IRC_training/start_at_50k/Delphes_training_t3_NoPID_augment_IRCSN.sh
+####

jobs/base_training_different_datasets/aug_IRC_SN/lgatr_QCD.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+sbatch <<EOT
+#!/bin/bash
+#SBATCH --partition=gpu               # Specify the partition
+#SBATCH --mem=25000                   # Request 10GB of memory
+#SBATCH --time=48:00:00
+#SBATCH --job-name=DelphesSVJTrain  # Name the job
+#SBATCH --gres=gpu:1
+#SBATCH --account=gpu_gres             # Specify the account
+#SBATCH --output=jobs/vega/QCDDTrLGATr_out_$1_$2_$3_R$4.log
+#SBATCH --error=jobs/vega/QCDDTrLGATr_err_$1_$2_$3_R$4.log
+
+
+source env.sh
+export APPTAINER_TMPDIR=/work/gkrzmanc/singularity_tmp
+export APPTAINER_CACHEDIR=/work/gkrzmanc/singularity_cache
+nvidia-smi
+
+srun singularity exec  -B /work -B /pnfs -B /t3home -H /t3home/gkrzmanc --nv docker://gkrz/lgatr:v3 python -m src.train -train QCDtrain_part0/qcd_test_0 QCDtrain_part0/qcd_test_1 QCDtrain_part0/qcd_test_2 QCDtrain_part0/qcd_test_3 QCDtrain_part0/qcd_test_4 QCDtrain_part0/qcd_test_5 QCDtrain_part0/qcd_test_6 QCDtrain_part0/qcd_test_7 QCDtrain_part0/qcd_test_8  -val QCDtrain_part0/qcd_test_9 -net src/models/LGATr/lgatr.py -bs 8  --gpus 0 --run-name GP_IRC_SN_LGATr_training_NoPID_Delphes_PU_PFfix_QCD_events_$1_$2_$3_$4 --val-dataset-size 10 --num-steps 200000 --attr-loss-weight 0.1 --coord-loss-weight 0.1 --beta-type pt+bc --gt-radius $4 --num-blocks $1 -mv-ch $2 -s-ch $3 --spatial-part-only --validation-steps 2000 --no-pid --augment-soft-particles  --load-model-weights train/LGATr_training_NoPID_Delphes_PU_PFfix_QCD_events_10_16_64_0.8_2025_05_16_19_46_57_48/step_50000_epoch_12.ckpt -irc
+
+exit 0
+EOT
+
+
+# Args: n_blocks mv_channels s_channels radius (default: 10, 16, 64, 0.8)
+# bash jobs/base_training_different_datasets/aug_IRC_S/lgatr_QCD.sh 10 16 64 0.8

jobs/base_training_different_datasets/lgatr_700_07.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+sbatch <<EOT
+#!/bin/bash
+#SBATCH --partition=gpu               # Specify the partition
+#SBATCH --mem=25000                   # Request 10GB of memory
+#SBATCH --time=48:00:00
+#SBATCH --job-name=DelphesSVJTrain  # Name the job
+#SBATCH --gres=gpu:1
+#SBATCH --account=gpu_gres             # Specify the account
+#SBATCH --output=jobs/vega/DTrLGATr_out_$1_$2_$3_R$4.log
+#SBATCH --error=jobs/vega/DTrLGATr_err_$1_$2_$3_R$4.log
+
+
+source env.sh
+export APPTAINER_TMPDIR=/work/gkrzmanc/singularity_tmp
+export APPTAINER_CACHEDIR=/work/gkrzmanc/singularity_cache
+nvidia-smi
+
+srun singularity exec  -B /work -B /pnfs -B /t3home -H /t3home/gkrzmanc --nv docker://gkrz/lgatr:v3 python -m src.train -train Delphes_020425_train3_PU_PFfix_part0/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak Delphes_020425_train3_PU_PFfix_part1/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak Delphes_020425_train3_PU_PFfix_part2/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak Delphes_020425_train3_PU_PFfix_part3/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak Delphes_020425_train3_PU_PFfix_part4/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak Delphes_020425_train3_PU_PFfix_part5/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak Delphes_020425_train3_PU_PFfix_part6/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak  Delphes_020425_train3_PU_PFfix_part7/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak  Delphes_020425_train3_PU_PFfix_part8/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak  -val Delphes_020425_train3_PU_PFfix_part9/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak -net src/models/LGATr/lgatr.py -bs 20  --gpus 0 --run-name LGATr_training_NoPID_Delphes_PU_PFfix_700_07_$1_$2_$3_$4 --val-dataset-size 1000 --num-steps 200000 --attr-loss-weight 0.1 --coord-loss-weight 0.1 --beta-type pt+bc --gt-radius $4 --num-blocks $1 -mv-ch $2 -s-ch $3 --spatial-part-only --validation-steps 2000 --no-pid
+
+exit 0
+EOT
+
+# Args: n_blocks mv_channels s_channels radius (default: 10, 16, 64, 0.8)
+# bash jobs/base_training_different_datasets/lgatr_700_07.sh 10 16 64 0.8
+

jobs/base_training_different_datasets/lgatr_700_07_and_900_03.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+sbatch <<EOT
+#!/bin/bash
+#SBATCH --partition=gpu               # Specify the partition
+#SBATCH --mem=25000                   # Request 10GB of memory
+#SBATCH --time=48:00:00
+#SBATCH --job-name=DelphesSVJTrain  # Name the job
+#SBATCH --gres=gpu:1
+#SBATCH --account=gpu_gres             # Specify the account
+#SBATCH --output=jobs/vega/DTrLGATr_out_$1_$2_$3_R$4.log
+#SBATCH --error=jobs/vega/DTrLGATr_err_$1_$2_$3_R$4.log
+
+
+source env.sh
+export APPTAINER_TMPDIR=/work/gkrzmanc/singularity_tmp
+export APPTAINER_CACHEDIR=/work/gkrzmanc/singularity_cache
+nvidia-smi
+
+srun singularity exec  -B /work -B /pnfs -B /t3home -H /t3home/gkrzmanc --nv docker://gkrz/lgatr:v3 python -m src.train -train Delphes_020425_train3_PU_PFfix_part0/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak Delphes_020425_train3_PU_PFfix_part1/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak Delphes_020425_train3_PU_PFfix_part2/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak Delphes_020425_train3_PU_PFfix_part3/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak Delphes_020425_train3_PU_PFfix_part4/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak Delphes_020425_train3_PU_PFfix_part5/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak Delphes_020425_train3_PU_PFfix_part6/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak  Delphes_020425_train3_PU_PFfix_part7/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak  Delphes_020425_train3_PU_PFfix_part8/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak  Delphes_020425_train2_PU_PFfix_part0/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part1/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part2/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part3/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part4/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part5/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part6/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak  Delphes_020425_train2_PU_PFfix_part7/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak  Delphes_020425_train2_PU_PFfix_part8/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak  -val Delphes_020425_train3_PU_PFfix_part9/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak -net src/models/LGATr/lgatr.py -bs 20  --gpus 0 --run-name LGATr_training_NoPID_Delphes_PU_PFfix_700_07_AND_900_03_$1_$2_$3_$4 --val-dataset-size 1000 --num-steps 200000 --attr-loss-weight 0.1 --coord-loss-weight 0.1 --beta-type pt+bc --gt-radius $4 --num-blocks $1 -mv-ch $2 -s-ch $3 --spatial-part-only --validation-steps 2000 --no-pid
+
+exit 0
+EOT
+
+# Args: n_blocks mv_channels s_channels radius (default: 10, 16, 64, 0.8)
+# bash jobs/base_training_different_datasets/lgatr_700_07_and_900_03.sh 10 16 64 0.8

jobs/base_training_different_datasets/lgatr_700_07_and_900_03_and_QCD.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+sbatch <<EOT
+#!/bin/bash
+#SBATCH --partition=gpu               # Specify the partition
+#SBATCH --mem=25000                   # Request 10GB of memory
+#SBATCH --time=48:00:00
+#SBATCH --job-name=DelphesSVJTrain  # Name the job
+#SBATCH --gres=gpu:1
+#SBATCH --account=gpu_gres             # Specify the account
+#SBATCH --output=jobs/vega/DTrLGATr_out_$1_$2_$3_R$4.log
+#SBATCH --error=jobs/vega/DTrLGATr_err_$1_$2_$3_R$4.log
+
+
+source env.sh
+export APPTAINER_TMPDIR=/work/gkrzmanc/singularity_tmp
+export APPTAINER_CACHEDIR=/work/gkrzmanc/singularity_cache
+nvidia-smi
+
+srun singularity exec  -B /work -B /pnfs -B /t3home -H /t3home/gkrzmanc --nv docker://gkrz/lgatr:v3 python -m src.train -train Delphes_020425_train3_PU_PFfix_part0/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak Delphes_020425_train3_PU_PFfix_part1/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak Delphes_020425_train3_PU_PFfix_part2/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak Delphes_020425_train3_PU_PFfix_part3/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak Delphes_020425_train3_PU_PFfix_part4/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak Delphes_020425_train3_PU_PFfix_part5/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak Delphes_020425_train3_PU_PFfix_part6/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak  Delphes_020425_train3_PU_PFfix_part7/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak  Delphes_020425_train3_PU_PFfix_part8/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak  Delphes_020425_train2_PU_PFfix_part0/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part1/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part2/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part3/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part4/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part5/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak Delphes_020425_train2_PU_PFfix_part6/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak  Delphes_020425_train2_PU_PFfix_part7/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak  Delphes_020425_train2_PU_PFfix_part8/SVJ_mZprime-900_mDark-20_rinv-0.3_alpha-peak  QCDtrain_part0/qcd_test_0 QCDtrain_part0/qcd_test_1 QCDtrain_part0/qcd_test_2 QCDtrain_part0/qcd_test_3 QCDtrain_part0/qcd_test_4 QCDtrain_part0/qcd_test_5 QCDtrain_part0/qcd_test_6 QCDtrain_part0/qcd_test_7 QCDtrain_part0/qcd_test_8  -val Delphes_020425_train3_PU_PFfix_part9/SVJ_mZprime-700_mDark-20_rinv-0.7_alpha-peak -net src/models/LGATr/lgatr.py -bs 20  --gpus 0 --run-name LGATr_training_NoPID_Delphes_PU_PFfix_700_07_AND_900_03_AND_QCD_$1_$2_$3_$4 --val-dataset-size 1000 --num-steps 200000 --attr-loss-weight 0.1 --coord-loss-weight 0.1 --beta-type pt+bc --gt-radius $4 --num-blocks $1 -mv-ch $2 -s-ch $3 --spatial-part-only --validation-steps 2000 --no-pid
+
+exit 0
+EOT
+
+# Args: n_blocks mv_channels s_channels radius (default: 10, 16, 64, 0.8)
+# bash jobs/base_training_different_datasets/lgatr_700_07_and_900_03_and_QCD.sh 10 16 64 0.8

jobs/base_training_different_datasets/lgatr_QCD.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+sbatch <<EOT
+#!/bin/bash
+#SBATCH --partition=gpu               # Specify the partition
+#SBATCH --mem=25000                   # Request 10GB of memory
+#SBATCH --time=48:00:00
+#SBATCH --job-name=DelphesSVJTrain  # Name the job
+#SBATCH --gres=gpu:1
+#SBATCH --account=gpu_gres             # Specify the account
+#SBATCH --output=jobs/vega/DTrLGATr_out_$1_$2_$3_R$4.log
+#SBATCH --error=jobs/vega/DTrLGATr_err_$1_$2_$3_R$4.log
+
+
+source env.sh
+export APPTAINER_TMPDIR=/work/gkrzmanc/singularity_tmp
+export APPTAINER_CACHEDIR=/work/gkrzmanc/singularity_cache
+nvidia-smi
+
+srun singularity exec  -B /work -B /pnfs -B /t3home -H /t3home/gkrzmanc --nv docker://gkrz/lgatr:v3 python -m src.train -train QCDtrain_part0/qcd_test_0 QCDtrain_part0/qcd_test_1 QCDtrain_part0/qcd_test_2 QCDtrain_part0/qcd_test_3 QCDtrain_part0/qcd_test_4 QCDtrain_part0/qcd_test_5 QCDtrain_part0/qcd_test_6 QCDtrain_part0/qcd_test_7 QCDtrain_part0/qcd_test_8  -val QCDtrain_part0/qcd_test_9 -net src/models/LGATr/lgatr.py -bs 20  --gpus 0 --run-name LGATr_training_NoPID_Delphes_PU_PFfix_QCD_events_$1_$2_$3_$4 --val-dataset-size 1000 --num-steps 200000 --attr-loss-weight 0.1 --coord-loss-weight 0.1 --beta-type pt+bc --gt-radius $4 --num-blocks $1 -mv-ch $2 -s-ch $3 --spatial-part-only --validation-steps 2000 --no-pid
+
+exit 0
+EOT
+
+# Args: n_blocks mv_channels s_channels radius (default: 10, 16, 64, 0.8)
+# bash jobs/base_training_different_datasets/lgatr_QCD.sh 10 16 64 0.8

jobs/clustering.slurm

@@ -0,0 +1,52 @@
+#!/bin/bash
+#SBATCH --partition=standard           # Specify the partition
+#SBATCH --account=t3                  # Specify the account
+#SBATCH --mem=10000                   # Request 10GB of memory
+#SBATCH --time=05:00:00               # Set the time limit to 1 hour
+#SBATCH --job-name=SVJ_clustering  # Name the job
+#SBATCH --output=jobs/clustering_out.log       # Redirect stdout to a log file
+#SBATCH --error=jobs/clustering_err.log         # Redirect stderr to a log file
+source env.sh
+export APPTAINER_TMPDIR=/work/gkrzmanc/singularity_tmp
+export APPTAINER_CACHEDIR=/work/gkrzmanc/singularity_cache
+#singularity exec -B /t3home/gkrzmanc/ -B /work/gkrzmanc/ --nv docker://gkrz/lgatr:v3 python -m scripts.compute_clustering --input train/Test_betaPt_BC_all_datasets_2025_01_07_17_50_45 --output-suffix 1010 --min-cluster-size 10 --min-samples 10
+
+# Eval of the model trained on m=900 rinv=0.7
+#singularity exec -B /t3home/gkrzmanc/ -B /work/gkrzmanc/ --nv docker://gkrz/lgatr:v3 python -m scripts.compute_clustering --input train/Test_betaPt_BC_all_datasets_2025_01_08_10_54_58 --output-suffix 1010 --min-cluster-size 10 --min-samples 10
+
+# Clustering eval on the L-GATr
+#singularity exec -B /t3home/gkrzmanc/ -B /work/gkrzmanc/ --nv docker://gkrz/lgatr:v3 python -m scripts.compute_clustering --input train/Test_LGATr_all_datasets_2025_01_08_19_27_54
+
+
+# Finetuned parameters for L-GATr_m_900_rinv_0.3
+#srun singularity exec -B /t3home/gkrzmanc/ -B /work/gkrzmanc/ --nv docker://gkrz/lgatr:v3 python -m scripts.compute_clustering --input train/Eval_LGATr_SB_spatial_part_only_1_step_13k_2025_01_13_17_28_03 --output-suffix FT --min-cluster-size 11 --min-samples 18 --epsilon 0.48
+
+# Finetuned parameters for Identity
+#srun singularity exec -B /t3home/gkrzmanc/ -B /work/gkrzmanc/ --nv docker://gkrz/lgatr:v3 python -m scripts.compute_clustering --input train/Eval_Identity_2025_01_13_17_08_48 --output-suffix FT --min-cluster-size 9 --min-samples 2 --epsilon 0.051
+
+
+# Finetuned parameters for L-GATr_m_900_rinv_0.3 ran on the model at 40k+ steps (not 13k) Eval_LGATr_SB_spatial_part_only_1_2025_01_13_14_31_58
+#srun singularity exec -B /t3home/gkrzmanc/ -B /work/gkrzmanc/ --nv docker://gkrz/lgatr:v3 python -m scripts.compute_clustering --input train/Eval_LGATr_SB_spatial_part_only_1_2025_01_13_14_31_58 --output-suffix FT --min-cluster-size 11 --min-samples 18 --epsilon 0.48
+
+#srun singularity exec -B /t3home/gkrzmanc/ -B /work/gkrzmanc/ --nv docker://gkrz/lgatr:v3 python -m scripts.compute_clustering --input train/Eval_GT_R_lgatr_R14_2025_01_18_13_28_47 --output-suffix SP1 --min-cluster-size 11 --min-samples 18 --epsilon 0.48 --spatial-part-only
+
+# Run
+#srun singularity exec -B /t3home/gkrzmanc/ -B /work/gkrzmanc/ --nv docker://gkrz/lgatr:v3 python -m scripts.compute_clustering --input train/Eval_Quark_dist_loss_2025_01_18_13_11_16 --output-suffix FT1 --min-cluster-size 8 --min-samples 3 --epsilon 0.09 --spatial-part-only
+
+
+# No coords loss
+# {'min_cluster_size': 5, 'min_samples': 19, 'epsilon': 0.17149658495077644}
+#srun singularity exec -B /t3home/gkrzmanc/ -B /work/gkrzmanc/ --nv docker://gkrz/lgatr:v3 python -m scripts.compute_clustering --input train/Eval_eval_19March2025_pt1e-2_500particles_NoQMinReprod_2025_04_04_11_13_28_127 --output-suffix FT1 --min-cluster-size 2 --min-samples 1 --epsilon 0.3 --spatial-part-only
+#srun singularity exec -B /t3home/gkrzmanc/ -B /work/gkrzmanc/ --nv docker://gkrz/lgatr:v3 python -m scripts.compute_clustering --input train/Eval_eval_19March2025_pt1e-2_500particles_NoQMinReprod_2025_04_04_11_13_23_90 --output-suffix FT1 --min-cluster-size 2 --min-samples 1 --epsilon 0.3 --spatial-part-only
+#srun singularity exec -B /t3home/gkrzmanc/ -B /work/gkrzmanc/ --nv docker://gkrz/lgatr:v3 python -m scripts.compute_clustering --input train/Eval_eval_19March2025_pt1e-2_500particles_NoQMinReprod_2025_04_04_11_13_20_253 --output-suffix FT1 --min-cluster-size 2 --min-samples 1 --epsilon 0.3 --spatial-part-only
+#srun singularity exec -B /t3home/gkrzmanc/ -B /work/gkrzmanc/ --nv docker://gkrz/lgatr:v3 python -m scripts.compute_clustering --input train/Eval_eval_19March2025_pt1e-2_500particles_NoQMinReprod_2025_04_04_11_13_18_71 --output-suffix FT1 --min-cluster-size 2 --min-samples 1 --epsilon 0.3 --spatial-part-only
+#srun singularity exec -B /t3home/gkrzmanc/ -B /work/gkrzmanc/ --nv docker://gkrz/lgatr:v3 python -m scripts.compute_clustering --input train/Eval_eval_19March2025_pt1e-2_500particles_NoQMinReprod_2025_04_04_10_32_06_532 --output-suffix FT1 --min-cluster-size 2 --min-samples 1 --epsilon 0.3 --spatial-part-only
+#srun singularity exec -B /t3home/gkrzmanc/ -B /work/gkrzmanc/ --nv docker://gkrz/lgatr:v3 python -m scripts.compute_clustering --input train/Eval_eval_19March2025_pt1e-2_500particles_NoQMinReprod_2025_04_04_10_32_03_809 --output-suffix FT1 --min-cluster-size 2 --min-samples 1 --epsilon 0.3 --spatial-part-only
+#srun singularity exec -B /t3home/gkrzmanc/ -B /work/gkrzmanc/ --nv docker://gkrz/lgatr:v3 python -m scripts.compute_clustering --input train/Eval_eval_19March2025_pt1e-2_500particles_NoQMinReprod_2025_04_04_10_32_00_500 --output-suffix FT1 --min-cluster-size 2 --min-samples 1 --epsilon 0.3 --spatial-part-only
+#srun singularity exec -B /t3home/gkrzmanc/ -B /work/gkrzmanc/ --nv docker://gkrz/lgatr:v3 python -m scripts.compute_clustering --input train/Eval_eval_19March2025_pt1e-2_500particles_NoQMinReprod_2025_04_04_10_31_57_566 --output-suffix FT1 --min-cluster-size 2 --min-samples 1 --epsilon 0.3 --spatial-part-only
+#srun singularity exec -B /t3home/gkrzmanc/ -B /work/gkrzmanc/ --nv docker://gkrz/lgatr:v3 python -m scripts.compute_clustering --input train/Eval_eval_19March2025_pt1e-2_500particles_NoQMinReprod_2025_04_04_10_31_53_462 --output-suffix FT1 --min-cluster-size 2 --min-samples 1 --epsilon 0.3 --spatial-part-only
+#srun singularity exec -B /t3home/gkrzmanc/ -B /work/gkrzmanc/ --nv docker://gkrz/lgatr:v3 python -m scripts.compute_clustering --input train/Eval_eval_19March2025_pt1e-2_500particles_NoQMinReprod_2025_04_04_10_31_51_450 --output-suffix FT1 --min-cluster-size 2 --min-samples 1 --epsilon 0.3 --spatial-part-only
+
+# Run the job:
+
+# sbatch jobs/clustering.slurm

jobs/clustering_503.slurm

@@ -0,0 +1,16 @@
+#!/bin/bash
+#SBATCH --partition=short           # Specify the partition
+#SBATCH --account=t3                  # Specify the account
+#SBATCH --mem=10000                   # Request 10GB of memory
+#SBATCH --time=01:00:00               # Set the time limit to 1 hour
+#SBATCH --job-name=SVJ_clustering  # Name the job
+#SBATCH --output=jobs/clustering_out.log       # Redirect stdout to a log file
+#SBATCH --error=jobs/clustering_err.log         # Redirect stderr to a log file
+source env.sh
+export APPTAINER_TMPDIR=/work/gkrzmanc/singularity_tmp
+export APPTAINER_CACHEDIR=/work/gkrzmanc/singularity_cache
+
+srun singularity exec -B /t3home/gkrzmanc/ -B /work/gkrzmanc/ --nv docker://gkrz/lgatr:v3 python -m scripts.plot_multiple_models_clustering
+
+# Run the job:
+# sbatch jobs/clustering_503.slurm