Upload weights and training details

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+complete_no_valid_oversample/train_confusion_matrix.png filter=lfs diff=lfs merge=lfs -text

complete_no_valid_oversample/EpiLaP/91214ed0b1664395b1826dc69a495ed4/checkpoints/epoch=299-step=300.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3d4da59e31a4bb7857696f6f4c004fdbb6dfdd3ba8ad8477fa13f80696007cef
+size 1091779902

complete_no_valid_oversample/best_checkpoint.list

@@ -0,0 +1 @@
+/lustre07/scratch/rabyj/recount3/epiclass_models/hg38_100kb_all_none/harmonized_donor_life_stage_1l_3000n/no-unknown/complete_no_valid_oversample/EpiLaP/91214ed0b1664395b1826dc69a495ed4/checkpoints/epoch=299-step=300.ckpt

complete_no_valid_oversample/launch_script_NN-dfreezev2.1-harmonized_donor_life_stage-complete_no_valid-oversample-100kb_all_none-job21525239.sh

@@ -0,0 +1,201 @@
+#!/bin/bash
+#SBATCH --time=6:00:00
+#SBATCH --account=def-jacquesp
+#SBATCH --job-name=NN-dfreezev2.1-harmonized_donor_life_stage-complete_no_valid-oversample-100kb_all_none
+#SBATCH --output=/lustre06/project/6007017/rabyj/epilap/output/sub/slurm_files/%x-job%j.out
+#SBATCH --nodes=1
+#SBATCH --gres=gpu:1
+#SBATCH --mem=64G
+#SBATCH --mail-user=joanny.raby@usherbrooke.ca
+#SBATCH --mail-type=END,FAIL
+
+# shellcheck disable=SC1091  # Don't warn about sourcing unreachable files
+
+export PYTHONUNBUFFERED=TRUE
+
+if [[ -n "$SLURM_JOB_ID" ]];
+then
+  echo "print =========================================="
+  echo "print SLURM_JOB_ID = $SLURM_JOB_ID"
+  echo "print SLURM_JOB_NODELIST = $SLURM_JOB_NODELIST"
+  echo "print =========================================="
+fi
+
+gen_path="/lustre06/project/6007017/rabyj"
+input_path="${gen_path}/epilap/input"
+output_path="${gen_path}/epilap/output/logs"
+gen_program_path="${gen_path}/sources/epi_ml"
+program_path="${gen_program_path}/src/python/epi_ml"
+
+slurm_out_folder="${gen_path}/epilap/output/sub/slurm_files"
+
+# --- use correct environment ---
+
+set -e
+if [[ -n "$SLURM_JOB_ID" ]];
+then
+  cd $SLURM_TMPDIR
+  bash ${gen_program_path}/src/bash_utils/setup_venv.sh -r ${gen_program_path}/requirements/minimal_requirements.txt -s ${gen_program_path}/src/python &> ${slurm_out_folder}/${SLURM_JOB_ID}_setup.log
+  source epiclass_env/bin/activate
+else
+  source /lustre07/scratch/rabyj/envs/epiclass/bin/activate
+fi
+
+
+# --- choose category + hparams + source files ---
+
+# MODIFY THINGS HERE
+
+# RESTORE="--restore" # COMMENT IF TRAINING # IMPORTANT
+NO_VALID="hell yeah" # COMMENT IF 10fold  TRAINING # IMPORTANT
+
+export MAX_SPLIT="69" # IMPORTANT
+category="harmonized_donor_life_stage" # IMPORTANT
+
+if [[ "$category" == "assay_epiclass" ]]; then
+  export ASSAY_LIST='["h3k27ac", "h3k27me3", "h3k36me3", "h3k4me1", "h3k4me3", "h3k9me3", "input", "rna_seq", "mrna_seq", "wgbs-standard", "wgbs-pbat"]' # as json
+elif [[ "$category" == "harmonized_donor_sex" ]]; then
+  export LABEL_LIST='["female", "male"]'
+fi
+
+export EXCLUDE_LIST='["other", "--", "NA", "", "unknown"]'
+export MIN_CLASS_SIZE="10" # IMPORTANT
+
+hparams="human_no_valid_oversample" # IMPORTANT
+
+release="epiatlas-dfreeze-v2.1"
+assembly="hg38"
+resolution="100kb" # IMPORTANT
+
+basename="${resolution}_all_none" # IMPORTANT
+list_name="${basename}_dfreeze_filterCtl_plus_4ctl" # IMPORTANT
+
+dataset=${assembly}"_"${release} # ex: hg38_2018-10
+
+echo $dataset
+
+export LAYER_SIZE="3000" # IMPORTANT
+export NB_LAYER="1"
+
+log="${output_path}/${release}/${assembly}_${basename}/${category}_${NB_LAYER}l_${LAYER_SIZE}n" # IMPORTANT# IMPORTANT# IMPORTANT# IMPORTANT
+log="${log}/no-unknown/complete_no_valid_oversample" # IMPORTANT
+
+
+# --- Creating correct paths for programs/launching ---
+
+timestamp=$(date +%s)
+
+hparams="${input_path}/hparams/${hparams}.json"
+hdf5_list="${input_path}/hdf5_list/hg38_epiatlas-freeze-v2/${list_name}.list"
+chroms="${input_path}/chromsizes/hg38.noy.chrom.sizes"
+metadata="${input_path}/metadata/dfreeze-v2/hg38_2023-epiatlas-dfreeze-pospurge-nodup_filterCtl.json"
+out1="${log}/output_job${SLURM_JOB_ID}_${SLURM_JOB_NAME}_${timestamp}.o"
+out2="${log}/output_job${SLURM_JOB_ID}_${SLURM_JOB_NAME}_${timestamp}.e"
+
+set -e
+echo "Input arguments:"
+for var in $hparams $hdf5_list $chroms $metadata
+do
+ls $var
+done
+
+
+# --- Pre-checks ---
+
+cd ${program_path}
+
+printf '\n%s\n' "Launching following command"
+printf '%s\n' "python ${program_path}/utils/check_dir.py ${log}"
+python ${program_path}/utils/check_dir.py ${log}
+
+printf '\n%s\n' "Launching following command"
+printf '%s\n' "python ${program_path}/utils/preconditions.py -m ${metadata}"
+python ${program_path}/utils/preconditions.py -m ${metadata}
+
+# Preconditions passed, copy launch script to log dir.
+if [[ -n "$SLURM_JOB_ID" ]];
+then
+scontrol write batch_script ${SLURM_JOB_ID} ${log}/launch_script_${SLURM_JOB_NAME}-job${SLURM_JOB_ID}.sh
+fi
+
+
+# --- Transfer files to node scratch ---
+
+if [[ -n "$SLURM_JOB_ID" ]];
+then
+  hdf5s_location="/lustre06/project/6007515/ihec_share/local_ihec_data/epiatlas/hg38/hdf5"
+  name="epiatlas_dfreeze_${resolution}_all_none"
+  tar_file="${hdf5s_location}/${name}.tar"  # IMPORTANT
+
+  cd $SLURM_TMPDIR
+
+  echo "Untaring $tar_file in $SLURM_TMPDIR"
+  tar -xf $tar_file
+
+  export HDF5_PARENT="${name}" # IMPORTANT
+  cd $name
+  scp ${hdf5s_location}/${name}-4ctl/* . #extra files in v2.1
+fi
+
+
+# --- MAIN PROGRAM ---
+
+echo "Time before launch: $(date +%F_%T)"
+printf '\n%s\n' "Launching following command"
+if [[ -n "$NO_VALID" ]]; #if variable exists
+then
+  # --- no valid launch ---
+  if [[ "$log" == *"10fold"* ]]; then
+    log="$log/notactually10foldbaka"
+    printf '\n%s\n' "Incoherent log path, changing log to $log"
+  fi
+
+  printf '%s\n' "python ${program_path}/epiatlas_training_no_valid.py $category ${hparams} ${hdf5_list} ${chroms} ${metadata} ${log} > ${out1} 2> ${out2}"
+  python ${program_path}/epiatlas_training_no_valid.py $category ${hparams} ${hdf5_list} ${chroms} ${metadata} ${log} > "${out1}" 2> "${out2}"
+  echo "Time after launch: $(date +%F_%T)"
+  exit
+
+elif [[ -n "$RESTORE" ]]; then
+  # --- kfold launch ---
+  printf '%s\n' "python ${program_path}/epiatlas_training.py $category ${hparams} ${hdf5_list} ${chroms} ${metadata} ${log} --restore > ${out1} 2> ${out2}"
+  python ${program_path}/epiatlas_training.py $category ${hparams} ${hdf5_list} ${chroms} ${metadata} ${log} --restore > "${out1}" 2> "${out2}"
+  exit
+
+else
+  # --- kfold launch ---
+  printf '%s\n' "python ${program_path}/epiatlas_training.py $category ${hparams} ${hdf5_list} ${chroms} ${metadata} ${log} > ${out1} 2> ${out2}"
+  python ${program_path}/epiatlas_training.py $category ${hparams} ${hdf5_list} ${chroms} ${metadata} ${log} > "${out1}" 2> "${out2}"
+fi
+echo "Time after launch: $(date +%F_%T)"
+
+
+
+# --- More logging ---
+set +e
+
+if [[ -z "$NO_VALID" ]];  #if variable is empty or unset
+then
+  export LOG="${log}"
+  export NO_TRUE="False"
+
+  cd ${log}
+  printf '\n%s\n' "Launching following command"
+  printf '%s\n' "cat split*/validation_prediction.csv | sort -ru > full-10fold-validation_prediction.csv"
+  cat split*/validation_prediction.csv | sort -ru > full-10fold-validation_prediction.csv
+
+  to_augment="${log}/full-10fold-validation_prediction.csv"
+
+  printf '\n%s\n' "Launching following command"
+  printf '%s\n' "python ${program_path}/utils/augment_predict_file.py ${to_augment} ${metadata} --all-categories"
+  python ${program_path}/utils/augment_predict_file.py ${to_augment} ${metadata} --all-categories
+
+  printf '%s\n' "python ${program_path}/utils/create_confusion_matrices.py --from_prediction ${to_augment}"
+  python ${program_path}/utils/create_confusion_matrices.py --from_prediction ${to_augment}
+fi
+
+# Copy slurm output file to log dir
+if [[ -n "$SLURM_JOB_ID" ]];
+then
+  slurm_out_file="${SLURM_JOB_NAME}-*${SLURM_JOB_ID}.out"
+  cp -v ${slurm_out_folder}/${slurm_out_file} ${log}/
+fi

complete_no_valid_oversample/output_job21525239_NN-dfreezev2.1-harmonized_donor_life_stage-complete_no_valid-oversample-100kb_all_none_1695761998.e

@@ -0,0 +1,95 @@
+CometLogger will be initialized in online mode
+COMET INFO: Experiment is live on comet.ml https://www.comet.com/rabyj/epilap/91214ed0b1664395b1826dc69a495ed4
+
+Using 16bit native Automatic Mixed Precision (AMP)
+GPU available: True, used: True
+TPU available: False, using: 0 TPU cores
+IPU available: False, using: 0 IPUs
+HPU available: False, using: 0 HPUs
+LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]
+COMET INFO: ---------------------------
+COMET INFO: Comet.ml Experiment Summary
+COMET INFO: ---------------------------
+COMET INFO:   Data:
+COMET INFO:     display_summary_level : 1
+COMET INFO:     url                   : https://www.comet.com/rabyj/epilap/91214ed0b1664395b1826dc69a495ed4
+COMET INFO:   Metrics [count] (min, max):
+COMET INFO:     train_acc [300]  : (0.783570408821106, 0.9960975050926208)
+COMET INFO:     train_loss [300] : (0.0264463908970356, 0.6765714883804321)
+COMET INFO:   Others:
+COMET INFO:     Code version / commit     : v0.7.5.1-7-ga6afe82
+COMET INFO:     Experience key            : 91214ed0b1664395b1826dc69a495ed4
+COMET INFO:     HDF5 Resolution           : 100.0kb
+COMET INFO:     Initial hdf5 loading time : 0:00:49
+COMET INFO:     Name                      : harmonized_donor_life_stage_1l_3000n-no-unknown-complete_no_valid_oversample
+COMET INFO:     SLURM_JOB_ID              : 21525239
+COMET INFO:     Total nb of files         : 15372
+COMET INFO:     category                  : harmonized_donor_life_stage
+COMET INFO:     test size                 : 0
+COMET INFO:     train size                : 66372
+COMET INFO:     validation size           : 0
+COMET INFO:   Parameters:
+COMET INFO:     hl_units                  : 3000
+COMET INFO:     hparams/batch_size        : 64
+COMET INFO:     hparams/is_training       : True
+COMET INFO:     hparams/keep_prob         : 0.5
+COMET INFO:     hparams/l2_scale          : 0.01
+COMET INFO:     hparams/learning_rate     : 1e-06
+COMET INFO:     hparams/measure_frequency : 1
+COMET INFO:     hparams/oversampling      : True
+COMET INFO:     hparams/training_epochs   : 300
+COMET INFO:     input_size                : 30321
+COMET INFO:     mapping/0                 : adult
+COMET INFO:     mapping/1                 : child
+COMET INFO:     mapping/2                 : embryonic
+COMET INFO:     mapping/3                 : fetal
+COMET INFO:     mapping/4                 : newborn
+COMET INFO:     nb_layer                  : 1
+COMET INFO:     output_size               : 5
+COMET INFO:   Uploads:
+COMET INFO:     asset               : 1 (46 bytes)
+COMET INFO:     environment details : 1
+COMET INFO:     installed packages  : 1
+COMET INFO:     model graph         : 1
+COMET INFO: ---------------------------
+COMET INFO: Uploading metrics, params, and assets to Comet before program termination (may take several seconds)
+COMET INFO: The Python SDK has 3600 seconds to finish before aborting...
+COMET INFO: Uploading 1 metrics, params and output messages
+COMET INFO: Experiment is live on comet.ml https://www.comet.com/rabyj/epilap/91214ed0b1664395b1826dc69a495ed4
+
+CometLogger will be initialized in online mode
+COMET INFO: -----------------------------------
+COMET INFO: Comet.ml ExistingExperiment Summary
+COMET INFO: -----------------------------------
+COMET INFO:   Data:
+COMET INFO:     display_summary_level : 1
+COMET INFO:     url                   : https://www.comet.com/rabyj/epilap/91214ed0b1664395b1826dc69a495ed4
+COMET INFO:   Others:
+COMET INFO:     Name : harmonized_donor_life_stage_1l_3000n-no-unknown-complete_no_valid_oversample
+COMET INFO:   Uploads:
+COMET INFO:     installed packages : 1
+COMET INFO: -----------------------------------
+COMET INFO: Experiment is live on comet.ml https://www.comet.com/rabyj/epilap/91214ed0b1664395b1826dc69a495ed4
+
+COMET INFO: -----------------------------------
+COMET INFO: Comet.ml ExistingExperiment Summary
+COMET INFO: -----------------------------------
+COMET INFO:   Data:
+COMET INFO:     display_summary_level : 1
+COMET INFO:     url                   : https://www.comet.com/rabyj/epilap/91214ed0b1664395b1826dc69a495ed4
+COMET INFO:   Metrics:
+COMET INFO:     Last epoch           : 300
+COMET INFO:     Training time        : 1:13:13
+COMET INFO:     tra_Accuracy         : 0.9980714917182922
+COMET INFO:     tra_F1Score          : 0.9980384111404419
+COMET INFO:     tra_MatthewsCorrCoef : 0.9975903630256653
+COMET INFO:     tra_Precision        : 0.9980669021606445
+COMET INFO:     tra_Recall           : 0.9980159997940063
+COMET INFO:   Uploads:
+COMET INFO:     asset              : 3 (289.66 KB)
+COMET INFO:     installed packages : 1
+COMET INFO: -----------------------------------
+COMET INFO: Uploading 1 metrics, params and output messages
+COMET INFO: Waiting for completion of the file uploads (may take several seconds)
+COMET INFO: The Python SDK has 10800 seconds to finish before aborting...
+COMET INFO: Still uploading 3 file(s), remaining 290.07 KB/290.07 KB

complete_no_valid_oversample/output_job21525239_NN-dfreezev2.1-harmonized_donor_life_stage-complete_no_valid-oversample-100kb_all_none_1695761998.o

@@ -0,0 +1,110 @@
+begin 2023-09-26 21:00:06
+Checking environment variables.
+EXCLUDE_LIST: ['other', '--', 'NA', '', 'unknown']
+Filtering metadata: Removing labels ['other', '--', 'NA', '', 'unknown'] from category 'harmonized_donor_life_stage'.
+No label list, considering all left classes : ['adult', 'child', 'embryonic', 'fetal', 'newborn']
+5/5 labels left from harmonized_donor_life_stage after removing classes with less than 10 signals.
+harmonized_donor_life_stage label breakdown for unique experiments (uuid):
+adult: 5001
+child: 623
+embryonic: 237
+newborn: 106
+fetal: 44
+For 6011 unique experiments in 5 classes
+
+Using files in /localscratch/rabyj.21525239.0/epiatlas_dfreeze_100kb_all_none
+Initial hdf5 loading time: 0:00:49
+The current experiment key is 91214ed0b1664395b1826dc69a495ed4
+The current commit is v0.7.5.1-7-ga6afe82
+Split 0 train size: 66372
+Split 0 validation size: 0
+Split 0 test size: 0
+Total nb of files: 15372
+--MODEL STRUCTURE--
+ LightningDenseClassifier(
+  (_pt_model): Sequential(
+    (0): Dropout(p=0.1, inplace=False)
+    (1): Linear(in_features=30321, out_features=3000, bias=True)
+    (2): Dropout(p=0.5, inplace=False)
+    (3): ReLU()
+    (4): Linear(in_features=3000, out_features=5, bias=True)
+  )
+  (metrics): MetricCollection(
+    (Accuracy): Accuracy()
+    (Precision): Precision()
+    (Recall): Recall()
+    (F1Score): F1Score()
+    (MatthewsCorrCoef): MatthewsCorrCoef()
+  )
+  (train_acc): Accuracy()
+  (valid_acc): Accuracy()
+)
+--MODEL SUMMARY--
+===================================================================================================================
+Layer (type:depth-idx)                   Input Shape               Output Shape              Param #
+===================================================================================================================
+LightningDenseClassifier                 --                        --                        --
+├─Sequential: 1-1                        [1, 30321]                [1, 5]                    --
+│    └─Dropout: 2-1                      [1, 30321]                [1, 30321]                --
+│    └─Linear: 2-2                       [1, 30321]                [1, 3000]                 90,966,000
+│    └─Dropout: 2-3                      [1, 3000]                 [1, 3000]                 --
+│    └─ReLU: 2-4                         [1, 3000]                 [1, 3000]                 --
+│    └─Linear: 2-5                       [1, 3000]                 [1, 5]                    15,005
+├─MetricCollection: 1-2                  --                        --                        --
+│    └─Accuracy: 2-6                     --                        --                        --
+│    └─Precision: 2-7                    --                        --                        --
+│    └─Recall: 2-8                       --                        --                        --
+│    └─F1Score: 2-9                      --                        --                        --
+│    └─MatthewsCorrCoef: 2-10            --                        --                        --
+├─Accuracy: 1-3                          --                        --                        --
+├─Accuracy: 1-4                          --                        --                        --
+===================================================================================================================
+Total params: 90,981,005
+Trainable params: 90,981,005
+Non-trainable params: 0
+Total mult-adds (M): 90.98
+===================================================================================================================
+Input size (MB): 0.12
+Forward/backward pass size (MB): 0.02
+Params size (MB): 363.92
+Estimated Total Size (MB): 364.07
+===================================================================================================================
+--TRAINING HYPERPARAMETERS--
+L2 scale : 0.01
+Dropout rate : 0.5
+Learning rate : 1e-06
+No early stopping.
+Training batch size : 64
+┏━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━┳━━━━━━━━┓
+┃    ┃ Name                     ┃ Type             ┃ Params ┃
+┡━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━╇━━━━━━━━┩
+│ 0  │ _pt_model                │ Sequential       │ 91.0 M │
+│ 1  │ _pt_model.0              │ Dropout          │      0 │
+│ 2  │ _pt_model.1              │ Linear           │ 91.0 M │
+│ 3  │ _pt_model.2              │ Dropout          │      0 │
+│ 4  │ _pt_model.3              │ ReLU             │      0 │
+│ 5  │ _pt_model.4              │ Linear           │ 15.0 K │
+│ 6  │ metrics                  │ MetricCollection │      0 │
+│ 7  │ metrics.Accuracy         │ Accuracy         │      0 │
+│ 8  │ metrics.Precision        │ Precision        │      0 │
+│ 9  │ metrics.Recall           │ Recall           │      0 │
+│ 10 │ metrics.F1Score          │ F1Score          │      0 │
+│ 11 │ metrics.MatthewsCorrCoef │ MatthewsCorrCoef │      0 │
+│ 12 │ train_acc                │ Accuracy         │      0 │
+│ 13 │ valid_acc                │ Accuracy         │      0 │
+└────┴──────────────────────────┴──────────────────┴────────┘
+Trainable params: 91.0 M                                                        
+Non-trainable params: 0                                                         
+Total params: 91.0 M                                                            
+Total estimated model params size (MB): 181                                     
+Saving model to /lustre06/project/6007017/rabyj/epilap/output/logs/epiatlas-dfreeze-v2.1/hg38_100kb_all_none/harmonized_donor_life_stage_1l_3000n/no-unknown/complete_no_valid_oversample/EpiLaP/91214ed0b1664395b1826dc69a495ed4/checkpoints/epoch=299-step=300.ckpt
+training time: 1:13:13
+Reading checkpoint list and taking last line.
+Loading model from /lustre06/project/6007017/rabyj/epilap/output/logs/epiatlas-dfreeze-v2.1/hg38_100kb_all_none/harmonized_donor_life_stage_1l_3000n/no-unknown/complete_no_valid_oversample/EpiLaP/91214ed0b1664395b1826dc69a495ed4/checkpoints/epoch=299-step=300.ckpt
+--- training set METRICS ---
+Accuracy 0.998
+Precision 0.998
+Recall 0.998
+F1Score 0.998
+MatthewsCorrCoef 0.998
+0.998 0.998 0.998 0.998 0.998

complete_no_valid_oversample/train_confusion_matrix.csv

@@ -0,0 +1,6 @@
+,adult,child,embryonic,fetal,newborn
+adult(12845),12757,69,4,4,11
+child(13033),4,12993,0,0,36
+embryonic(13807),0,0,13807,0,0
+fetal(13219),0,0,0,13219,0
+newborn(13468),0,0,0,0,13468

complete_no_valid_oversample/train_confusion_matrix_relative.csv

@@ -0,0 +1,6 @@
+,adult,child,embryonic,fetal,newborn
+adult(12845),0.9931,0.0054,0.0003,0.0003,0.0009
+child(13033),0.0003,0.9969,0.0000,0.0000,0.0028
+embryonic(13807),0.0000,0.0000,1.0000,0.0000,0.0000
+fetal(13219),0.0000,0.0000,0.0000,1.0000,0.0000
+newborn(13468),0.0000,0.0000,0.0000,0.0000,1.0000

complete_no_valid_oversample/training_mapping.tsv

@@ -0,0 +1,5 @@
+0	adult
+1	child
+2	embryonic
+3	fetal
+4	newborn