Upload weights and training details

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+complete_no_valid_oversample/train_confusion_matrix.png filter=lfs diff=lfs merge=lfs -text

complete_no_valid_oversample/EpiLaP/15da476b92f140eab818ece369248f4c/checkpoints/epoch=299-step=300.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2dbc36215cd83c3beff66c619c9dc90f0a23db4bcb6e443527e5dbbd351362dc
+size 1091671870

complete_no_valid_oversample/best_checkpoint.list

@@ -0,0 +1 @@
+/lustre07/scratch/rabyj/recount3/epiclass_models/hg38_100kb_all_none/harmonized_sample_cancer_high_1l_3000n/complete_no_valid_oversample/EpiLaP/15da476b92f140eab818ece369248f4c/checkpoints/epoch=299-step=300.ckpt

complete_no_valid_oversample/launch_script_NN-dfreezev2.1-harmonized_sample_cancer_high-w-oversampling-no_valid-100kb_all_none-job21525586.sh

@@ -0,0 +1,202 @@
+#!/bin/bash
+#SBATCH --time=6:00:00
+#SBATCH --account=def-jacquesp
+#SBATCH --job-name=NN-dfreezev2.1-harmonized_sample_cancer_high-w-oversampling-no_valid-100kb_all_none
+#SBATCH --output=/lustre06/project/6007017/rabyj/epilap/output/sub/slurm_files/%x-job%j.out
+#SBATCH --nodes=1
+#SBATCH --gres=gpu:1
+#SBATCH --mem=64G
+#SBATCH --mail-user=joanny.raby@usherbrooke.ca
+#SBATCH --mail-type=END,FAIL
+
+# shellcheck disable=SC1091  # Don't warn about sourcing unreachable files
+
+export PYTHONUNBUFFERED=TRUE
+
+if [[ -n "$SLURM_JOB_ID" ]];
+then
+  echo "print =========================================="
+  echo "print SLURM_JOB_ID = $SLURM_JOB_ID"
+  echo "print SLURM_JOB_NODELIST = $SLURM_JOB_NODELIST"
+  echo "print =========================================="
+fi
+
+gen_path="/lustre06/project/6007017/rabyj"
+input_path="${gen_path}/epilap/input"
+output_path="${gen_path}/epilap/output/logs"
+gen_program_path="${gen_path}/sources/epi_ml"
+program_path="${gen_program_path}/src/python/epi_ml"
+
+slurm_out_folder="${gen_path}/epilap/output/sub/slurm_files"
+
+# --- use correct environment ---
+
+set -e
+if [[ -n "$SLURM_JOB_ID" ]];
+then
+  cd $SLURM_TMPDIR
+  bash ${gen_program_path}/src/bash_utils/setup_venv.sh -r ${gen_program_path}/requirements/minimal_requirements.txt -s ${gen_program_path}/src/python &> ${slurm_out_folder}/${SLURM_JOB_ID}_setup.log
+  source epiclass_env/bin/activate
+else
+  source /lustre07/scratch/rabyj/envs/epiclass/bin/activate
+fi
+
+
+# --- choose category + hparams + source files ---
+
+# MODIFY THINGS HERE
+
+# RESTORE="--restore" # COMMENT IF TRAINING # IMPORTANT
+NO_VALID="hell yeah" # COMMENT IF 10fold  TRAINING # IMPORTANT
+
+export MAX_SPLIT="69" # IMPORTANT
+category="harmonized_sample_cancer_high" # IMPORTANT
+
+if [[ "$category" == "assay_epiclass" ]]; then
+  export ASSAY_LIST='["h3k27ac", "h3k27me3", "h3k36me3", "h3k4me1", "h3k4me3", "h3k9me3", "input", "rna_seq", "mrna_seq", "wgbs-standard", "wgbs-pbat"]' # as json
+elif [[ "$category" == "harmonized_donor_sex" ]]; then
+  export LABEL_LIST='["female", "male"]'
+elif [[ "$category" == "harmonized_sample_disease_high" ]]; then
+  export LABEL_LIST='["Healthy/None", "Cancer"]'
+fi
+
+
+export EXCLUDE_LIST='["other", "--", "NA", "", "unknown"]'
+export MIN_CLASS_SIZE="10" # IMPORTANT
+
+hparams="human_no_valid_oversample" # IMPORTANT
+
+release="epiatlas-dfreeze-v2.1"
+assembly="hg38"
+resolution="100kb" # IMPORTANT
+
+basename="${resolution}_all_none" # IMPORTANT
+list_name="${basename}_dfreeze_filterCtl_plus_4ctl" # IMPORTANT
+
+dataset=${assembly}"_"${release} # ex: hg38_2018-10
+
+echo $dataset
+
+export LAYER_SIZE="3000" # IMPORTANT
+export NB_LAYER="1"
+
+log="${output_path}/${release}/${assembly}_${basename}/${category}_${NB_LAYER}l_${LAYER_SIZE}n" # IMPORTANT# IMPORTANT# IMPORTANT# IMPORTANT
+log="${log}/complete_no_valid_oversample" # IMPORTANT
+
+
+# --- Creating correct paths for programs/launching ---
+
+timestamp=$(date +%s)
+
+hparams="${input_path}/hparams/${hparams}.json"
+hdf5_list="${input_path}/hdf5_list/hg38_epiatlas-freeze-v2/${list_name}.list"
+chroms="${input_path}/chromsizes/hg38.noy.chrom.sizes"
+metadata="${input_path}/metadata/dfreeze-v2/hg38_2023-epiatlas-dfreeze_v2.1_w_encode_noncore_2.json"
+out1="${log}/output_job${SLURM_JOB_ID}_${SLURM_JOB_NAME}_${timestamp}.o"
+out2="${log}/output_job${SLURM_JOB_ID}_${SLURM_JOB_NAME}_${timestamp}.e"
+
+set -e
+echo "Input arguments:"
+for var in $hparams $hdf5_list $chroms $metadata
+do
+ls $var
+done
+
+
+# --- Pre-checks ---
+
+cd ${program_path}
+
+printf '\n%s\n' "Launching following command"
+printf '%s\n' "python ${program_path}/utils/check_dir.py ${log}"
+python ${program_path}/utils/check_dir.py ${log}
+
+printf '\n%s\n' "Launching following command"
+printf '%s\n' "python ${program_path}/utils/preconditions.py -m ${metadata}"
+python ${program_path}/utils/preconditions.py -m ${metadata}
+
+# Preconditions passed, copy launch script to log dir.
+if [[ -n "$SLURM_JOB_ID" ]];
+then
+scontrol write batch_script ${SLURM_JOB_ID} ${log}/launch_script_${SLURM_JOB_NAME}-job${SLURM_JOB_ID}.sh
+fi
+
+
+# --- Transfer files to node scratch ---
+
+if [[ -n "$SLURM_JOB_ID" ]];
+then
+  hdf5s_location="/lustre06/project/6007515/ihec_share/local_ihec_data/epiatlas/hg38/hdf5"
+  name="epiatlas_dfreeze_${resolution}_all_none"
+  tar_file="${hdf5s_location}/${name}.tar"  # IMPORTANT
+
+  cd $SLURM_TMPDIR
+
+  echo "Untaring $tar_file in $SLURM_TMPDIR"
+  tar -xf $tar_file
+
+  export HDF5_PARENT="${name}" # IMPORTANT
+  cd $name
+  scp ${hdf5s_location}/${name}-4ctl/* . #extra files in v2.1
+fi
+
+
+# --- MAIN PROGRAM ---
+
+echo "Time before launch: $(date +%F_%T)"
+printf '\n%s\n' "Launching following command"
+if [[ -n "$NO_VALID" ]]; #if variable exists
+then
+  # --- no valid launch ---
+  if [[ "$log" == *"10fold"* ]]; then
+    log="$log/notactually10foldbaka"
+    printf '\n%s\n' "Incoherent log path, changing log to $log"
+  fi
+
+  printf '%s\n' "python ${program_path}/epiatlas_training_no_valid.py $category ${hparams} ${hdf5_list} ${chroms} ${metadata} ${log} > ${out1} 2> ${out2}"
+  python ${program_path}/epiatlas_training_no_valid.py $category ${hparams} ${hdf5_list} ${chroms} ${metadata} ${log} > "${out1}" 2> "${out2}"
+  echo "Time after launch: $(date +%F_%T)"
+  exit
+
+elif [[ -n "$RESTORE" ]]; then
+  # --- kfold launch ---
+  printf '%s\n' "python ${program_path}/epiatlas_training.py $category ${hparams} ${hdf5_list} ${chroms} ${metadata} ${log} --restore > ${out1} 2> ${out2}"
+  python ${program_path}/epiatlas_training.py $category ${hparams} ${hdf5_list} ${chroms} ${metadata} ${log} --restore > "${out1}" 2> "${out2}"
+  exit
+
+else
+  # --- kfold launch ---
+  printf '%s\n' "python ${program_path}/epiatlas_training.py $category ${hparams} ${hdf5_list} ${chroms} ${metadata} ${log} > ${out1} 2> ${out2}"
+  python ${program_path}/epiatlas_training.py $category ${hparams} ${hdf5_list} ${chroms} ${metadata} ${log} > "${out1}" 2> "${out2}"
+fi
+echo "Time after launch: $(date +%F_%T)"
+
+
+
+# --- More logging ---
+set +e
+
+export LOG="${log}"
+export NO_TRUE="False"
+
+cd ${log}
+printf '\n%s\n' "Launching following command"
+printf '%s\n' "cat split*/validation_prediction.csv | sort -ru > full-10fold-validation_prediction.csv"
+cat split*/validation_prediction.csv | sort -ru > full-10fold-validation_prediction.csv
+
+to_augment="${log}/full-10fold-validation_prediction.csv"
+
+printf '\n%s\n' "Launching following command"
+printf '%s\n' "python ${program_path}/utils/augment_predict_file.py ${to_augment} ${metadata} --all-categories"
+python ${program_path}/utils/augment_predict_file.py ${to_augment} ${metadata} --all-categories
+
+printf '%s\n' "python ${program_path}/utils/create_confusion_matrices.py --from_prediction ${to_augment}"
+python ${program_path}/utils/create_confusion_matrices.py --from_prediction ${to_augment}
+
+
+# Copy slurm output file to log dir
+if [[ -n "$SLURM_JOB_ID" ]];
+then
+  slurm_out_file="${SLURM_JOB_NAME}-*${SLURM_JOB_ID}.out"
+  cp -v ${slurm_out_folder}/${slurm_out_file} ${log}/
+fi

complete_no_valid_oversample/output_job21525586_NN-dfreezev2.1-harmonized_sample_cancer_high-w-oversampling-no_valid-100kb_all_none_1695763876.e

@@ -0,0 +1,89 @@
+CometLogger will be initialized in online mode
+COMET INFO: Experiment is live on comet.ml https://www.comet.com/rabyj/epilap/15da476b92f140eab818ece369248f4c
+
+Using 16bit native Automatic Mixed Precision (AMP)
+GPU available: True, used: True
+TPU available: False, using: 0 TPU cores
+IPU available: False, using: 0 IPUs
+HPU available: False, using: 0 HPUs
+LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]
+COMET INFO: ---------------------------
+COMET INFO: Comet.ml Experiment Summary
+COMET INFO: ---------------------------
+COMET INFO:   Data:
+COMET INFO:     display_summary_level : 1
+COMET INFO:     url                   : https://www.comet.com/rabyj/epilap/15da476b92f140eab818ece369248f4c
+COMET INFO:   Metrics [count] (min, max):
+COMET INFO:     train_acc [300]  : (0.8000696897506714, 0.9963696599006653)
+COMET INFO:     train_loss [300] : (0.020579759031534195, 0.43990352749824524)
+COMET INFO:   Others:
+COMET INFO:     Code version / commit     : v0.7.5.1-7-ga6afe82
+COMET INFO:     Experience key            : 15da476b92f140eab818ece369248f4c
+COMET INFO:     HDF5 Resolution           : 100.0kb
+COMET INFO:     Initial hdf5 loading time : 0:01:11
+COMET INFO:     Name                      : hg38_100kb_all_none-harmonized_sample_cancer_high_1l_3000n-complete_no_valid_oversample
+COMET INFO:     SLURM_JOB_ID              : 21525586
+COMET INFO:     Total nb of files         : 20922
+COMET INFO:     category                  : harmonized_sample_cancer_high
+COMET INFO:     test size                 : 0
+COMET INFO:     train size                : 34491
+COMET INFO:     validation size           : 0
+COMET INFO:   Parameters:
+COMET INFO:     hl_units                  : 3000
+COMET INFO:     hparams/batch_size        : 64
+COMET INFO:     hparams/is_training       : True
+COMET INFO:     hparams/keep_prob         : 0.5
+COMET INFO:     hparams/l2_scale          : 0.01
+COMET INFO:     hparams/learning_rate     : 1e-06
+COMET INFO:     hparams/measure_frequency : 1
+COMET INFO:     hparams/oversampling      : True
+COMET INFO:     hparams/training_epochs   : 300
+COMET INFO:     input_size                : 30321
+COMET INFO:     mapping/0                 : cancer
+COMET INFO:     mapping/1                 : non-cancer
+COMET INFO:     nb_layer                  : 1
+COMET INFO:     output_size               : 2
+COMET INFO:   Uploads:
+COMET INFO:     asset               : 1 (22 bytes)
+COMET INFO:     environment details : 1
+COMET INFO:     installed packages  : 1
+COMET INFO:     model graph         : 1
+COMET INFO: ---------------------------
+COMET INFO: Uploading metrics, params, and assets to Comet before program termination (may take several seconds)
+COMET INFO: The Python SDK has 3600 seconds to finish before aborting...
+COMET INFO: Uploading 1 metrics, params and output messages
+COMET INFO: Experiment is live on comet.ml https://www.comet.com/rabyj/epilap/15da476b92f140eab818ece369248f4c
+
+CometLogger will be initialized in online mode
+COMET INFO: -----------------------------------
+COMET INFO: Comet.ml ExistingExperiment Summary
+COMET INFO: -----------------------------------
+COMET INFO:   Data:
+COMET INFO:     display_summary_level : 1
+COMET INFO:     url                   : https://www.comet.com/rabyj/epilap/15da476b92f140eab818ece369248f4c
+COMET INFO:   Others:
+COMET INFO:     Name : hg38_100kb_all_none-harmonized_sample_cancer_high_1l_3000n-complete_no_valid_oversample
+COMET INFO:   Uploads:
+COMET INFO:     installed packages : 1
+COMET INFO: -----------------------------------
+COMET INFO: Experiment is live on comet.ml https://www.comet.com/rabyj/epilap/15da476b92f140eab818ece369248f4c
+
+COMET INFO: -----------------------------------
+COMET INFO: Comet.ml ExistingExperiment Summary
+COMET INFO: -----------------------------------
+COMET INFO:   Data:
+COMET INFO:     display_summary_level : 1
+COMET INFO:     url                   : https://www.comet.com/rabyj/epilap/15da476b92f140eab818ece369248f4c
+COMET INFO:   Metrics:
+COMET INFO:     Last epoch           : 300
+COMET INFO:     Training time        : 0:44:16
+COMET INFO:     tra_Accuracy         : 0.9986663460731506
+COMET INFO:     tra_F1Score          : 0.9986650943756104
+COMET INFO:     tra_MatthewsCorrCoef : 0.9973304271697998
+COMET INFO:     tra_Precision        : 0.9986739754676819
+COMET INFO:     tra_Recall           : 0.9986563920974731
+COMET INFO:   Uploads:
+COMET INFO:     asset              : 3 (163.54 KB)
+COMET INFO:     installed packages : 1
+COMET INFO: -----------------------------------
+COMET INFO: Uploading 1 metrics, params and output messages

complete_no_valid_oversample/output_job21525586_NN-dfreezev2.1-harmonized_sample_cancer_high-w-oversampling-no_valid-100kb_all_none_1695763876.o

@@ -0,0 +1,107 @@
+begin 2023-09-26 21:31:24
+Checking environment variables.
+EXCLUDE_LIST: ['other', '--', 'NA', '', 'unknown']
+Filtering metadata: Removing labels ['other', '--', 'NA', '', 'unknown'] from category 'harmonized_sample_cancer_high'.
+No label list, considering all left classes : ['cancer', 'non-cancer']
+2/2 labels left from harmonized_sample_cancer_high after removing classes with less than 10 signals.
+harmonized_sample_cancer_high label breakdown for unique experiments (uuid):
+non-cancer: 6614
+cancer: 1627
+For 8241 unique experiments in 2 classes
+
+Using files in /localscratch/rabyj.21525586.0/epiatlas_dfreeze_100kb_all_none
+Initial hdf5 loading time: 0:01:11
+The current experiment key is 15da476b92f140eab818ece369248f4c
+The current commit is v0.7.5.1-7-ga6afe82
+Split 0 train size: 34491
+Split 0 validation size: 0
+Split 0 test size: 0
+Total nb of files: 20922
+--MODEL STRUCTURE--
+ LightningDenseClassifier(
+  (_pt_model): Sequential(
+    (0): Dropout(p=0.1, inplace=False)
+    (1): Linear(in_features=30321, out_features=3000, bias=True)
+    (2): Dropout(p=0.5, inplace=False)
+    (3): ReLU()
+    (4): Linear(in_features=3000, out_features=2, bias=True)
+  )
+  (metrics): MetricCollection(
+    (Accuracy): Accuracy()
+    (Precision): Precision()
+    (Recall): Recall()
+    (F1Score): F1Score()
+    (MatthewsCorrCoef): MatthewsCorrCoef()
+  )
+  (train_acc): Accuracy()
+  (valid_acc): Accuracy()
+)
+--MODEL SUMMARY--
+===================================================================================================================
+Layer (type:depth-idx)                   Input Shape               Output Shape              Param #
+===================================================================================================================
+LightningDenseClassifier                 --                        --                        --
+├─Sequential: 1-1                        [1, 30321]                [1, 2]                    --
+│    └─Dropout: 2-1                      [1, 30321]                [1, 30321]                --
+│    └─Linear: 2-2                       [1, 30321]                [1, 3000]                 90,966,000
+│    └─Dropout: 2-3                      [1, 3000]                 [1, 3000]                 --
+│    └─ReLU: 2-4                         [1, 3000]                 [1, 3000]                 --
+│    └─Linear: 2-5                       [1, 3000]                 [1, 2]                    6,002
+├─MetricCollection: 1-2                  --                        --                        --
+│    └─Accuracy: 2-6                     --                        --                        --
+│    └─Precision: 2-7                    --                        --                        --
+│    └─Recall: 2-8                       --                        --                        --
+│    └─F1Score: 2-9                      --                        --                        --
+│    └─MatthewsCorrCoef: 2-10            --                        --                        --
+├─Accuracy: 1-3                          --                        --                        --
+├─Accuracy: 1-4                          --                        --                        --
+===================================================================================================================
+Total params: 90,972,002
+Trainable params: 90,972,002
+Non-trainable params: 0
+Total mult-adds (M): 90.97
+===================================================================================================================
+Input size (MB): 0.12
+Forward/backward pass size (MB): 0.02
+Params size (MB): 363.89
+Estimated Total Size (MB): 364.03
+===================================================================================================================
+--TRAINING HYPERPARAMETERS--
+L2 scale : 0.01
+Dropout rate : 0.5
+Learning rate : 1e-06
+No early stopping.
+Training batch size : 64
+┏━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━┳━━━━━━━━┓
+┃    ┃ Name                     ┃ Type             ┃ Params ┃
+┡━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━╇━━━━━━━━┩
+│ 0  │ _pt_model                │ Sequential       │ 91.0 M │
+│ 1  │ _pt_model.0              │ Dropout          │      0 │
+│ 2  │ _pt_model.1              │ Linear           │ 91.0 M │
+│ 3  │ _pt_model.2              │ Dropout          │      0 │
+│ 4  │ _pt_model.3              │ ReLU             │      0 │
+│ 5  │ _pt_model.4              │ Linear           │  6.0 K │
+│ 6  │ metrics                  │ MetricCollection │      0 │
+│ 7  │ metrics.Accuracy         │ Accuracy         │      0 │
+│ 8  │ metrics.Precision        │ Precision        │      0 │
+│ 9  │ metrics.Recall           │ Recall           │      0 │
+│ 10 │ metrics.F1Score          │ F1Score          │      0 │
+│ 11 │ metrics.MatthewsCorrCoef │ MatthewsCorrCoef │      0 │
+│ 12 │ train_acc                │ Accuracy         │      0 │
+│ 13 │ valid_acc                │ Accuracy         │      0 │
+└────┴──────────────────────────┴──────────────────┴────────┘
+Trainable params: 91.0 M                                                        
+Non-trainable params: 0                                                         
+Total params: 91.0 M                                                            
+Total estimated model params size (MB): 181                                     
+Saving model to /lustre06/project/6007017/rabyj/epilap/output/logs/epiatlas-dfreeze-v2.1/hg38_100kb_all_none/harmonized_sample_cancer_high_1l_3000n/complete_no_valid_oversample/EpiLaP/15da476b92f140eab818ece369248f4c/checkpoints/epoch=299-step=300.ckpt
+training time: 0:44:16
+Reading checkpoint list and taking last line.
+Loading model from /lustre06/project/6007017/rabyj/epilap/output/logs/epiatlas-dfreeze-v2.1/hg38_100kb_all_none/harmonized_sample_cancer_high_1l_3000n/complete_no_valid_oversample/EpiLaP/15da476b92f140eab818ece369248f4c/checkpoints/epoch=299-step=300.ckpt
+--- training set METRICS ---
+Accuracy 0.999
+Precision 0.999
+Recall 0.999
+F1Score 0.999
+MatthewsCorrCoef 0.997
+0.999 0.999 0.999 0.999 0.997

complete_no_valid_oversample/train_confusion_matrix.csv

@@ -0,0 +1,3 @@
+,cancer,non-cancer
+cancer(17766),17748,18
+non-cancer(16725),28,16697

complete_no_valid_oversample/train_confusion_matrix_relative.csv

@@ -0,0 +1,3 @@
+,cancer,non-cancer
+cancer(17766),0.9990,0.0010
+non-cancer(16725),0.0017,0.9983

complete_no_valid_oversample/training_mapping.tsv

@@ -0,0 +1,2 @@
+0	cancer
+1	non-cancer