Add files using upload-large-folder tool

.gitattributes

@@ -33,3 +33,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+fold_2/logs.models.fold_2.ENCSR926SNI/logfile.modelling.fold_2.ENCSR926SNI.stdout_v1.txt filter=lfs diff=lfs merge=lfs -text
+fold_2/logs.models.fold_2.ENCSR926SNI/logfile.modelling.fold_2.ENCSR926SNI.stdout.txt filter=lfs diff=lfs merge=lfs -text
+fold_3/logs.models.fold_3.ENCSR926SNI/logfile.modelling.fold_3.ENCSR926SNI.stdout_v1.txt filter=lfs diff=lfs merge=lfs -text

fold_0/logs.models.fold_0.ENCSR926SNI/logfile.modelling.fold_0.ENCSR926SNI.args.json

@@ -0,0 +1,23 @@
+{
+    "genome": "reference/hg38.genome.fa",
+    "bigwig": "data/ENCSR926SNI.bigWig",
+    "peaks": "chrombppnet_model_encsr283tme_bias//filtered.peaks.bed",
+    "nonpeaks": "chrombppnet_model_encsr283tme_bias//filtered.nonpeaks.bed",
+    "output_prefix": "chrombppnet_model_encsr283tme_bias//chrombpnet",
+    "chr_fold_path": "splits/fold_0.json",
+    "trackables": [
+        "logcount_predictions_loss",
+        "loss",
+        "logits_profile_predictions_loss",
+        "val_logcount_predictions_loss",
+        "val_loss",
+        "val_logits_profile_predictions_loss"
+    ],
+    "epochs": 50,
+    "early_stop": 5,
+    "batch_size": 64,
+    "learning_rate": 0.001,
+    "params": "chrombppnet_model_encsr283tme_bias//chrombpnet_model_params.tsv",
+    "seed": 1234,
+    "architecture_from_file": "/scratch/chrombpnet/src/training/models/chrombpnet_with_bias_model.py"
+}

fold_0/logs.models.fold_0.ENCSR926SNI/logfile.modelling.fold_0.ENCSR926SNI.bias_formatting.stderr.txt

@@ -0,0 +1,38 @@
+INFO:    underlay of /etc/localtime required more than 50 (88) bind mounts
+INFO:    underlay of /usr/bin/nvidia-smi required more than 50 (355) bind mounts
+2023-07-17 00:18:05.523076: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-07-17 00:18:07.777687: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set
+2023-07-17 00:18:07.780650: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1
+2023-07-17 00:18:08.387706: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: 
+pciBusID: 0000:84:00.0 name: NVIDIA A100-SXM4-80GB computeCapability: 8.0
+coreClock: 1.41GHz coreCount: 108 deviceMemorySize: 79.15GiB deviceMemoryBandwidth: 1.85TiB/s
+2023-07-17 00:18:08.387758: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-07-17 00:18:08.405710: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2023-07-17 00:18:08.405757: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2023-07-17 00:18:08.415274: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
+2023-07-17 00:18:08.419576: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
+2023-07-17 00:18:08.434865: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
+2023-07-17 00:18:08.438992: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
+2023-07-17 00:18:08.439880: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+2023-07-17 00:18:08.450615: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
+2023-07-17 00:18:08.450891: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+2023-07-17 00:18:08.451587: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set
+2023-07-17 00:18:08.461832: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: 
+pciBusID: 0000:84:00.0 name: NVIDIA A100-SXM4-80GB computeCapability: 8.0
+coreClock: 1.41GHz coreCount: 108 deviceMemorySize: 79.15GiB deviceMemoryBandwidth: 1.85TiB/s
+2023-07-17 00:18:08.461876: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-07-17 00:18:08.461908: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2023-07-17 00:18:08.461932: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2023-07-17 00:18:08.461953: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
+2023-07-17 00:18:08.461984: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
+2023-07-17 00:18:08.462006: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
+2023-07-17 00:18:08.462026: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
+2023-07-17 00:18:08.462046: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+2023-07-17 00:18:08.486941: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
+2023-07-17 00:18:08.488802: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-07-17 00:18:11.280616: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix:
+2023-07-17 00:18:11.280716: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267]      0 
+2023-07-17 00:18:11.280730: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0:   N 
+2023-07-17 00:18:11.287134: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 75650 MB memory) -> physical GPU (device: 0, name: NVIDIA A100-SXM4-80GB, pci bus id: 0000:84:00.0, compute capability: 8.0)
+2023-07-17 00:18:12.371942: W tensorflow/python/util/util.cc:348] Sets are not currently considered sequences, but this may change in the future, so consider avoiding using them.

fold_0/logs.models.fold_0.ENCSR926SNI/logfile.modelling.fold_0.ENCSR926SNI.chrombpnet_data_params.tsv

@@ -0,0 +1,3 @@
+counts_sum_min_thresh	2.0
+counts_sum_max_thresh	672.28
+trainings_pts_post_thresh	165773

fold_0/logs.models.fold_0.ENCSR926SNI/logfile.modelling.fold_0.ENCSR926SNI.chrombpnet_formatting.stderr.txt

@@ -0,0 +1,40 @@
+INFO:    underlay of /etc/localtime required more than 50 (88) bind mounts
+INFO:    underlay of /usr/bin/nvidia-smi required more than 50 (355) bind mounts
+2023-07-17 13:45:49.801778: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-07-17 13:45:52.897792: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set
+2023-07-17 13:45:52.902453: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1
+2023-07-17 13:45:52.962656: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: 
+pciBusID: 0000:04:00.0 name: Tesla P100-PCIE-16GB computeCapability: 6.0
+coreClock: 1.3285GHz coreCount: 56 deviceMemorySize: 15.89GiB deviceMemoryBandwidth: 681.88GiB/s
+2023-07-17 13:45:52.962716: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-07-17 13:45:52.988154: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2023-07-17 13:45:52.988260: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2023-07-17 13:45:53.001389: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
+2023-07-17 13:45:53.007628: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
+2023-07-17 13:45:53.033610: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
+2023-07-17 13:45:53.039667: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
+2023-07-17 13:45:53.040933: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+2023-07-17 13:45:53.074440: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
+2023-07-17 13:45:53.074888: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+2023-07-17 13:45:53.075823: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set
+2023-07-17 13:45:53.087737: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: 
+pciBusID: 0000:04:00.0 name: Tesla P100-PCIE-16GB computeCapability: 6.0
+coreClock: 1.3285GHz coreCount: 56 deviceMemorySize: 15.89GiB deviceMemoryBandwidth: 681.88GiB/s
+2023-07-17 13:45:53.087850: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-07-17 13:45:53.087918: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2023-07-17 13:45:53.087945: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2023-07-17 13:45:53.087970: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
+2023-07-17 13:45:53.087995: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
+2023-07-17 13:45:53.088020: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
+2023-07-17 13:45:53.088045: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
+2023-07-17 13:45:53.088070: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+2023-07-17 13:45:53.110111: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
+2023-07-17 13:45:53.111613: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-07-17 13:45:54.934027: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix:
+2023-07-17 13:45:54.934083: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267]      0 
+2023-07-17 13:45:54.934101: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0:   N 
+2023-07-17 13:45:54.971744: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 14957 MB memory) -> physical GPU (device: 0, name: Tesla P100-PCIE-16GB, pci bus id: 0000:04:00.0, compute capability: 6.0)
+2023-07-17 13:45:57.325829: W tensorflow/python/util/util.cc:348] Sets are not currently considered sequences, but this may change in the future, so consider avoiding using them.
+/opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/layers/core.py:1059: UserWarning:  is not loaded, but a Lambda layer uses it. It may cause errors.
+  , UserWarning)

fold_0/logs.models.fold_0.ENCSR926SNI/logfile.modelling.fold_0.ENCSR926SNI.chrombpnet_model_params.tsv

@@ -0,0 +1,9 @@
+counts_loss_weight	2.0
+filters	512
+n_dil_layers	8
+bias_model_path	chrombppnet_model_encsr283tme_bias/bias_model_scaled.h5
+inputlen	2114
+outputlen	1000
+max_jitter	500
+chr_fold_path	splits/fold_0.json
+negative_sampling_ratio	0.1

fold_0/logs.models.fold_0.ENCSR926SNI/logfile.modelling.fold_0.ENCSR926SNI.chrombpnet_no_bias_formatting.stderr.txt

@@ -0,0 +1 @@
+singularity exec --nv /home/groups/akundaje/anusri/simg/tf-atlas_gcp-modeling.sif python get_new_tf_model_format.py -i /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR926SNI//chrombppnet_model_encsr283tme_bias/chrombpnet_wo_bias.h5 -o /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR926SNI//chrombppnet_model_encsr283tme_bias/new_model_formats/chrombpnet_wo_bias

fold_0/model.bias_scaled.fold_0.ENCSR926SNI.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:365dcced74db1356803f8865c2569a4aaa2db798a6e1761ec10ce2ebfb116067
+size 2688440

fold_0/model.bias_scaled.fold_0.ENCSR926SNI.tar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3c904969a84c854b05163be9ea2eaa6cc79e48a64413cb005e8ba3120feadc23
+size 1208320

fold_0/model.chrombpnet.fold_0.ENCSR926SNI.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4229d44f4cd5f3a3b2c6297670a13dfe587b891822f42cf3422f9d67ae5f497a
+size 26448016

fold_0/model.chrombpnet.fold_0.ENCSR926SNI.tar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8a20a17ec8da3a10a863d8f39b29da532bb07633b5f5bd4f1b947931705a3a3f
+size 27617280

fold_0/model.chrombpnet_nobias.fold_0.ENCSR926SNI.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:474d09ddfdfbe50646c670a6dac38aa1c4b44b346c643b388cc9e267d15fc16f
+size 25583536

fold_0/model.chrombpnet_nobias.fold_0.ENCSR926SNI.tar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c703dba7397d31d71449757a216dc7e1a4b66bca1a31f7fd7c605985582bdcdb
+size 26081280

fold_1/model.bias_scaled.fold_1.ENCSR926SNI.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:36c953652d1731de63a4cfd7886ef017c5b869e1ff6917e7d35a8c97ed3e339d
+size 2688440

fold_1/model.bias_scaled.fold_1.ENCSR926SNI.tar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e7b438542208dfa4590fd7a75b937642e336b4f089ca1124fd5686552f375dea
+size 1198080

fold_1/model.chrombpnet.fold_1.ENCSR926SNI.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a8acde075f449bf0cba6c187de704a964d185417496b196202ffd86368470104
+size 26447928

fold_1/model.chrombpnet.fold_1.ENCSR926SNI.tar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:44da129b6f3ca5d519dc0a2702a5a7ca95edf22cd10d1ec7f3728069a2d697ac
+size 27607040

fold_1/model.chrombpnet_nobias.fold_1.ENCSR926SNI.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1a0f45573bc1a78aa8f6d8fcdfa412e4e31d3d9cfd85db56cc4ac84cea30889d
+size 25583536

fold_1/model.chrombpnet_nobias.fold_1.ENCSR926SNI.tar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:73e9bb8f0a20e1be89419bcdaea567b5b57e579e179b79acf78fca1a4c7fb5ef
+size 26081280

fold_2/logs.models.fold_2.ENCSR926SNI/logfile.modelling.fold_2.ENCSR926SNI.args.json

@@ -0,0 +1,23 @@
+{
+    "genome": "/scratch/groups/akundaje/anusri/chromatin_atlas/reference/hg38.genome.fa",
+    "bigwig": "/oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR926SNI//preprocessing/bigWigs/ENCSR926SNI.bigWig",
+    "peaks": "/scratch/groups/akundaje/anusri/chromatin_atlas/DNASE/ENCSR926SNI//chrombppnet_model_encsr283tme_bias_fold_2//filtered.peaks.bed",
+    "nonpeaks": "/scratch/groups/akundaje/anusri/chromatin_atlas/DNASE/ENCSR926SNI//chrombppnet_model_encsr283tme_bias_fold_2//filtered.nonpeaks.bed",
+    "output_prefix": "/scratch/groups/akundaje/anusri/chromatin_atlas/DNASE/ENCSR926SNI//chrombppnet_model_encsr283tme_bias_fold_2//chrombpnet",
+    "chr_fold_path": "/scratch/groups/akundaje/anusri/chromatin_atlas/splits/fold_2.json",
+    "trackables": [
+        "logcount_predictions_loss",
+        "loss",
+        "logits_profile_predictions_loss",
+        "val_logcount_predictions_loss",
+        "val_loss",
+        "val_logits_profile_predictions_loss"
+    ],
+    "epochs": 50,
+    "early_stop": 5,
+    "batch_size": 64,
+    "learning_rate": 0.001,
+    "params": "/scratch/groups/akundaje/anusri/chromatin_atlas/DNASE/ENCSR926SNI//chrombppnet_model_encsr283tme_bias_fold_2//chrombpnet_model_params.tsv",
+    "seed": 1234,
+    "architecture_from_file": "/home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/models/chrombpnet_with_bias_model.py"
+}

fold_2/logs.models.fold_2.ENCSR926SNI/logfile.modelling.fold_2.ENCSR926SNI.bias_formatting.stderr.txt

@@ -0,0 +1,38 @@
+INFO:    underlay of /etc/localtime required more than 50 (88) bind mounts
+INFO:    underlay of /usr/bin/nvidia-smi required more than 50 (355) bind mounts
+2023-07-17 00:18:04.681791: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-07-17 00:18:06.987369: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set
+2023-07-17 00:18:06.990826: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1
+2023-07-17 00:18:07.134359: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: 
+pciBusID: 0000:07:00.0 name: NVIDIA A100-SXM4-80GB computeCapability: 8.0
+coreClock: 1.41GHz coreCount: 108 deviceMemorySize: 79.15GiB deviceMemoryBandwidth: 1.85TiB/s
+2023-07-17 00:18:07.134448: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-07-17 00:18:07.153096: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2023-07-17 00:18:07.153149: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2023-07-17 00:18:07.162576: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
+2023-07-17 00:18:07.167136: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
+2023-07-17 00:18:07.183431: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
+2023-07-17 00:18:07.187703: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
+2023-07-17 00:18:07.188634: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+2023-07-17 00:18:07.194265: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
+2023-07-17 00:18:07.194547: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+2023-07-17 00:18:07.195263: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set
+2023-07-17 00:18:07.197543: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: 
+pciBusID: 0000:07:00.0 name: NVIDIA A100-SXM4-80GB computeCapability: 8.0
+coreClock: 1.41GHz coreCount: 108 deviceMemorySize: 79.15GiB deviceMemoryBandwidth: 1.85TiB/s
+2023-07-17 00:18:07.197566: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-07-17 00:18:07.197581: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2023-07-17 00:18:07.197593: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2023-07-17 00:18:07.197604: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
+2023-07-17 00:18:07.197615: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
+2023-07-17 00:18:07.197625: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
+2023-07-17 00:18:07.197635: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
+2023-07-17 00:18:07.197645: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+2023-07-17 00:18:07.202092: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
+2023-07-17 00:18:07.203170: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-07-17 00:18:09.306791: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix:
+2023-07-17 00:18:09.306949: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267]      0 
+2023-07-17 00:18:09.306960: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0:   N 
+2023-07-17 00:18:09.389090: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 75650 MB memory) -> physical GPU (device: 0, name: NVIDIA A100-SXM4-80GB, pci bus id: 0000:07:00.0, compute capability: 8.0)
+2023-07-17 00:18:10.623909: W tensorflow/python/util/util.cc:348] Sets are not currently considered sequences, but this may change in the future, so consider avoiding using them.

fold_2/logs.models.fold_2.ENCSR926SNI/logfile.modelling.fold_2.ENCSR926SNI.bias_formatting.stdout.txt

@@ -0,0 +1 @@
+singularity exec --nv /home/groups/akundaje/anusri/simg/tf-atlas_gcp-modeling.sif python get_new_tf_model_format.py -i /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR926SNI//chrombppnet_model_encsr283tme_bias_fold_2/bias_model_scaled.h5 -o /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR926SNI//chrombppnet_model_encsr283tme_bias_fold_2/new_model_formats/bias_model_scaled

fold_2/logs.models.fold_2.ENCSR926SNI/logfile.modelling.fold_2.ENCSR926SNI.chrombpnet.params.json

@@ -0,0 +1,11 @@
+{
+    "counts_loss_weight": "2.0",
+    "filters": "512",
+    "n_dil_layers": "8",
+    "bias_model_path": "/scratch/groups/akundaje/anusri/chromatin_atlas/DNASE/ENCSR926SNI//chrombppnet_model_encsr283tme_bias_fold_2/bias_model_scaled.h5",
+    "inputlen": "2114",
+    "outputlen": "1000",
+    "max_jitter": "500",
+    "chr_fold_path": "/scratch/groups/akundaje/anusri/chromatin_atlas/splits/fold_2.json",
+    "negative_sampling_ratio": "0.1"
+}

fold_2/logs.models.fold_2.ENCSR926SNI/logfile.modelling.fold_2.ENCSR926SNI.chrombpnet_data_params.tsv

@@ -0,0 +1,3 @@
+counts_sum_min_thresh	2.0
+counts_sum_max_thresh	674.0
+trainings_pts_post_thresh	172387

fold_2/logs.models.fold_2.ENCSR926SNI/logfile.modelling.fold_2.ENCSR926SNI.chrombpnet_formatting.stderr.txt

@@ -0,0 +1,40 @@
+INFO:    underlay of /etc/localtime required more than 50 (88) bind mounts
+INFO:    underlay of /usr/bin/nvidia-smi required more than 50 (355) bind mounts
+2023-07-17 13:45:51.338327: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-07-17 13:45:54.042094: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set
+2023-07-17 13:45:54.045730: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1
+2023-07-17 13:45:54.945150: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: 
+pciBusID: 0000:c0:00.0 name: NVIDIA A100-SXM4-80GB computeCapability: 8.0
+coreClock: 1.41GHz coreCount: 108 deviceMemorySize: 79.15GiB deviceMemoryBandwidth: 1.85TiB/s
+2023-07-17 13:45:54.945254: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-07-17 13:45:54.967890: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2023-07-17 13:45:54.968045: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2023-07-17 13:45:54.978870: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
+2023-07-17 13:45:54.984127: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
+2023-07-17 13:45:55.002440: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
+2023-07-17 13:45:55.007348: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
+2023-07-17 13:45:55.008484: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+2023-07-17 13:45:55.053483: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
+2023-07-17 13:45:55.053908: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+2023-07-17 13:45:55.054916: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set
+2023-07-17 13:45:55.077851: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: 
+pciBusID: 0000:c0:00.0 name: NVIDIA A100-SXM4-80GB computeCapability: 8.0
+coreClock: 1.41GHz coreCount: 108 deviceMemorySize: 79.15GiB deviceMemoryBandwidth: 1.85TiB/s
+2023-07-17 13:45:55.077955: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-07-17 13:45:55.078014: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2023-07-17 13:45:55.078038: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2023-07-17 13:45:55.078055: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
+2023-07-17 13:45:55.078071: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
+2023-07-17 13:45:55.078087: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
+2023-07-17 13:45:55.078102: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
+2023-07-17 13:45:55.078118: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+2023-07-17 13:45:55.120513: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
+2023-07-17 13:45:55.122332: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-07-17 13:45:57.755231: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix:
+2023-07-17 13:45:57.755332: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267]      0 
+2023-07-17 13:45:57.755350: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0:   N 
+2023-07-17 13:45:57.806289: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 75650 MB memory) -> physical GPU (device: 0, name: NVIDIA A100-SXM4-80GB, pci bus id: 0000:c0:00.0, compute capability: 8.0)
+2023-07-17 13:46:00.372660: W tensorflow/python/util/util.cc:348] Sets are not currently considered sequences, but this may change in the future, so consider avoiding using them.
+/opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/layers/core.py:1059: UserWarning:  is not loaded, but a Lambda layer uses it. It may cause errors.
+  , UserWarning)

fold_2/logs.models.fold_2.ENCSR926SNI/logfile.modelling.fold_2.ENCSR926SNI.chrombpnet_formatting.stdout.txt

@@ -0,0 +1 @@
+singularity exec --nv /home/groups/akundaje/anusri/simg/tf-atlas_gcp-modeling.sif python get_new_tf_model_format.py -i /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR926SNI//chrombppnet_model_encsr283tme_bias_fold_2/chrombpnet.h5 -o /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR926SNI//chrombppnet_model_encsr283tme_bias_fold_2/new_model_formats/chrombpnet

fold_2/logs.models.fold_2.ENCSR926SNI/logfile.modelling.fold_2.ENCSR926SNI.chrombpnet_model_params.tsv

@@ -0,0 +1,9 @@
+counts_loss_weight	2.0
+filters	512
+n_dil_layers	8
+bias_model_path	/scratch/groups/akundaje/anusri/chromatin_atlas/DNASE/ENCSR926SNI//chrombppnet_model_encsr283tme_bias_fold_2/bias_model_scaled.h5
+inputlen	2114
+outputlen	1000
+max_jitter	500
+chr_fold_path	/scratch/groups/akundaje/anusri/chromatin_atlas/splits/fold_2.json
+negative_sampling_ratio	0.1

fold_2/logs.models.fold_2.ENCSR926SNI/logfile.modelling.fold_2.ENCSR926SNI.chrombpnet_no_bias_formatting.stderr.txt

@@ -0,0 +1 @@
+singularity exec --nv /home/groups/akundaje/anusri/simg/tf-atlas_gcp-modeling.sif python get_new_tf_model_format.py -i /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR926SNI//chrombppnet_model_encsr283tme_bias_fold_2/chrombpnet_wo_bias.h5 -o /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR926SNI//chrombppnet_model_encsr283tme_bias_fold_2/new_model_formats/chrombpnet_wo_bias

fold_2/logs.models.fold_2.ENCSR926SNI/logfile.modelling.fold_2.ENCSR926SNI.chrombpnet_no_bias_formatting.stdout.txt

@@ -0,0 +1 @@
+singularity exec --nv /home/groups/akundaje/anusri/simg/tf-atlas_gcp-modeling.sif python get_new_tf_model_format.py -i /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR926SNI//chrombppnet_model_encsr283tme_bias_fold_2/chrombpnet_wo_bias.h5 -o /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR926SNI//chrombppnet_model_encsr283tme_bias_fold_2/new_model_formats/chrombpnet_wo_bias

fold_2/logs.models.fold_2.ENCSR926SNI/logfile.modelling.fold_2.ENCSR926SNI.epoch_loss.csv

@@ -0,0 +1,16 @@
+epoch,logcount_predictions_loss,logits_profile_predictions_loss,loss,val_logcount_predictions_loss,val_logits_profile_predictions_loss,val_loss
+0,0.9856865406036377,148.90162658691406,150.87289428710938,0.43896692991256714,174.7969207763672,175.67483520507812
+1,0.41199353337287903,144.85240173339844,145.67616271972656,0.4082932770252228,172.8484344482422,173.66488647460938
+2,0.3741987645626068,143.9105682373047,144.65890502929688,0.39326509833335876,172.4113006591797,173.19789123535156
+3,0.3531371057033539,143.2674102783203,143.97372436523438,0.34511858224868774,172.2469024658203,172.9371337890625
+4,0.3418785035610199,142.69363403320312,143.37730407714844,0.34173935651779175,171.83248901367188,172.5160675048828
+5,0.3287820816040039,142.40179443359375,143.0593719482422,0.3311428129673004,171.18658447265625,171.84884643554688
+6,0.32178813219070435,141.8069305419922,142.45066833496094,0.3262394070625305,170.99339294433594,171.64590454101562
+7,0.3131655752658844,141.54100036621094,142.16748046875,0.3220241963863373,171.26219177246094,171.90621948242188
+8,0.3074494004249573,141.0019989013672,141.61691284179688,0.33417433500289917,171.57652282714844,172.24488830566406
+9,0.3028903901576996,140.80726623535156,141.4130859375,0.32696807384490967,170.9103546142578,171.56430053710938
+10,0.2976846694946289,140.65277099609375,141.24795532226562,0.3533153235912323,171.73422241210938,172.4409637451172
+11,0.2945251166820526,140.12649536132812,140.71568298339844,0.32521095871925354,171.94894409179688,172.59933471679688
+12,0.29125717282295227,139.9418182373047,140.52456665039062,0.3229660093784332,171.82447814941406,172.4704132080078
+13,0.2717306315898895,138.17076110839844,138.71426391601562,0.31471094489097595,171.7954559326172,172.42491149902344
+14,0.2647063732147217,137.3122100830078,137.84156799316406,0.3224015533924103,172.56912231445312,173.21392822265625

fold_2/logs.models.fold_2.ENCSR926SNI/logfile.modelling.fold_2.ENCSR926SNI.stdout.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:10fcf2cf658bb786273e1ace3b253163fb7dfe87580f995a99aab6262dd957bc
+size 11698591

fold_2/logs.models.fold_2.ENCSR926SNI/logfile.modelling.fold_2.ENCSR926SNI.stdout_v1.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8642a625ec14406126f8a2a21656593ef7920442e632f64f183fb985a4c5a1e4
+size 11696918

fold_2/model.bias_scaled.fold_2.ENCSR926SNI.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0902d03e2981bc8bd7ad3e91d10e458aac36cde273fa09fd7947d65f4f890146
+size 2688440

fold_2/model.bias_scaled.fold_2.ENCSR926SNI.tar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f1e83a6f8171bfaaba3ec2d020bc6013c13c7dcb53b8605412b451fa4f682ce8
+size 1198080

fold_2/model.chrombpnet.fold_2.ENCSR926SNI.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:71a2251d353617ad9be202878a705126aaf337c6e68b4615d1118d81c268a6fd
+size 26447928

fold_2/model.chrombpnet.fold_2.ENCSR926SNI.tar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a1ded50e77ba359376817f0b47e82f71c9333382eae7ad044865bb15ce0d8d9d
+size 27607040

fold_2/model.chrombpnet_nobias.fold_2.ENCSR926SNI.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:15fc19fc8321d5b1f5494c2ee233d57372835572e1051dcc277b53df59b13c5e
+size 25583536

fold_2/model.chrombpnet_nobias.fold_2.ENCSR926SNI.tar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e54c6e49b6f7ee7576dbba59ab838ee962930868c931224f794352398fbf96f1
+size 26081280

fold_3/logs.models.fold_3.ENCSR926SNI/logfile.modelling.fold_3.ENCSR926SNI.stdout_v1.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:46f2a8010057c1bcb9a4a92a2b1aa79776af8672dc46479d5c760d8f3952f897
+size 16240120

fold_3/model.bias_scaled.fold_3.ENCSR926SNI.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c2c4ecf8242df7709c8515f91929c9055d4d6d86ca26d1444c848f95580781a
+size 2688440

fold_3/model.bias_scaled.fold_3.ENCSR926SNI.tar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:52f6eec7248ea8c481dc2c4ed250caf1df5c9a3d22befc8fa55ef7373ee5ce59
+size 1198080

fold_3/model.chrombpnet.fold_3.ENCSR926SNI.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:50a8664272023b8d7eaa4f57c395e77c77672d19def1dc6c2d7fd78dac038ee2
+size 26447928

fold_3/model.chrombpnet.fold_3.ENCSR926SNI.tar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ac3486268db84d61e7dbd56aa34f51771e3da7389faec253b242365a5ad9fed0
+size 27607040

fold_3/model.chrombpnet_nobias.fold_3.ENCSR926SNI.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:34976eb7a3880b56edf2faf2c462a30e7cefeef1649b332fd1026162e78c1839
+size 25583536

fold_3/model.chrombpnet_nobias.fold_3.ENCSR926SNI.tar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:781572f31399cd1110e561c80da24fc003870bb33aa5ccd389eeda8936a64636
+size 26081280

fold_4/model.bias_scaled.fold_4.ENCSR926SNI.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd71c55a9ab09eeea3bcce837ff48ebbc295eac38b180814e986b085ffb6a82e
+size 2688440