encode-chrombpnet-DNASE-ENCSR605SQL-ENCSR894QMU / fold_1 /logs.models.fold_1.ENCSR605SQL /logfile.modelling.fold_1.ENCSR605SQL.stderr.txt
| 2022-10-06 02:26:26.905902: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-10-06 02:31:40.245128: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set | |
| 2022-10-06 02:31:40.247401: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1 | |
| 2022-10-06 02:31:40.287649: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: | |
| pciBusID: 0000:03:00.0 name: NVIDIA TITAN V computeCapability: 7.0 | |
| coreClock: 1.455GHz coreCount: 80 deviceMemorySize: 11.78GiB deviceMemoryBandwidth: 607.97GiB/s | |
| 2022-10-06 02:31:40.287737: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-10-06 02:31:40.316331: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-10-06 02:31:40.316455: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-10-06 02:31:40.331725: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10 | |
| 2022-10-06 02:31:40.338710: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10 | |
| 2022-10-06 02:31:40.364060: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10 | |
| 2022-10-06 02:31:40.371176: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11 | |
| 2022-10-06 02:31:40.372765: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| 2022-10-06 02:31:40.375011: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0 | |
| 2022-10-06 02:31:40.375405: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA | |
| To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. | |
| 2022-10-06 02:31:40.375494: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set | |
| 2022-10-06 02:31:40.375795: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: | |
| pciBusID: 0000:03:00.0 name: NVIDIA TITAN V computeCapability: 7.0 | |
| coreClock: 1.455GHz coreCount: 80 deviceMemorySize: 11.78GiB deviceMemoryBandwidth: 607.97GiB/s | |
| 2022-10-06 02:31:40.375834: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-10-06 02:31:40.375861: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-10-06 02:31:40.375885: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-10-06 02:31:40.375910: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10 | |
| 2022-10-06 02:31:40.375940: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10 | |
| 2022-10-06 02:31:40.375966: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10 | |
| 2022-10-06 02:31:40.375989: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11 | |
| 2022-10-06 02:31:40.376012: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| 2022-10-06 02:31:40.376460: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0 | |
| 2022-10-06 02:31:40.378217: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-10-06 02:31:42.241053: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix: | |
| 2022-10-06 02:31:42.241156: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267] 0 | |
| 2022-10-06 02:31:42.241176: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0: N | |
| 2022-10-06 02:31:42.246168: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10912 MB memory) -> physical GPU (device: 0, name: NVIDIA TITAN V, pci bus id: 0000:03:00.0, compute capability: 7.0) | |
| 2022-10-06 02:31:43.584855: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:116] None of the MLIR optimization passes are enabled (registered 2) | |
| 2022-10-06 02:31:43.594263: I tensorflow/core/platform/profile_utils/cpu_utils.cc:112] CPU Frequency: 2400240000 Hz | |
| 2022-10-06 02:31:43.811313: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-10-06 02:31:44.998456: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-10-06 02:31:45.007538: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| 2022-10-06 02:32:22.172698: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-10-06 02:32:24.213566: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set | |
| 2022-10-06 02:32:24.214579: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1 | |
| 2022-10-06 02:32:24.254490: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: | |
| pciBusID: 0000:03:00.0 name: NVIDIA TITAN V computeCapability: 7.0 | |
| coreClock: 1.455GHz coreCount: 80 deviceMemorySize: 11.78GiB deviceMemoryBandwidth: 607.97GiB/s | |
| 2022-10-06 02:32:24.254595: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-10-06 02:32:24.257906: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-10-06 02:32:24.257994: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-10-06 02:32:24.259462: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10 | |
| 2022-10-06 02:32:24.259725: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10 | |
| 2022-10-06 02:32:24.263227: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10 | |
| 2022-10-06 02:32:24.263942: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11 | |
| 2022-10-06 02:32:24.264108: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| 2022-10-06 02:32:24.264642: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0 | |
| 2022-10-06 02:32:24.264999: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA | |
| To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. | |
| 2022-10-06 02:32:24.265088: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set | |
| 2022-10-06 02:32:24.265403: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: | |
| pciBusID: 0000:03:00.0 name: NVIDIA TITAN V computeCapability: 7.0 | |
| coreClock: 1.455GHz coreCount: 80 deviceMemorySize: 11.78GiB deviceMemoryBandwidth: 607.97GiB/s | |
| 2022-10-06 02:32:24.265459: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-10-06 02:32:24.265488: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-10-06 02:32:24.265512: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-10-06 02:32:24.265534: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10 | |
| 2022-10-06 02:32:24.265556: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10 | |
| 2022-10-06 02:32:24.265609: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10 | |
| 2022-10-06 02:32:24.265632: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11 | |
| 2022-10-06 02:32:24.265654: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| 2022-10-06 02:32:24.266195: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0 | |
| 2022-10-06 02:32:24.266240: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-10-06 02:32:24.920909: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix: | |
| 2022-10-06 02:32:24.921028: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267] 0 | |
| 2022-10-06 02:32:24.921052: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0: N | |
| 2022-10-06 02:32:24.922010: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10912 MB memory) -> physical GPU (device: 0, name: NVIDIA TITAN V, pci bus id: 0000:03:00.0, compute capability: 7.0) | |
| 2022-10-06 02:37:42.438099: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:116] None of the MLIR optimization passes are enabled (registered 2) | |
| 2022-10-06 02:37:42.438621: I tensorflow/core/platform/profile_utils/cpu_utils.cc:112] CPU Frequency: 2400240000 Hz | |
| 2022-10-06 02:37:44.591334: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-10-06 02:37:44.923850: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-10-06 02:37:44.942688: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 0.2098s vs `on_train_batch_end` time: 0.2989s). Check your callbacks. | |
| 2022-10-06 06:15:06.364892: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-10-06 06:15:09.830843: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set | |
| 2022-10-06 06:15:09.831939: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1 | |
| 2022-10-06 06:15:09.874138: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: | |
| pciBusID: 0000:03:00.0 name: NVIDIA TITAN V computeCapability: 7.0 | |
| coreClock: 1.455GHz coreCount: 80 deviceMemorySize: 11.78GiB deviceMemoryBandwidth: 607.97GiB/s | |
| 2022-10-06 06:15:09.874221: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-10-06 06:15:09.877585: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-10-06 06:15:09.877661: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-10-06 06:15:09.879102: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10 | |
| 2022-10-06 06:15:09.879414: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10 | |
| 2022-10-06 06:15:09.882958: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10 | |
| 2022-10-06 06:15:09.883707: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11 | |
| 2022-10-06 06:15:09.883881: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| 2022-10-06 06:15:09.884529: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0 | |
| 2022-10-06 06:15:09.884928: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA | |
| To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. | |
| 2022-10-06 06:15:09.885023: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set | |
| 2022-10-06 06:15:09.885345: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: | |
| pciBusID: 0000:03:00.0 name: NVIDIA TITAN V computeCapability: 7.0 | |
| coreClock: 1.455GHz coreCount: 80 deviceMemorySize: 11.78GiB deviceMemoryBandwidth: 607.97GiB/s | |
| 2022-10-06 06:15:09.885380: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-10-06 06:15:09.885409: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-10-06 06:15:09.885436: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-10-06 06:15:09.885462: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10 | |
| 2022-10-06 06:15:09.885487: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10 | |
| 2022-10-06 06:15:09.885512: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10 | |
| 2022-10-06 06:15:09.885538: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11 | |
| 2022-10-06 06:15:09.885563: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| 2022-10-06 06:15:09.886121: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0 | |
| 2022-10-06 06:15:09.886169: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-10-06 06:15:10.534368: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix: | |
| 2022-10-06 06:15:10.534471: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267] 0 | |
| 2022-10-06 06:15:10.534490: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0: N | |
| 2022-10-06 06:15:10.535492: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10912 MB memory) -> physical GPU (device: 0, name: NVIDIA TITAN V, pci bus id: 0000:03:00.0, compute capability: 7.0) | |
| 2022-10-06 06:16:40.978095: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:116] None of the MLIR optimization passes are enabled (registered 2) | |
| 2022-10-06 06:16:40.983134: I tensorflow/core/platform/profile_utils/cpu_utils.cc:112] CPU Frequency: 2400240000 Hz | |
| 2022-10-06 06:16:41.111477: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-10-06 06:16:41.461058: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-10-06 06:16:41.463593: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| /opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/layers/core.py:1059: UserWarning: is not loaded, but a Lambda layer uses it. It may cause errors. | |
| , UserWarning) | |
| /home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:69: RuntimeWarning: invalid value encountered in true_divide | |
| cur_jsd=jensenshannon(true_counts[idx,:]/np.nansum(true_counts[idx,:]),pred_probs[idx,:]) | |
| /home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/utils/metrics_utils.py:196: RuntimeWarning: invalid value encountered in true_divide | |
| profile_prob = profile / np.sum(profile) | |
| /home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:78: RuntimeWarning: invalid value encountered in true_divide | |
| shuffled_labels_prob=shuffled_labels/np.nansum(shuffled_labels) | |
| /home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:88: RuntimeWarning: invalid value encountered in true_divide | |
| curr_jsd_rnd=jensenshannon(true_counts[idx,:]/np.nansum(true_counts[idx,:]),shuffled_labels_prob) | |
| No artists with labels found to put in legend. Note that artists whose label start with an underscore are ignored when legend() is called with no argument. | |
| findfont: Font family ['normal'] not found. Falling back to DejaVu Sans. | |
| findfont: Font family ['normal'] not found. Falling back to DejaVu Sans. | |
| findfont: Font family ['normal'] not found. Falling back to DejaVu Sans. | |
| No artists with labels found to put in legend. Note that artists whose label start with an underscore are ignored when legend() is called with no argument. | |
| No artists with labels found to put in legend. Note that artists whose label start with an underscore are ignored when legend() is called with no argument. | |
| 2022-10-06 06:20:41.316658: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-10-06 06:20:44.485992: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set | |
| 2022-10-06 06:20:44.487165: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1 | |
| 2022-10-06 06:20:44.527755: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: | |
| pciBusID: 0000:03:00.0 name: NVIDIA TITAN V computeCapability: 7.0 | |
| coreClock: 1.455GHz coreCount: 80 deviceMemorySize: 11.78GiB deviceMemoryBandwidth: 607.97GiB/s | |
| 2022-10-06 06:20:44.527860: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-10-06 06:20:44.531382: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-10-06 06:20:44.531492: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-10-06 06:20:44.532963: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10 | |
| 2022-10-06 06:20:44.533239: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10 | |
| 2022-10-06 06:20:44.536873: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10 | |
| 2022-10-06 06:20:44.537687: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11 | |
| 2022-10-06 06:20:44.537870: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| 2022-10-06 06:20:44.539053: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0 | |
| 2022-10-06 06:20:44.539469: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA | |
| To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. | |
| 2022-10-06 06:20:44.539598: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set | |
| 2022-10-06 06:20:44.540686: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: | |
| pciBusID: 0000:03:00.0 name: NVIDIA TITAN V computeCapability: 7.0 | |
| coreClock: 1.455GHz coreCount: 80 deviceMemorySize: 11.78GiB deviceMemoryBandwidth: 607.97GiB/s | |
| 2022-10-06 06:20:44.540728: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-10-06 06:20:44.540780: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-10-06 06:20:44.540810: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-10-06 06:20:44.540840: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10 | |
| 2022-10-06 06:20:44.540869: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10 | |
| 2022-10-06 06:20:44.540897: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10 | |
| 2022-10-06 06:20:44.540948: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11 | |
| 2022-10-06 06:20:44.540980: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| 2022-10-06 06:20:44.541483: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0 | |
| 2022-10-06 06:20:44.541527: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-10-06 06:20:45.214688: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix: | |
| 2022-10-06 06:20:45.214803: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267] 0 | |
| 2022-10-06 06:20:45.214825: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0: N | |
| 2022-10-06 06:20:45.215866: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10912 MB memory) -> physical GPU (device: 0, name: NVIDIA TITAN V, pci bus id: 0000:03:00.0, compute capability: 7.0) | |
| WARNING:tensorflow:No training configuration found in the save file, so the model was *not* compiled. Compile it manually. | |
| 2022-10-06 06:22:10.841980: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:116] None of the MLIR optimization passes are enabled (registered 2) | |
| 2022-10-06 06:22:10.845365: I tensorflow/core/platform/profile_utils/cpu_utils.cc:112] CPU Frequency: 2400240000 Hz | |
| 2022-10-06 06:22:10.929642: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-10-06 06:22:11.278746: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-10-06 06:22:11.280547: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| /home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:69: RuntimeWarning: invalid value encountered in true_divide | |
| cur_jsd=jensenshannon(true_counts[idx,:]/np.nansum(true_counts[idx,:]),pred_probs[idx,:]) | |
| /home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/utils/metrics_utils.py:196: RuntimeWarning: invalid value encountered in true_divide | |
| profile_prob = profile / np.sum(profile) | |
| /home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:78: RuntimeWarning: invalid value encountered in true_divide | |
| shuffled_labels_prob=shuffled_labels/np.nansum(shuffled_labels) | |
| /home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:88: RuntimeWarning: invalid value encountered in true_divide | |
| curr_jsd_rnd=jensenshannon(true_counts[idx,:]/np.nansum(true_counts[idx,:]),shuffled_labels_prob) | |
| No artists with labels found to put in legend. Note that artists whose label start with an underscore are ignored when legend() is called with no argument. | |
| findfont: Font family ['normal'] not found. Falling back to DejaVu Sans. | |
| findfont: Font family ['normal'] not found. Falling back to DejaVu Sans. | |
| findfont: Font family ['normal'] not found. Falling back to DejaVu Sans. | |
| No artists with labels found to put in legend. Note that artists whose label start with an underscore are ignored when legend() is called with no argument. | |
| No artists with labels found to put in legend. Note that artists whose label start with an underscore are ignored when legend() is called with no argument. | |
| 2022-10-06 06:25:57.959609: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-10-06 06:26:01.218216: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set | |
| 2022-10-06 06:26:01.219372: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1 | |
| 2022-10-06 06:26:01.260953: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: | |
| pciBusID: 0000:03:00.0 name: NVIDIA TITAN V computeCapability: 7.0 | |
| coreClock: 1.455GHz coreCount: 80 deviceMemorySize: 11.78GiB deviceMemoryBandwidth: 607.97GiB/s | |
| 2022-10-06 06:26:01.261057: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-10-06 06:26:01.264453: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-10-06 06:26:01.264520: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-10-06 06:26:01.265910: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10 | |
| 2022-10-06 06:26:01.266194: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10 | |
| 2022-10-06 06:26:01.269637: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10 | |
| 2022-10-06 06:26:01.270424: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11 | |
| 2022-10-06 06:26:01.270610: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| 2022-10-06 06:26:01.271253: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0 | |
| 2022-10-06 06:26:01.271639: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA | |
| To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. | |
| 2022-10-06 06:26:01.271734: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set | |
| 2022-10-06 06:26:01.272052: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: | |
| pciBusID: 0000:03:00.0 name: NVIDIA TITAN V computeCapability: 7.0 | |
| coreClock: 1.455GHz coreCount: 80 deviceMemorySize: 11.78GiB deviceMemoryBandwidth: 607.97GiB/s | |
| 2022-10-06 06:26:01.272088: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-10-06 06:26:01.272119: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-10-06 06:26:01.272146: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-10-06 06:26:01.272198: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10 | |
| 2022-10-06 06:26:01.272228: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10 | |
| 2022-10-06 06:26:01.272254: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10 | |
| 2022-10-06 06:26:01.272280: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11 | |
| 2022-10-06 06:26:01.272306: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| 2022-10-06 06:26:01.272799: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0 | |
| 2022-10-06 06:26:01.272842: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-10-06 06:26:01.935269: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix: | |
| 2022-10-06 06:26:01.935385: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267] 0 | |
| 2022-10-06 06:26:01.935406: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0: N | |
| 2022-10-06 06:26:01.937033: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10912 MB memory) -> physical GPU (device: 0, name: NVIDIA TITAN V, pci bus id: 0000:03:00.0, compute capability: 7.0) | |
| 2022-10-06 06:27:28.086945: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:116] None of the MLIR optimization passes are enabled (registered 2) | |
| 2022-10-06 06:27:28.089238: I tensorflow/core/platform/profile_utils/cpu_utils.cc:112] CPU Frequency: 2400240000 Hz | |
| 2022-10-06 06:27:28.142244: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-10-06 06:27:28.456712: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-10-06 06:27:28.458476: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| /home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:69: RuntimeWarning: invalid value encountered in true_divide | |
| cur_jsd=jensenshannon(true_counts[idx,:]/np.nansum(true_counts[idx,:]),pred_probs[idx,:]) | |
| /home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/utils/metrics_utils.py:196: RuntimeWarning: invalid value encountered in true_divide | |
| profile_prob = profile / np.sum(profile) | |
| /home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:78: RuntimeWarning: invalid value encountered in true_divide | |
| shuffled_labels_prob=shuffled_labels/np.nansum(shuffled_labels) | |
| /home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:88: RuntimeWarning: invalid value encountered in true_divide | |
| curr_jsd_rnd=jensenshannon(true_counts[idx,:]/np.nansum(true_counts[idx,:]),shuffled_labels_prob) | |
| No artists with labels found to put in legend. Note that artists whose label start with an underscore are ignored when legend() is called with no argument. | |
| findfont: Font family ['normal'] not found. Falling back to DejaVu Sans. | |
| findfont: Font family ['normal'] not found. Falling back to DejaVu Sans. | |
| findfont: Font family ['normal'] not found. Falling back to DejaVu Sans. | |
| No artists with labels found to put in legend. Note that artists whose label start with an underscore are ignored when legend() is called with no argument. | |
| No artists with labels found to put in legend. Note that artists whose label start with an underscore are ignored when legend() is called with no argument. | |
| 2022-10-06 06:28:39.939343: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-10-06 06:28:41.573102: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set | |
| 2022-10-06 06:28:41.574219: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1 | |
| 2022-10-06 06:28:41.612739: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: | |
| pciBusID: 0000:03:00.0 name: NVIDIA TITAN V computeCapability: 7.0 | |
| coreClock: 1.455GHz coreCount: 80 deviceMemorySize: 11.78GiB deviceMemoryBandwidth: 607.97GiB/s | |
| 2022-10-06 06:28:41.612852: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-10-06 06:28:41.616282: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-10-06 06:28:41.616401: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-10-06 06:28:41.617871: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10 | |
| 2022-10-06 06:28:41.618159: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10 | |
| 2022-10-06 06:28:41.621685: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10 | |
| 2022-10-06 06:28:41.622444: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11 | |
| 2022-10-06 06:28:41.622635: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| 2022-10-06 06:28:41.623276: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0 | |
| 2022-10-06 06:28:41.623642: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA | |
| To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. | |
| 2022-10-06 06:28:41.623742: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set | |
| 2022-10-06 06:28:41.624075: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: | |
| pciBusID: 0000:03:00.0 name: NVIDIA TITAN V computeCapability: 7.0 | |
| coreClock: 1.455GHz coreCount: 80 deviceMemorySize: 11.78GiB deviceMemoryBandwidth: 607.97GiB/s | |
| 2022-10-06 06:28:41.624121: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-10-06 06:28:41.624151: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-10-06 06:28:41.624179: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-10-06 06:28:41.624207: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10 | |
| 2022-10-06 06:28:41.624234: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10 | |
| 2022-10-06 06:28:41.624261: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10 | |
| 2022-10-06 06:28:41.624310: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11 | |
| 2022-10-06 06:28:41.624337: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| 2022-10-06 06:28:41.624824: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0 | |
| 2022-10-06 06:28:41.624877: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-10-06 06:28:42.267964: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix: | |
| 2022-10-06 06:28:42.268095: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267] 0 | |
| 2022-10-06 06:28:42.268119: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0: N | |
| 2022-10-06 06:28:42.269056: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10912 MB memory) -> physical GPU (device: 0, name: NVIDIA TITAN V, pci bus id: 0000:03:00.0, compute capability: 7.0) | |
| WARNING:tensorflow:No training configuration found in the save file, so the model was *not* compiled. Compile it manually. | |
| 2022-10-06 06:28:59.327084: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:116] None of the MLIR optimization passes are enabled (registered 2) | |
| 2022-10-06 06:28:59.327719: I tensorflow/core/platform/profile_utils/cpu_utils.cc:112] CPU Frequency: 2400240000 Hz | |
| 2022-10-06 06:28:59.607502: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-10-06 06:28:59.988305: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-10-06 06:28:59.990285: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| 2022-10-06 06:29:05.355649: W tensorflow/core/common_runtime/bfc_allocator.cc:248] Allocator (GPU_0_bfc) ran out of memory trying to allocate 3.96GiB with freed_by_count=0. The caller indicates that this is not a failure, but may mean that there could be performance gains if more memory were available. | |
| 2022-10-06 06:29:05.356193: W tensorflow/core/common_runtime/bfc_allocator.cc:248] Allocator (GPU_0_bfc) ran out of memory trying to allocate 3.96GiB with freed_by_count=0. The caller indicates that this is not a failure, but may mean that there could be performance gains if more memory were available. | |
| 2022-10-06 06:29:05.852809: W tensorflow/core/common_runtime/bfc_allocator.cc:248] Allocator (GPU_0_bfc) ran out of memory trying to allocate 3.77GiB with freed_by_count=0. The caller indicates that this is not a failure, but may mean that there could be performance gains if more memory were available. | |
| 2022-10-06 06:29:05.853316: W tensorflow/core/common_runtime/bfc_allocator.cc:248] Allocator (GPU_0_bfc) ran out of memory trying to allocate 3.77GiB with freed_by_count=0. The caller indicates that this is not a failure, but may mean that there could be performance gains if more memory were available. | |
| mkdir: cannot create directory ‘/scratch/groups/akundaje/anusri/chromatin_atlas/DNASE/ENCSR605SQL//chrombppnet_model_encsr283tme_bias_fold_1//footprints’: File exists | |
| 2022-10-06 06:32:04.597010: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-10-06 06:32:06.202678: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set | |
| 2022-10-06 06:32:06.203857: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1 | |
| 2022-10-06 06:32:06.246040: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: | |
| pciBusID: 0000:03:00.0 name: NVIDIA TITAN V computeCapability: 7.0 | |
| coreClock: 1.455GHz coreCount: 80 deviceMemorySize: 11.78GiB deviceMemoryBandwidth: 607.97GiB/s | |
| 2022-10-06 06:32:06.246155: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-10-06 06:32:06.249465: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-10-06 06:32:06.249546: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-10-06 06:32:06.250952: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10 | |
| 2022-10-06 06:32:06.251209: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10 | |
| 2022-10-06 06:32:06.254704: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10 | |
| 2022-10-06 06:32:06.255468: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11 | |
| 2022-10-06 06:32:06.255655: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| 2022-10-06 06:32:06.256340: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0 | |
| 2022-10-06 06:32:06.256718: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA | |
| To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. | |
| 2022-10-06 06:32:06.256817: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set | |
| 2022-10-06 06:32:06.257176: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: | |
| pciBusID: 0000:03:00.0 name: NVIDIA TITAN V computeCapability: 7.0 | |
| coreClock: 1.455GHz coreCount: 80 deviceMemorySize: 11.78GiB deviceMemoryBandwidth: 607.97GiB/s | |
| 2022-10-06 06:32:06.257220: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-10-06 06:32:06.257252: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-10-06 06:32:06.257295: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-10-06 06:32:06.257322: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10 | |
| 2022-10-06 06:32:06.257349: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10 | |
| 2022-10-06 06:32:06.257376: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10 | |
| 2022-10-06 06:32:06.257402: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11 | |
| 2022-10-06 06:32:06.257429: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| 2022-10-06 06:32:06.258781: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0 | |
| 2022-10-06 06:32:06.258846: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-10-06 06:32:06.925123: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix: | |
| 2022-10-06 06:32:06.925228: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267] 0 | |
| 2022-10-06 06:32:06.925249: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0: N | |
| 2022-10-06 06:32:06.926222: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10912 MB memory) -> physical GPU (device: 0, name: NVIDIA TITAN V, pci bus id: 0000:03:00.0, compute capability: 7.0) | |
| 2022-10-06 06:32:23.001002: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:116] None of the MLIR optimization passes are enabled (registered 2) | |
| 2022-10-06 06:32:23.001692: I tensorflow/core/platform/profile_utils/cpu_utils.cc:112] CPU Frequency: 2400240000 Hz | |
| 2022-10-06 06:32:23.194152: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-10-06 06:32:23.587528: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-10-06 06:32:23.589453: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |