diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..21bdeaf851be26bf4131b06c61f73dd2bab8ccca 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,11 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_0/progress.png filter=lfs diff=lfs merge=lfs -text +Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_1/progress.png filter=lfs diff=lfs merge=lfs -text +Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_2/progress.png filter=lfs diff=lfs merge=lfs -text +Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_3/progress.png filter=lfs diff=lfs merge=lfs -text +Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_4/progress.png filter=lfs diff=lfs merge=lfs -text +Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_0/progress.png filter=lfs diff=lfs merge=lfs -text +Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_1/progress.png filter=lfs diff=lfs merge=lfs -text +Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_2/progress.png filter=lfs diff=lfs merge=lfs -text diff --git a/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_0/debug.json b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_0/debug.json new file mode 100644 index 0000000000000000000000000000000000000000..e3c7ef2de7144a9772af1d9438902855559bf89c --- /dev/null +++ b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_0/debug.json @@ -0,0 +1,106 @@ +{ + "all_tr_losses": "[0.33703446, 0.245874, 0.20384128, 0.19027823, 0.17232056, 0.17565493, 0.15834498, 0.14658068, 0.15074384, 0.15878943, 0.16765323, 0.14644063, 0.14780703, 0.13994478, 0.14089847, 0.13259973, 0.1294894, 0.13168564, 0.13727094, 0.12322069, 0.1391203, 0.139793, 0.13415627, 0.13622817, 0.13856299, 0.12430216, 0.13102552, 0.12888575, 0.12994584, 0.12861799, 0.1136061, 0.12196406, 0.12626949, 0.13798071, 0.1246965, 0.12701035, 0.12746197, 0.1239965, 0.12073733, 0.11822609, 0.11616988, 0.118598886, 0.11856192, 0.119665414, 0.1165078, 0.12099173, 0.11922734, 0.11969286, 0.11703803, 0.115240365, 0.11297671, 0.114790976, 0.10922662, 0.11488171, 0.11453125, 0.10819345, 0.113893956, 0.106148824, 0.11596619, 0.10987344, 0.10946266, 0.11625536, 0.110896274, 0.11666065, 0.11128778, 0.10665962, 0.10384848, 0.10787832, 0.10700743, 0.105068035, 0.10964044, 0.10834642, 0.104676664, 0.10858993, 0.10325108, 0.106033444, 0.10648876, 0.10127316, 0.10753345, 0.103597306, 0.11094735, 0.104656205, 0.10037432, 0.10148334, 0.10345265, 0.10251114, 0.099607326, 0.10200581, 0.098598756, 0.10067512, 0.100201815, 0.09806329, 0.10444377]", + "all_val_eval_metrics": "[0.24931305466128356, 0.4266110160090153, 0.5166685259504604, 0.4423168852681748, 0.6139433036241648, 0.6516204582278422, 0.6189222721406125, 0.6413063344985869, 0.665092366906004, 0.6947147104919859, 0.6683744516164448, 0.6736381439436037, 0.700017381967438, 0.6730759539684324, 0.7154415276136535, 0.6694335787092658, 0.7192062580337659, 0.7044392301422628, 0.7385992185846558, 0.7161301051467699, 0.7369110132477408, 0.7564931665840715, 0.7050542082952844, 0.7383295048202763, 0.7363267486563524, 0.7474278933635006, 0.7232240493114174, 0.7471583706697802, 0.7589230466820907, 0.7424222985280224, 0.7350535327813896, 0.7524819191153387, 0.694568537220704, 0.7469446388453539, 0.7381758215499443, 0.737004824372373, 0.7473608181248375, 0.7227166179962762, 0.7555413616261274, 0.7406594892576732, 0.7364759507678018, 0.7484061102330988, 0.7489865782929418, 0.7502880666822354, 0.7605923448265312, 0.7649212228713784, 0.7506574964829025, 0.7653932620891901, 0.7627684916894186, 0.7362864575110082, 0.7246187154809203, 0.7573861141126148, 0.7646790277164011, 0.7577838008902644, 0.7546682915865979, 0.7727560666071309, 0.7416302187811665, 0.7694858066007401, 0.7446922180130513, 0.7420685030243155, 0.7684926564568911, 0.7650307492496151, 0.765980062045606, 0.7519805120045, 0.7735396243562005, 0.7754028667100326, 0.7694532069282431, 0.7717403203125063, 0.7643728398963558, 0.7773234921768635, 0.7689947898983847, 0.7713590280372293, 0.7743324021786255, 0.7604616566054231, 0.7711240265350421, 0.7652490678219319, 0.773581396919216, 0.769286730258646, 0.7612245709294823, 0.770080856652207, 0.7657448581798589, 0.7783318644337596, 0.7799934980280342, 0.7710235992523882, 0.7585932516756204, 0.7591866048825404, 0.7573924081938319, 0.775203482934095, 0.7557587434778792, 0.7669206932986474, 0.7655476706708516, 0.7701122568719798, 0.7673466319562363]", + "all_val_losses": "[0.25257468, 0.19878879, 0.17702252, 0.16008532, 0.16101126, 0.14179625, 0.13051416, 0.14159404, 0.1414402, 0.15901689, 0.12986815, 0.12475631, 0.12460776, 0.14963786, 0.12489662, 0.11893177, 0.12789702, 0.12574145, 0.14572573, 0.12154336, 0.13174886, 0.11971721, 0.123625204, 0.12817937, 0.13126977, 0.12630185, 0.12804723, 0.13088135, 0.120637454, 0.12525234, 0.12320873, 0.12474595, 0.12430963, 0.11961495, 0.119082205, 0.1295736, 0.114206515, 0.1255392, 0.12905027, 0.11628304, 0.11136678, 0.10540939, 0.12599255, 0.116712674, 0.118062906, 0.13245945, 0.120466255, 0.111738354, 0.13182618, 0.11221204, 0.12556745, 0.12311455, 0.13487303, 0.11514746, 0.11674822, 0.115926884, 0.09430425, 0.119221926, 0.11627844, 0.111686766, 0.101318166, 0.10261526, 0.127733, 0.118940614, 0.11854798, 0.10676561, 0.107232295, 0.09977675, 0.10480553, 0.100758284, 0.1097453, 0.11992353, 0.10713313, 0.10082048, 0.10904568, 0.118674114, 0.11127121, 0.098175794, 0.11891383, 0.111774474, 0.10249209, 0.11591384, 0.10567763, 0.10943615, 0.117287934, 0.13489087, 0.107407406, 0.09537521, 0.09355242, 0.10312289, 0.10185069, 0.110794134, 0.1276715]", + "all_val_losses_tr_mode": "[]", + "also_val_in_tr_mode": "False", + "amp_grad_scaler": "", + "args": "Namespace(batch_size=2, config='./configs/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32.yaml', continue_training=False, crop_size=[64, 192, 192], dbs=False, deep_supervision_scales=[], deterministic=False, disable_decoder=False, disable_ds=True, disable_postprocessing_on_folds=False, disable_saving=False, drop_path=0.0, find_lr=False, find_zero_weight_decay=False, fix_ds_net_numpool=False, fold='0', fp32=False, hdfs_base='GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32', initial_lr=0.0003, is_sigmoid=False, is_spatial_aug_only=False, is_ssl=False, layer_decay=1.0, local_rank=0, loss_name='', lrschedule='warmup_cosine', mask_ratio=0.75, max_loss_cal='v1', max_num_epochs=125, merge_femur=False, min_lr=0, model='Generic_TransUNet_max_ppbp', model_params={'is_masked_attn': True, 'max_dec_layers': 3, 'is_max_bottleneck_transformer': True, 'vit_depth': 1, 'max_msda': '', 'is_max_ms': True, 'max_ms_idxs': [-4, -3, -2], 'max_hidden_dim': 192, 'mw': 1.0, 'is_max_ds': True, 'is_masking': True, 'is_max_hungarian': True, 'num_queries': 20, 'is_max_cls': True, 'is_mhsa_float32': True}, n_class=17, network='3d_fullres', network_trainer='nnUNetTrainerV2_DDP', npz=False, opt_betas=None, opt_eps=1e-08, optim_name='adamw', p='nnUNetPlansv2.1', plan_update='', pretrained=False, pretrained_config=None, pretrained_weights=None, rank=0, reclip=None, resume='auto', skip_grad_nan=False, task='Task008_HepaticVessel', task_pretrained='Task801_WORD', total_batch_size=None, use_compressed_data=False, val_final=False, val_folder='validation_raw', valbest=False, validation_only=False, vallatest=False, warmup_epochs=10, weight_decay=3e-05, world_size=8)", + "base_num_features": "32", + "basic_generator_patch_size": "[178 308 233]", + "batch_dice": "True", + "batch_size": "2", + "best_MA_tr_loss_for_patience": "None", + "best_epoch_based_on_MA_tr_loss": "None", + "best_val_eval_criterion_MA": "None", + "classes": "[1, 2]", + "conv_per_stage": "2", + "data_aug_params": "{'selected_data_channels': None, 'selected_seg_channels': [0], 'do_elastic': False, 'elastic_deform_alpha': (0.0, 900.0), 'elastic_deform_sigma': (9.0, 13.0), 'p_eldef': 0.2, 'do_scaling': True, 'scale_range': (0.7, 1.4), 'independent_scale_factor_for_each_axis': False, 'p_independent_scale_per_axis': 1, 'p_scale': 0.2, 'do_rotation': True, 'rotation_x': (-0.5235987755982988, 0.5235987755982988), 'rotation_y': (-0.5235987755982988, 0.5235987755982988), 'rotation_z': (-0.5235987755982988, 0.5235987755982988), 'rotation_p_per_axis': 1, 'p_rot': 0.2, 'random_crop': False, 'random_crop_dist_to_border': None, 'do_gamma': True, 'gamma_retain_stats': True, 'gamma_range': (0.7, 1.5), 'p_gamma': 0.3, 'do_mirror': True, 'mirror_axes': (0, 1, 2), 'dummy_2D': False, 'mask_was_used_for_normalization': OrderedDict([(0, False)]), 'border_mode_data': 'constant', 'all_segmentation_labels': None, 'move_last_seg_chanel_to_data': False, 'cascade_do_cascade_augmentations': False, 'cascade_random_binary_transform_p': 0.4, 'cascade_random_binary_transform_p_per_label': 1, 'cascade_random_binary_transform_size': (1, 8), 'cascade_remove_conn_comp_p': 0.2, 'cascade_remove_conn_comp_max_size_percent_threshold': 0.15, 'cascade_remove_conn_comp_fill_with_other_class_p': 0.0, 'do_additive_brightness': False, 'additive_brightness_p_per_sample': 0.15, 'additive_brightness_p_per_channel': 0.5, 'additive_brightness_mu': 0.0, 'additive_brightness_sigma': 0.1, 'num_threads': 12, 'num_cached_per_thread': 2, 'patch_size_for_spatialtransform': array([ 64, 192, 192])}", + "dataset_directory": "/opt/tiger/project/data/nnUNet_preprocessed/Task008_HepaticVessel", + "deep_supervision_scales": "[[1, 1, 1], [0.5, 0.5, 0.5], [0.25, 0.25, 0.25], [0.125, 0.125, 0.125], [0.0625, 0.0625, 0.0625]]", + "deterministic": "False", + "disable_ds": "True", + "distribute_batch_size": "False", + "dl_tr": "", + "dl_val": "", + "do_dummy_2D_aug": "False", + "ds_loss_weights": "[1. 0. 0. 0. 0.]", + "epoch": "93", + "experiment_name": "nnUNetTrainerV2_DDP", + "find_zero_weight_decay": "False", + "fold": "0", + "folder_with_preprocessed_data": "/opt/tiger/project/data/nnUNet_preprocessed/Task008_HepaticVessel/nnUNetData_plans_v2.1_stage1", + "fp16": "True", + "global_batch_size": "16", + "gt_niftis_folder": "/opt/tiger/project/data/nnUNet_preprocessed/Task008_HepaticVessel/gt_segmentations", + "inference_pad_border_mode": "constant", + "inference_pad_kwargs": "{'constant_values': 0}", + "init_args": "('/opt/tiger/project/data/nnUNet_preprocessed/Task008_HepaticVessel/nnUNetPlansv2.1_plans_3D.pkl', 0, 0, '/opt/tiger/project/data/nnUNet_trained_models/UNet_IN_NANFang/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32', '/opt/tiger/project/data/nnUNet_preprocessed/Task008_HepaticVessel', True, 1, True, False, False, True)", + "initial_lr": "0.0003", + "input_size": "[64, 192, 192]", + "is_spatial_aug_only": "False", + "is_ssl": "False", + "layer_decay": "1.0", + "local_rank": "0", + "log_file": "/opt/tiger/project/data/nnUNet_trained_models/UNet_IN_NANFang/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_0/training_log_2022_5_19_23_48_51.txt", + "loss": "None", + "lr_scheduler": "", + "lr_scheduler_eps": "0.001", + "lr_scheduler_name": "warmup_cosine", + "lr_scheduler_patience": "30", + "lr_threshold": "1e-06", + "max_num_epochs": "125", + "min_lr": "0", + "min_region_size_per_class": "None", + "min_size_per_class": "None", + "model": "Generic_TransUNet_max_ppbp", + "model_params": "{'is_masked_attn': True, 'max_dec_layers': 3, 'is_max_bottleneck_transformer': True, 'vit_depth': 1, 'max_msda': '', 'is_max_ms': True, 'max_ms_idxs': [-4, -3, -2], 'max_hidden_dim': 192, 'mw': 1.0, 'is_max_ds': True, 'is_masking': True, 'is_max_hungarian': True, 'num_queries': 20, 'is_max_cls': True, 'is_mhsa_float32': True}", + "net_conv_kernel_sizes": "[[3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3]]", + "net_num_pool_op_kernel_sizes": "[[2, 2, 2], [2, 2, 2], [2, 2, 2], [2, 2, 2], [1, 2, 2]]", + "net_pool_per_axis": "[4, 5, 5]", + "normalization_schemes": "OrderedDict([(0, 'CT')])", + "num_batches_per_epoch": "250", + "num_classes": "3", + "num_input_channels": "1", + "num_val_batches_per_epoch": "50", + "online_eval_fn": "[]", + "online_eval_foreground_dc": "[]", + "online_eval_fp": "[]", + "online_eval_tp": "[]", + "only_keep_largest_connected_component": "None", + "optim_name": "adamw", + "optimizer": "AdamW (\nParameter Group 0\n amsgrad: False\n betas: (0.9, 0.999)\n eps: 1e-08\n initial_lr: 0.0003\n lr: 5.375636079717583e-05\n weight_decay: 3e-05\n)", + "output_folder": "/opt/tiger/project/data/nnUNet_trained_models/UNet_IN_NANFang/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_0", + "output_folder_base": "/opt/tiger/project/data/nnUNet_trained_models/UNet_IN_NANFang/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32", + "oversample_foreground_percent": "0.0", + "pad_all_sides": "None", + "patch_size": "[ 64 192 192]", + "patience": "50", + "pin_memory": "True", + "plans_file": "/opt/tiger/project/data/nnUNet_preprocessed/Task008_HepaticVessel/nnUNetPlansv2.1_plans_3D.pkl", + "reclip": "None", + "regions_class_order": "None", + "resume": "auto", + "save_best_checkpoint": "True", + "save_every": "1", + "save_final_checkpoint": "True", + "save_intermediate_checkpoints": "True", + "save_latest_only": "True", + "stage": "1", + "threeD": "True", + "tr_gen": "", + "train_loss_MA": "None", + "train_loss_MA_alpha": "0.93", + "train_loss_MA_eps": "0.0005", + "transpose_backward": "[0, 1, 2]", + "transpose_forward": "[0, 1, 2]", + "unpack_data": "True", + "use_mask_for_norm": "OrderedDict([(0, False)])", + "use_progress_bar": "False", + "val_eval_criterion_MA": "None", + "val_eval_criterion_alpha": "0.9", + "val_gen": "", + "warmup_epochs": "10", + "was_initialized": "True", + "weight_decay": "3e-05" +} \ No newline at end of file diff --git a/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_0/model_best.model b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_0/model_best.model new file mode 100644 index 0000000000000000000000000000000000000000..a8751214144575d97504a6667728985c301eed9f --- /dev/null +++ b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_0/model_best.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:acc3c382ff646fa41dd75eeea3490c0b929199e3b52533a7c4d5144c7e6ee02e +size 496527929 diff --git a/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_0/model_best.model.pkl b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_0/model_best.model.pkl new file mode 100644 index 0000000000000000000000000000000000000000..7ddd2b311ebcd7421c91feb26cf26a1875ec99f2 --- /dev/null +++ b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_0/model_best.model.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a7e84f1583a7b2b7d0ff43ee2d2c257d6c2589715299940df967ac18db1196c6 +size 188932 diff --git a/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_0/model_final_checkpoint.model b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_0/model_final_checkpoint.model new file mode 100644 index 0000000000000000000000000000000000000000..f023023fece177cea1a71b64e302aaa91999e2d9 --- /dev/null +++ b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_0/model_final_checkpoint.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8ca558721b5616594b3c4006cfc12be201310d5270b80fbbb903fa62dde0d575 +size 496529594 diff --git a/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_0/model_final_checkpoint.model.pkl b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_0/model_final_checkpoint.model.pkl new file mode 100644 index 0000000000000000000000000000000000000000..7ddd2b311ebcd7421c91feb26cf26a1875ec99f2 --- /dev/null +++ b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_0/model_final_checkpoint.model.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a7e84f1583a7b2b7d0ff43ee2d2c257d6c2589715299940df967ac18db1196c6 +size 188932 diff --git a/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_0/model_latest.model b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_0/model_latest.model new file mode 100644 index 0000000000000000000000000000000000000000..c014ac1aaab5887e8ef7413c363b19960130e5be --- /dev/null +++ b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_0/model_latest.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:528cc64b7f13d252ed7709c0a8761b69327ba81eaf89d5dc83248fd11f63a4ed +size 496529594 diff --git a/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_0/model_latest.model.pkl b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_0/model_latest.model.pkl new file mode 100644 index 0000000000000000000000000000000000000000..7ddd2b311ebcd7421c91feb26cf26a1875ec99f2 --- /dev/null +++ b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_0/model_latest.model.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a7e84f1583a7b2b7d0ff43ee2d2c257d6c2589715299940df967ac18db1196c6 +size 188932 diff --git a/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_0/progress.png b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_0/progress.png new file mode 100644 index 0000000000000000000000000000000000000000..0be16c504e68e5cc78b269de4987543682a0cc59 --- /dev/null +++ b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_0/progress.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29583d8a4ff47345d209fe1a2ccea10cfc0345c0f3cf9de2c9519c0fc63ca4cb +size 283872 diff --git a/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_0/training_log_2022_5_19_23_48_51.txt b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_0/training_log_2022_5_19_23_48_51.txt new file mode 100644 index 0000000000000000000000000000000000000000..4319b6bd1d83256396eba1ec1871f95abb3ed160 --- /dev/null +++ b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_0/training_log_2022_5_19_23_48_51.txt @@ -0,0 +1,459 @@ +Starting... +2022-05-19 23:48:51.257375: Using splits from existing split file: /opt/tiger/project/data/nnUNet_preprocessed/Task008_HepaticVessel/splits_final.pkl +2022-05-19 23:48:51.257992: The split file contains 5 splits. +2022-05-19 23:48:51.258045: Desired fold for training: 0 +2022-05-19 23:48:51.258093: This split has 242 training and 61 validation cases. +2022-05-19 23:50:29.435884: TRAINING KEYS: + odict_keys(['hepaticvessel_001', 'hepaticvessel_002', 'hepaticvessel_004', 'hepaticvessel_005', 'hepaticvessel_007', 'hepaticvessel_008', 'hepaticvessel_010', 'hepaticvessel_011', 'hepaticvessel_013', 'hepaticvessel_016', 'hepaticvessel_020', 'hepaticvessel_022', 'hepaticvessel_023', 'hepaticvessel_025', 'hepaticvessel_027', 'hepaticvessel_029', 'hepaticvessel_031', 'hepaticvessel_032', 'hepaticvessel_033', 'hepaticvessel_039', 'hepaticvessel_040', 'hepaticvessel_042', 'hepaticvessel_044', 'hepaticvessel_046', 'hepaticvessel_049', 'hepaticvessel_051', 'hepaticvessel_052', 'hepaticvessel_057', 'hepaticvessel_059', 'hepaticvessel_061', 'hepaticvessel_062', 'hepaticvessel_065', 'hepaticvessel_067', 'hepaticvessel_068', 'hepaticvessel_070', 'hepaticvessel_071', 'hepaticvessel_072', 'hepaticvessel_075', 'hepaticvessel_078', 'hepaticvessel_079', 'hepaticvessel_081', 'hepaticvessel_083', 'hepaticvessel_086', 'hepaticvessel_088', 'hepaticvessel_089', 'hepaticvessel_090', 'hepaticvessel_091', 'hepaticvessel_092', 'hepaticvessel_093', 'hepaticvessel_094', 'hepaticvessel_096', 'hepaticvessel_097', 'hepaticvessel_102', 'hepaticvessel_103', 'hepaticvessel_104', 'hepaticvessel_108', 'hepaticvessel_111', 'hepaticvessel_113', 'hepaticvessel_115', 'hepaticvessel_116', 'hepaticvessel_117', 'hepaticvessel_119', 'hepaticvessel_121', 'hepaticvessel_123', 'hepaticvessel_124', 'hepaticvessel_125', 'hepaticvessel_128', 'hepaticvessel_131', 'hepaticvessel_132', 'hepaticvessel_133', 'hepaticvessel_136', 'hepaticvessel_137', 'hepaticvessel_138', 'hepaticvessel_139', 'hepaticvessel_140', 'hepaticvessel_141', 'hepaticvessel_142', 'hepaticvessel_144', 'hepaticvessel_145', 'hepaticvessel_148', 'hepaticvessel_151', 'hepaticvessel_154', 'hepaticvessel_158', 'hepaticvessel_159', 'hepaticvessel_160', 'hepaticvessel_163', 'hepaticvessel_165', 'hepaticvessel_166', 'hepaticvessel_171', 'hepaticvessel_172', 'hepaticvessel_173', 'hepaticvessel_174', 'hepaticvessel_177', 'hepaticvessel_178', 'hepaticvessel_179', 'hepaticvessel_181', 'hepaticvessel_183', 'hepaticvessel_184', 'hepaticvessel_185', 'hepaticvessel_186', 'hepaticvessel_189', 'hepaticvessel_192', 'hepaticvessel_193', 'hepaticvessel_195', 'hepaticvessel_196', 'hepaticvessel_197', 'hepaticvessel_198', 'hepaticvessel_199', 'hepaticvessel_200', 'hepaticvessel_201', 'hepaticvessel_203', 'hepaticvessel_207', 'hepaticvessel_208', 'hepaticvessel_209', 'hepaticvessel_210', 'hepaticvessel_211', 'hepaticvessel_214', 'hepaticvessel_215', 'hepaticvessel_219', 'hepaticvessel_221', 'hepaticvessel_222', 'hepaticvessel_223', 'hepaticvessel_225', 'hepaticvessel_227', 'hepaticvessel_229', 'hepaticvessel_230', 'hepaticvessel_231', 'hepaticvessel_233', 'hepaticvessel_235', 'hepaticvessel_236', 'hepaticvessel_237', 'hepaticvessel_238', 'hepaticvessel_240', 'hepaticvessel_242', 'hepaticvessel_245', 'hepaticvessel_246', 'hepaticvessel_255', 'hepaticvessel_256', 'hepaticvessel_258', 'hepaticvessel_259', 'hepaticvessel_262', 'hepaticvessel_263', 'hepaticvessel_265', 'hepaticvessel_268', 'hepaticvessel_271', 'hepaticvessel_278', 'hepaticvessel_279', 'hepaticvessel_280', 'hepaticvessel_281', 'hepaticvessel_284', 'hepaticvessel_285', 'hepaticvessel_286', 'hepaticvessel_287', 'hepaticvessel_289', 'hepaticvessel_290', 'hepaticvessel_291', 'hepaticvessel_293', 'hepaticvessel_294', 'hepaticvessel_296', 'hepaticvessel_297', 'hepaticvessel_299', 'hepaticvessel_300', 'hepaticvessel_303', 'hepaticvessel_305', 'hepaticvessel_307', 'hepaticvessel_308', 'hepaticvessel_314', 'hepaticvessel_318', 'hepaticvessel_319', 'hepaticvessel_320', 'hepaticvessel_321', 'hepaticvessel_322', 'hepaticvessel_323', 'hepaticvessel_324', 'hepaticvessel_325', 'hepaticvessel_327', 'hepaticvessel_330', 'hepaticvessel_332', 'hepaticvessel_335', 'hepaticvessel_336', 'hepaticvessel_338', 'hepaticvessel_340', 'hepaticvessel_341', 'hepaticvessel_344', 'hepaticvessel_345', 'hepaticvessel_348', 'hepaticvessel_349', 'hepaticvessel_350', 'hepaticvessel_353', 'hepaticvessel_356', 'hepaticvessel_362', 'hepaticvessel_367', 'hepaticvessel_369', 'hepaticvessel_371', 'hepaticvessel_372', 'hepaticvessel_373', 'hepaticvessel_374', 'hepaticvessel_378', 'hepaticvessel_379', 'hepaticvessel_382', 'hepaticvessel_383', 'hepaticvessel_384', 'hepaticvessel_385', 'hepaticvessel_386', 'hepaticvessel_388', 'hepaticvessel_389', 'hepaticvessel_391', 'hepaticvessel_393', 'hepaticvessel_396', 'hepaticvessel_397', 'hepaticvessel_398', 'hepaticvessel_400', 'hepaticvessel_401', 'hepaticvessel_402', 'hepaticvessel_404', 'hepaticvessel_406', 'hepaticvessel_407', 'hepaticvessel_408', 'hepaticvessel_409', 'hepaticvessel_411', 'hepaticvessel_412', 'hepaticvessel_416', 'hepaticvessel_419', 'hepaticvessel_420', 'hepaticvessel_423', 'hepaticvessel_425', 'hepaticvessel_427', 'hepaticvessel_429', 'hepaticvessel_431', 'hepaticvessel_432', 'hepaticvessel_433', 'hepaticvessel_434', 'hepaticvessel_438', 'hepaticvessel_440', 'hepaticvessel_442', 'hepaticvessel_443', 'hepaticvessel_446', 'hepaticvessel_447', 'hepaticvessel_453', 'hepaticvessel_454', 'hepaticvessel_455', 'hepaticvessel_456']) +2022-05-19 23:50:29.436077: VALIDATION KEYS: + odict_keys(['hepaticvessel_018', 'hepaticvessel_019', 'hepaticvessel_026', 'hepaticvessel_028', 'hepaticvessel_030', 'hepaticvessel_050', 'hepaticvessel_053', 'hepaticvessel_058', 'hepaticvessel_066', 'hepaticvessel_077', 'hepaticvessel_080', 'hepaticvessel_082', 'hepaticvessel_084', 'hepaticvessel_085', 'hepaticvessel_087', 'hepaticvessel_098', 'hepaticvessel_100', 'hepaticvessel_110', 'hepaticvessel_112', 'hepaticvessel_127', 'hepaticvessel_129', 'hepaticvessel_146', 'hepaticvessel_147', 'hepaticvessel_150', 'hepaticvessel_157', 'hepaticvessel_161', 'hepaticvessel_164', 'hepaticvessel_167', 'hepaticvessel_175', 'hepaticvessel_194', 'hepaticvessel_206', 'hepaticvessel_213', 'hepaticvessel_217', 'hepaticvessel_218', 'hepaticvessel_224', 'hepaticvessel_234', 'hepaticvessel_244', 'hepaticvessel_248', 'hepaticvessel_266', 'hepaticvessel_269', 'hepaticvessel_270', 'hepaticvessel_272', 'hepaticvessel_274', 'hepaticvessel_275', 'hepaticvessel_282', 'hepaticvessel_309', 'hepaticvessel_329', 'hepaticvessel_333', 'hepaticvessel_358', 'hepaticvessel_359', 'hepaticvessel_361', 'hepaticvessel_363', 'hepaticvessel_368', 'hepaticvessel_375', 'hepaticvessel_422', 'hepaticvessel_424', 'hepaticvessel_437', 'hepaticvessel_441', 'hepaticvessel_444', 'hepaticvessel_445', 'hepaticvessel_458']) +2022-05-19 23:50:39.562442: loading checkpoint /opt/tiger/project/data/nnUNet_trained_models/UNet_IN_NANFang/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_0/model_latest.model train= True +2022-05-19 23:50:40.782292: lr: 5.1e-05 +2022-05-19 23:50:54.898852: +epoch: 93 +2022-05-19 23:58:50.384474: train loss : 0.1051 +2022-05-19 23:59:16.686494: validation loss: 0.1173 +2022-05-19 23:59:16.687123: Average global foreground Dice: [0.6786, 0.8481] +2022-05-19 23:59:16.687212: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-19 23:59:17.112055: lr: 5.1e-05 +2022-05-19 23:59:17.112260: saving scheduled checkpoint file... +2022-05-19 23:59:17.207761: saving checkpoint... +2022-05-19 23:59:18.036854: done, saving took 0.92 seconds +2022-05-19 23:59:18.044720: done +2022-05-19 23:59:32.595264: This epoch took 517.695660 s + +2022-05-19 23:59:32.595533: +epoch: 94 +2022-05-20 00:07:15.572337: train loss : 0.1075 +2022-05-20 00:07:41.835408: validation loss: 0.0935 +2022-05-20 00:07:41.835972: Average global foreground Dice: [0.6847, 0.8776] +2022-05-20 00:07:41.836137: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-20 00:07:42.239912: lr: 4.8e-05 +2022-05-20 00:07:42.240106: saving scheduled checkpoint file... +2022-05-20 00:07:42.298090: saving checkpoint... +2022-05-20 00:07:43.182063: done, saving took 0.94 seconds +2022-05-20 00:07:43.191494: done +2022-05-20 00:07:43.231501: saving checkpoint... +2022-05-20 00:07:43.832907: done, saving took 0.64 seconds +2022-05-20 00:08:05.325558: This epoch took 512.729770 s + +2022-05-20 00:08:05.325897: +epoch: 95 +2022-05-20 00:15:47.981698: train loss : 0.0986 +2022-05-20 00:16:14.300343: validation loss: 0.1020 +2022-05-20 00:16:14.300973: Average global foreground Dice: [0.6792, 0.8661] +2022-05-20 00:16:14.301059: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-20 00:16:14.702829: lr: 4.5e-05 +2022-05-20 00:16:14.703036: saving scheduled checkpoint file... +2022-05-20 00:16:14.761148: saving checkpoint... +2022-05-20 00:16:15.674159: done, saving took 0.97 seconds +2022-05-20 00:16:15.684278: done +2022-05-20 00:16:15.725320: saving checkpoint... +2022-05-20 00:16:16.609183: done, saving took 0.92 seconds +2022-05-20 00:17:02.944608: This epoch took 537.618602 s + +2022-05-20 00:17:02.944928: +epoch: 96 +2022-05-20 00:24:45.448443: train loss : 0.1029 +2022-05-20 00:25:11.735332: validation loss: 0.1024 +2022-05-20 00:25:11.735900: Average global foreground Dice: [0.6728, 0.8555] +2022-05-20 00:25:11.735984: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-20 00:25:12.136425: lr: 4.2e-05 +2022-05-20 00:25:12.136624: saving scheduled checkpoint file... +2022-05-20 00:25:12.193799: saving checkpoint... +2022-05-20 00:25:13.058001: done, saving took 0.92 seconds +2022-05-20 00:25:13.066621: done +2022-05-20 00:25:36.029841: This epoch took 513.084797 s + +2022-05-20 00:25:36.030130: +epoch: 97 +2022-05-20 00:33:18.608969: train loss : 0.0923 +2022-05-20 00:33:44.970985: validation loss: 0.1178 +2022-05-20 00:33:44.971601: Average global foreground Dice: [0.6633, 0.8634] +2022-05-20 00:33:44.971686: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-20 00:33:45.375843: lr: 3.9e-05 +2022-05-20 00:33:45.376031: saving scheduled checkpoint file... +2022-05-20 00:33:45.433744: saving checkpoint... +2022-05-20 00:33:46.338244: done, saving took 0.96 seconds +2022-05-20 00:33:46.347102: done +2022-05-20 00:34:15.189025: This epoch took 519.158788 s + +2022-05-20 00:34:15.189291: +epoch: 98 +2022-05-20 00:41:58.007195: train loss : 0.0986 +2022-05-20 00:42:24.339576: validation loss: 0.1003 +2022-05-20 00:42:24.340255: Average global foreground Dice: [0.6778, 0.8453] +2022-05-20 00:42:24.340338: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-20 00:42:24.743111: lr: 3.6e-05 +2022-05-20 00:42:24.743304: saving scheduled checkpoint file... +2022-05-20 00:42:24.806367: saving checkpoint... +2022-05-20 00:42:25.714694: done, saving took 0.97 seconds +2022-05-20 00:42:25.724246: done +2022-05-20 00:42:50.617247: This epoch took 515.427840 s + +2022-05-20 00:42:50.617600: +epoch: 99 +2022-05-20 00:50:33.546897: train loss : 0.0886 +2022-05-20 00:50:59.868541: validation loss: 0.0985 +2022-05-20 00:50:59.869164: Average global foreground Dice: [0.6767, 0.8476] +2022-05-20 00:50:59.869246: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-20 00:51:00.272167: lr: 3.4e-05 +2022-05-20 00:51:00.272364: saving scheduled checkpoint file... +2022-05-20 00:51:00.334502: saving checkpoint... +2022-05-20 00:51:01.176983: done, saving took 0.90 seconds +2022-05-20 00:51:01.185279: done +2022-05-20 00:51:28.496241: This epoch took 517.878529 s + +2022-05-20 00:51:28.496536: +epoch: 100 +2022-05-20 00:59:11.134774: train loss : 0.0890 +2022-05-20 00:59:37.465693: validation loss: 0.1076 +2022-05-20 00:59:37.466276: Average global foreground Dice: [0.6876, 0.8667] +2022-05-20 00:59:37.466359: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-20 00:59:37.874352: lr: 3.1e-05 +2022-05-20 00:59:37.874558: saving scheduled checkpoint file... +2022-05-20 00:59:37.932466: saving checkpoint... +2022-05-20 00:59:38.848706: done, saving took 0.97 seconds +2022-05-20 00:59:38.857605: done +2022-05-20 00:59:38.894561: saving checkpoint... +2022-05-20 00:59:39.768356: done, saving took 0.91 seconds +2022-05-20 01:00:17.246230: This epoch took 528.749581 s + +2022-05-20 01:00:17.246567: +epoch: 101 +2022-05-20 01:07:59.731180: train loss : 0.0926 +2022-05-20 01:08:26.038965: validation loss: 0.1043 +2022-05-20 01:08:26.039565: Average global foreground Dice: [0.6687, 0.8613] +2022-05-20 01:08:26.039651: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-20 01:08:26.511847: lr: 2.9e-05 +2022-05-20 01:08:26.512047: saving scheduled checkpoint file... +2022-05-20 01:08:26.569945: saving checkpoint... +2022-05-20 01:08:27.485320: done, saving took 0.97 seconds +2022-05-20 01:08:27.495139: done +2022-05-20 01:08:49.330272: This epoch took 512.083585 s + +2022-05-20 01:08:49.330897: +epoch: 102 +2022-05-20 01:16:31.882948: train loss : 0.0988 +2022-05-20 01:16:58.200999: validation loss: 0.1163 +2022-05-20 01:16:58.201947: Average global foreground Dice: [0.6942, 0.8482] +2022-05-20 01:16:58.202044: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-20 01:16:58.609138: lr: 2.6e-05 +2022-05-20 01:16:58.609334: saving scheduled checkpoint file... +2022-05-20 01:16:58.666744: saving checkpoint... +2022-05-20 01:16:59.502350: done, saving took 0.89 seconds +2022-05-20 01:16:59.511892: done +2022-05-20 01:16:59.557263: saving checkpoint... +2022-05-20 01:17:00.325649: done, saving took 0.81 seconds +2022-05-20 01:19:55.707306: This epoch took 666.376227 s + +2022-05-20 01:19:55.707617: +epoch: 103 +2022-05-20 01:27:38.293492: train loss : 0.0893 +2022-05-20 01:28:04.627843: validation loss: 0.1044 +2022-05-20 01:28:04.628515: Average global foreground Dice: [0.6755, 0.8558] +2022-05-20 01:28:04.628602: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-20 01:28:05.030528: lr: 2.4e-05 +2022-05-20 01:28:05.030717: saving scheduled checkpoint file... +2022-05-20 01:28:05.088472: saving checkpoint... +2022-05-20 01:28:05.999219: done, saving took 0.97 seconds +2022-05-20 01:28:06.007967: done +2022-05-20 01:29:01.742134: This epoch took 546.034411 s + +2022-05-20 01:29:01.742395: +epoch: 104 +2022-05-20 01:36:44.285238: train loss : 0.0977 +2022-05-20 01:37:10.613660: validation loss: 0.1055 +2022-05-20 01:37:10.614282: Average global foreground Dice: [0.6763, 0.8733] +2022-05-20 01:37:10.614366: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-20 01:37:11.165598: lr: 2.2e-05 +2022-05-20 01:37:11.165834: saving scheduled checkpoint file... +2022-05-20 01:37:11.228286: saving checkpoint... +2022-05-20 01:37:12.072917: done, saving took 0.91 seconds +2022-05-20 01:37:12.082669: done +2022-05-20 01:37:12.119464: saving checkpoint... +2022-05-20 01:37:12.945687: done, saving took 0.86 seconds +2022-05-20 01:37:37.482560: This epoch took 515.740038 s + +2022-05-20 01:37:37.482852: +epoch: 105 +2022-05-20 01:45:20.472046: train loss : 0.1015 +2022-05-20 01:45:46.792297: validation loss: 0.1087 +2022-05-20 01:45:46.793066: Average global foreground Dice: [0.6842, 0.8695] +2022-05-20 01:45:46.793154: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-20 01:45:47.268014: lr: 2e-05 +2022-05-20 01:45:47.268207: saving scheduled checkpoint file... +2022-05-20 01:45:47.325853: saving checkpoint... +2022-05-20 01:45:48.240048: done, saving took 0.97 seconds +2022-05-20 01:45:48.249109: done +2022-05-20 01:45:48.285815: saving checkpoint... +2022-05-20 01:45:49.126188: done, saving took 0.88 seconds +2022-05-20 01:48:16.299839: This epoch took 638.816876 s + +2022-05-20 01:48:16.300112: +epoch: 106 +2022-05-20 01:55:59.132738: train loss : 0.0985 +2022-05-20 01:56:25.476089: validation loss: 0.0985 +2022-05-20 01:56:25.476704: Average global foreground Dice: [0.6772, 0.8671] +2022-05-20 01:56:25.476788: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-20 01:56:25.884570: lr: 1.8e-05 +2022-05-20 01:56:25.884767: saving scheduled checkpoint file... +2022-05-20 01:56:25.942616: saving checkpoint... +2022-05-20 01:56:26.860924: done, saving took 0.98 seconds +2022-05-20 01:56:26.870733: done +2022-05-20 01:56:26.920466: saving checkpoint... +2022-05-20 01:56:27.852342: done, saving took 0.98 seconds +2022-05-20 01:56:47.607734: This epoch took 511.307517 s + +2022-05-20 01:56:47.608024: +epoch: 107 +2022-05-20 02:04:31.711817: train loss : 0.1004 +2022-05-20 02:04:58.039280: validation loss: 0.1094 +2022-05-20 02:04:58.039928: Average global foreground Dice: [0.6832, 0.8575] +2022-05-20 02:04:58.040011: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-20 02:04:58.442301: lr: 1.6e-05 +2022-05-20 02:04:58.442496: saving scheduled checkpoint file... +2022-05-20 02:04:58.500450: saving checkpoint... +2022-05-20 02:04:59.355402: done, saving took 0.91 seconds +2022-05-20 02:04:59.364923: done +2022-05-20 02:04:59.404097: saving checkpoint... +2022-05-20 02:05:00.236540: done, saving took 0.87 seconds +2022-05-20 02:05:19.533822: This epoch took 511.925693 s + +2022-05-20 02:05:19.534085: +epoch: 108 +2022-05-20 02:13:02.235750: train loss : 0.1050 +2022-05-20 02:13:28.574315: validation loss: 0.1035 +2022-05-20 02:13:28.575045: Average global foreground Dice: [0.6736, 0.8646] +2022-05-20 02:13:28.575130: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-20 02:13:28.977365: lr: 1.4e-05 +2022-05-20 02:13:28.977578: saving scheduled checkpoint file... +2022-05-20 02:13:29.035761: saving checkpoint... +2022-05-20 02:13:29.966818: done, saving took 0.99 seconds +2022-05-20 02:13:29.975782: done +2022-05-20 02:13:30.013183: saving checkpoint... +2022-05-20 02:13:30.904726: done, saving took 0.93 seconds +2022-05-20 02:14:20.017928: This epoch took 540.483616 s + +2022-05-20 02:14:20.018280: +epoch: 109 +2022-05-20 02:22:02.996914: train loss : 0.0922 +2022-05-20 02:22:29.338303: validation loss: 0.1069 +2022-05-20 02:22:29.338900: Average global foreground Dice: [0.6791, 0.8653] +2022-05-20 02:22:29.338984: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-20 02:22:29.741534: lr: 1.2e-05 +2022-05-20 02:22:29.741723: saving scheduled checkpoint file... +2022-05-20 02:22:29.799325: saving checkpoint... +2022-05-20 02:22:30.730340: done, saving took 0.99 seconds +2022-05-20 02:22:30.753898: done +2022-05-20 02:22:30.791040: saving checkpoint... +2022-05-20 02:22:31.693027: done, saving took 0.94 seconds +2022-05-20 02:23:19.686598: This epoch took 539.668195 s + +2022-05-20 02:23:19.686899: +epoch: 110 +2022-05-20 02:31:02.481437: train loss : 0.0976 +2022-05-20 02:31:28.836215: validation loss: 0.1074 +2022-05-20 02:31:28.836871: Average global foreground Dice: [0.6729, 0.8662] +2022-05-20 02:31:28.836956: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-20 02:31:29.247230: lr: 1.1e-05 +2022-05-20 02:31:29.247439: saving scheduled checkpoint file... +2022-05-20 02:31:29.305346: saving checkpoint... +2022-05-20 02:31:30.255769: done, saving took 1.01 seconds +2022-05-20 02:31:30.264834: done +2022-05-20 02:31:30.301665: saving checkpoint... +2022-05-20 02:31:31.221288: done, saving took 0.96 seconds +2022-05-20 02:31:59.297751: This epoch took 519.610735 s + +2022-05-20 02:31:59.298295: +epoch: 111 +2022-05-20 02:39:42.628542: train loss : 0.0952 +2022-05-20 02:40:08.969127: validation loss: 0.1235 +2022-05-20 02:40:08.969854: Average global foreground Dice: [0.6641, 0.8732] +2022-05-20 02:40:08.969939: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-20 02:40:09.375850: lr: 9e-06 +2022-05-20 02:40:09.376051: saving scheduled checkpoint file... +2022-05-20 02:40:09.433630: saving checkpoint... +2022-05-20 02:40:10.268741: done, saving took 0.89 seconds +2022-05-20 02:40:10.281483: done +2022-05-20 02:40:44.673769: This epoch took 525.375242 s + +2022-05-20 02:40:44.674232: +epoch: 112 +2022-05-20 02:48:27.491620: train loss : 0.0917 +2022-05-20 02:48:53.825438: validation loss: 0.1077 +2022-05-20 02:48:53.826009: Average global foreground Dice: [0.6779, 0.8763] +2022-05-20 02:48:53.826092: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-20 02:48:54.231390: lr: 8e-06 +2022-05-20 02:48:54.231585: saving scheduled checkpoint file... +2022-05-20 02:48:54.288993: saving checkpoint... +2022-05-20 02:48:55.167226: done, saving took 0.94 seconds +2022-05-20 02:48:55.175916: done +2022-05-20 02:48:55.211892: saving checkpoint... +2022-05-20 02:48:56.000624: done, saving took 0.82 seconds +2022-05-20 02:49:16.572285: This epoch took 511.897707 s + +2022-05-20 02:49:16.572624: +epoch: 113 +2022-05-20 02:56:59.628493: train loss : 0.0952 +2022-05-20 02:57:25.966178: validation loss: 0.1075 +2022-05-20 02:57:25.966736: Average global foreground Dice: [0.679, 0.8666] +2022-05-20 02:57:25.966820: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-20 02:57:26.374386: lr: 7e-06 +2022-05-20 02:57:26.374579: saving scheduled checkpoint file... +2022-05-20 02:57:26.432114: saving checkpoint... +2022-05-20 02:57:27.399751: done, saving took 1.03 seconds +2022-05-20 02:57:27.409125: done +2022-05-20 02:57:27.446420: saving checkpoint... +2022-05-20 02:57:28.369764: done, saving took 0.96 seconds +2022-05-20 02:57:52.518135: This epoch took 515.945405 s + +2022-05-20 02:57:52.518406: +epoch: 114 +2022-05-20 03:05:35.819391: train loss : 0.0866 +2022-05-20 03:06:02.155639: validation loss: 0.1065 +2022-05-20 03:06:02.156239: Average global foreground Dice: [0.6804, 0.8632] +2022-05-20 03:06:02.156324: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-20 03:06:02.561257: lr: 6e-06 +2022-05-20 03:06:02.561461: saving scheduled checkpoint file... +2022-05-20 03:06:02.619571: saving checkpoint... +2022-05-20 03:06:03.557004: done, saving took 1.00 seconds +2022-05-20 03:06:03.566626: done +2022-05-20 03:06:03.607573: saving checkpoint... +2022-05-20 03:06:04.532976: done, saving took 0.97 seconds +2022-05-20 03:06:36.565466: This epoch took 524.046920 s + +2022-05-20 03:06:36.565741: +epoch: 115 +2022-05-20 03:14:19.249373: train loss : 0.0968 +2022-05-20 03:14:45.599937: validation loss: 0.1125 +2022-05-20 03:14:45.600540: Average global foreground Dice: [0.6752, 0.8804] +2022-05-20 03:14:45.600625: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-20 03:14:46.007247: lr: 5e-06 +2022-05-20 03:14:46.007459: saving scheduled checkpoint file... +2022-05-20 03:14:46.064964: saving checkpoint... +2022-05-20 03:14:47.052503: done, saving took 1.04 seconds +2022-05-20 03:14:47.062558: done +2022-05-20 03:14:47.103672: saving checkpoint... +2022-05-20 03:14:48.029160: done, saving took 0.97 seconds +2022-05-20 03:15:08.900486: This epoch took 512.334484 s + +2022-05-20 03:15:08.900937: +epoch: 116 +2022-05-20 03:22:51.727525: train loss : 0.0989 +2022-05-20 03:23:18.064680: validation loss: 0.1056 +2022-05-20 03:23:18.065268: Average global foreground Dice: [0.6766, 0.857] +2022-05-20 03:23:18.065351: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-20 03:23:18.470927: lr: 4e-06 +2022-05-20 03:23:18.471126: saving scheduled checkpoint file... +2022-05-20 03:23:18.529032: saving checkpoint... +2022-05-20 03:23:19.331541: done, saving took 0.86 seconds +2022-05-20 03:23:19.340604: done +2022-05-20 03:23:38.261772: This epoch took 509.360708 s + +2022-05-20 03:23:38.262043: +epoch: 117 +2022-05-20 03:31:21.822348: train loss : 0.0949 +2022-05-20 03:31:48.177223: validation loss: 0.1081 +2022-05-20 03:31:48.177839: Average global foreground Dice: [0.6756, 0.8533] +2022-05-20 03:31:48.177922: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-20 03:31:48.591123: lr: 3e-06 +2022-05-20 03:31:48.591320: saving scheduled checkpoint file... +2022-05-20 03:31:48.649183: saving checkpoint... +2022-05-20 03:31:49.587246: done, saving took 1.00 seconds +2022-05-20 03:31:49.596138: done +2022-05-20 03:32:46.489815: This epoch took 548.227656 s + +2022-05-20 03:32:46.490309: +epoch: 118 +2022-05-20 03:40:29.391732: train loss : 0.0937 +2022-05-20 03:40:55.720541: validation loss: 0.1077 +2022-05-20 03:40:55.721191: Average global foreground Dice: [0.6845, 0.8654] +2022-05-20 03:40:55.721326: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-20 03:40:56.139208: lr: 2e-06 +2022-05-20 03:40:56.139404: saving scheduled checkpoint file... +2022-05-20 03:40:56.197829: saving checkpoint... +2022-05-20 03:40:57.126968: done, saving took 0.99 seconds +2022-05-20 03:40:57.135924: done +2022-05-20 03:41:15.609170: This epoch took 509.118477 s + +2022-05-20 03:41:15.609459: +epoch: 119 +2022-05-20 03:48:58.805049: train loss : 0.0983 +2022-05-20 03:49:25.164801: validation loss: 0.1256 +2022-05-20 03:49:25.165486: Average global foreground Dice: [0.6756, 0.8645] +2022-05-20 03:49:25.165570: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-20 03:49:25.569607: lr: 1e-06 +2022-05-20 03:49:25.569806: saving scheduled checkpoint file... +2022-05-20 03:49:25.627492: saving checkpoint... +2022-05-20 03:49:26.562666: done, saving took 0.99 seconds +2022-05-20 03:49:26.572377: done +2022-05-20 03:49:52.132060: This epoch took 516.522492 s + +2022-05-20 03:49:52.132335: +epoch: 120 +2022-05-20 03:57:35.176126: train loss : 0.0999 +2022-05-20 03:58:01.531392: validation loss: 0.1005 +2022-05-20 03:58:01.532017: Average global foreground Dice: [0.6722, 0.8625] +2022-05-20 03:58:01.532104: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-20 03:58:01.950476: lr: 1e-06 +2022-05-20 03:58:01.950664: saving scheduled checkpoint file... +2022-05-20 03:58:02.008911: saving checkpoint... +2022-05-20 03:58:02.898809: done, saving took 0.95 seconds +2022-05-20 03:58:02.908306: done +2022-05-20 03:58:28.736899: This epoch took 516.604453 s + +2022-05-20 03:58:28.737174: +epoch: 121 +2022-05-20 04:06:11.718487: train loss : 0.0949 +2022-05-20 04:06:38.102492: validation loss: 0.1138 +2022-05-20 04:06:38.103064: Average global foreground Dice: [0.6876, 0.851] +2022-05-20 04:06:38.103145: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-20 04:06:38.513355: lr: 1e-06 +2022-05-20 04:06:38.513543: saving scheduled checkpoint file... +2022-05-20 04:06:38.571612: saving checkpoint... +2022-05-20 04:06:39.500064: done, saving took 0.99 seconds +2022-05-20 04:06:39.509925: done +2022-05-20 04:07:06.909122: This epoch took 518.171830 s + +2022-05-20 04:07:06.909507: +epoch: 122 +2022-05-20 04:14:49.729285: train loss : 0.0974 +2022-05-20 04:15:16.127015: validation loss: 0.1199 +2022-05-20 04:15:16.127755: Average global foreground Dice: [0.665, 0.867] +2022-05-20 04:15:16.127844: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-20 04:15:16.534158: lr: 0.0 +2022-05-20 04:15:16.534334: saving scheduled checkpoint file... +2022-05-20 04:15:16.591745: saving checkpoint... +2022-05-20 04:15:17.451854: done, saving took 0.92 seconds +2022-05-20 04:15:17.460165: done +2022-05-20 04:15:39.474860: This epoch took 512.565244 s + +2022-05-20 04:15:39.475165: +epoch: 123 +2022-05-20 04:23:22.843241: train loss : 0.0932 +2022-05-20 04:23:49.158400: validation loss: 0.0986 +2022-05-20 04:23:49.159024: Average global foreground Dice: [0.6783, 0.8611] +2022-05-20 04:23:49.159109: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-20 04:23:49.565722: lr: 0.0 +2022-05-20 04:23:49.565911: saving scheduled checkpoint file... +2022-05-20 04:23:49.625033: saving checkpoint... +2022-05-20 04:23:50.571559: done, saving took 1.01 seconds +2022-05-20 04:23:50.581537: done +2022-05-20 04:24:08.915749: This epoch took 509.440476 s + +2022-05-20 04:24:08.916007: +epoch: 124 +2022-05-20 04:31:52.328473: train loss : 0.0927 +2022-05-20 04:32:18.650634: validation loss: 0.0996 +2022-05-20 04:32:18.652027: Average global foreground Dice: [0.6767, 0.8734] +2022-05-20 04:32:18.652123: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-20 04:32:19.064733: lr: 0.0 +2022-05-20 04:32:19.064918: saving scheduled checkpoint file... +2022-05-20 04:32:19.121712: saving checkpoint... +2022-05-20 04:32:20.058147: done, saving took 0.99 seconds +2022-05-20 04:32:20.068127: done +2022-05-20 04:32:44.013047: This epoch took 515.096796 s + +2022-05-20 04:32:44.069348: saving checkpoint... +2022-05-20 04:32:44.745355: done, saving took 0.73 seconds diff --git a/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_1/debug.json b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_1/debug.json new file mode 100644 index 0000000000000000000000000000000000000000..e087b8bb4a40f8e0512414ab60fd16334326f588 --- /dev/null +++ b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_1/debug.json @@ -0,0 +1,106 @@ +{ + "all_tr_losses": "[0.3387577, 0.22670184, 0.20843363, 0.18841243, 0.17535995, 0.16141209, 0.15405571, 0.15061752, 0.15452078, 0.15204631, 0.13644604, 0.15092108, 0.14956972, 0.15689449, 0.1389342, 0.15144578, 0.14459546, 0.14335355, 0.13697049, 0.13083953, 0.13117258, 0.13288935, 0.121476956, 0.13318454, 0.13405542, 0.12568034, 0.12733866, 0.12671013, 0.12438219, 0.12844828, 0.12240422, 0.123266265, 0.1184496, 0.13073105, 0.1177899, 0.12049695, 0.11747541, 0.11702523, 0.123542555, 0.123520136, 0.115563944, 0.12850183, 0.11967583, 0.10957871, 0.1181682, 0.1152821, 0.11642279, 0.113893405, 0.119086206, 0.11678498, 0.123117104, 0.115178846, 0.11567018, 0.11985725, 0.1193085, 0.11845499, 0.11243107, 0.10831526, 0.10665122, 0.11258734, 0.115851745, 0.103712484, 0.11779399, 0.112267956, 0.113086045, 0.105029725, 0.09838733, 0.107105955, 0.10617215, 0.104107745, 0.10817049]", + "all_val_eval_metrics": "[0.233077535801157, 0.4480676378630247, 0.4404973365469221, 0.5027243923707809, 0.5741376998387168, 0.5867073937865842, 0.6321437400284091, 0.4927105040099135, 0.6383731827011241, 0.6758598205366171, 0.6528591895183398, 0.6637023021051314, 0.6314350073658983, 0.6958482818557177, 0.6989928391929008, 0.6912398961084707, 0.6056596686051938, 0.7117652265461507, 0.7327134497429773, 0.7178654087577003, 0.7049232178373495, 0.7322875032534468, 0.7283864767120573, 0.703206818910838, 0.6927400603591269, 0.6898464873178634, 0.7178142991927289, 0.7232832695864841, 0.7031412997778539, 0.7459311406337356, 0.7169932042971585, 0.6961739934424764, 0.7405176514536038, 0.7500159841744369, 0.7223372438887824, 0.7509090590750573, 0.73228006616551, 0.690102673090841, 0.7264675974757193, 0.7339441598712162, 0.734639125446374, 0.747450182941839, 0.7020488500687379, 0.7402710892243097, 0.7477285325027372, 0.7586862586951484, 0.7242972876877314, 0.7194051296686022, 0.7512135982956042, 0.753453557303387, 0.7631378971540221, 0.7503152041647418, 0.7545195706241937, 0.7526524973870545, 0.7577355105991712, 0.7543985646253323, 0.7504404477878608, 0.7558896428716659, 0.7615744371323204, 0.7533564094350783, 0.7656408155315066, 0.7363721082843939, 0.752561029693577, 0.7385154899854834, 0.7577475393231858, 0.7401747061930488, 0.7604726162422142, 0.7553712579261274, 0.7234086586128576, 0.7414167035104149, 0.7605660772328076]", + "all_val_losses": "[0.2742561, 0.24531353, 0.18815613, 0.17646404, 0.15832248, 0.2023902, 0.15069067, 0.16915901, 0.15229218, 0.14243098, 0.14826919, 0.15193358, 0.14406703, 0.14335576, 0.1447152, 0.13216203, 0.16123854, 0.1316906, 0.11502855, 0.11714242, 0.13802771, 0.13044147, 0.123741314, 0.1385207, 0.117034934, 0.14418751, 0.12603754, 0.12535158, 0.13195664, 0.1259039, 0.12737633, 0.13186885, 0.1261872, 0.12275005, 0.14040247, 0.13383172, 0.12274504, 0.14137703, 0.1274172, 0.118551254, 0.123299085, 0.13093303, 0.12997475, 0.10903824, 0.11815379, 0.12656236, 0.12919399, 0.12636602, 0.14482778, 0.11013544, 0.10813198, 0.1311769, 0.12588133, 0.12132832, 0.114321955, 0.1252415, 0.12425144, 0.108929895, 0.118857205, 0.11405925, 0.119004786, 0.119694024, 0.10662134, 0.13051103, 0.12756462, 0.11987198, 0.12853155, 0.12863775, 0.1274451, 0.12693381, 0.12606291]", + "all_val_losses_tr_mode": "[]", + "also_val_in_tr_mode": "False", + "amp_grad_scaler": "", + "args": "Namespace(batch_size=2, config='./configs/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32.yaml', continue_training=False, crop_size=[64, 192, 192], dbs=False, deep_supervision_scales=[], deterministic=False, disable_decoder=False, disable_ds=True, disable_postprocessing_on_folds=False, disable_saving=False, drop_path=0.0, find_lr=False, find_zero_weight_decay=False, fix_ds_net_numpool=False, fold='1', fp32=False, hdfs_base='GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32', initial_lr=0.0003, is_sigmoid=False, is_spatial_aug_only=False, is_ssl=False, layer_decay=1.0, local_rank=0, loss_name='', lrschedule='warmup_cosine', mask_ratio=0.75, max_loss_cal='v1', max_num_epochs=125, merge_femur=False, min_lr=0, model='Generic_TransUNet_max_ppbp', model_params={'is_masked_attn': True, 'max_dec_layers': 3, 'is_max_bottleneck_transformer': True, 'vit_depth': 1, 'max_msda': '', 'is_max_ms': True, 'max_ms_idxs': [-4, -3, -2], 'max_hidden_dim': 192, 'mw': 1.0, 'is_max_ds': True, 'is_masking': True, 'is_max_hungarian': True, 'num_queries': 20, 'is_max_cls': True, 'is_mhsa_float32': True}, n_class=17, network='3d_fullres', network_trainer='nnUNetTrainerV2_DDP', npz=False, opt_betas=None, opt_eps=1e-08, optim_name='adamw', p='nnUNetPlansv2.1', plan_update='', pretrained=False, pretrained_config=None, pretrained_weights=None, rank=0, reclip=None, resume='auto', skip_grad_nan=False, task='Task008_HepaticVessel', task_pretrained='Task801_WORD', total_batch_size=None, use_compressed_data=False, val_final=False, val_folder='validation_raw', valbest=False, validation_only=False, warmup_epochs=10, weight_decay=3e-05, world_size=8)", + "base_num_features": "32", + "basic_generator_patch_size": "[178 308 233]", + "batch_dice": "True", + "batch_size": "2", + "best_MA_tr_loss_for_patience": "None", + "best_epoch_based_on_MA_tr_loss": "None", + "best_val_eval_criterion_MA": "None", + "classes": "[1, 2]", + "conv_per_stage": "2", + "data_aug_params": "{'selected_data_channels': None, 'selected_seg_channels': [0], 'do_elastic': False, 'elastic_deform_alpha': (0.0, 900.0), 'elastic_deform_sigma': (9.0, 13.0), 'p_eldef': 0.2, 'do_scaling': True, 'scale_range': (0.7, 1.4), 'independent_scale_factor_for_each_axis': False, 'p_independent_scale_per_axis': 1, 'p_scale': 0.2, 'do_rotation': True, 'rotation_x': (-0.5235987755982988, 0.5235987755982988), 'rotation_y': (-0.5235987755982988, 0.5235987755982988), 'rotation_z': (-0.5235987755982988, 0.5235987755982988), 'rotation_p_per_axis': 1, 'p_rot': 0.2, 'random_crop': False, 'random_crop_dist_to_border': None, 'do_gamma': True, 'gamma_retain_stats': True, 'gamma_range': (0.7, 1.5), 'p_gamma': 0.3, 'do_mirror': True, 'mirror_axes': (0, 1, 2), 'dummy_2D': False, 'mask_was_used_for_normalization': OrderedDict([(0, False)]), 'border_mode_data': 'constant', 'all_segmentation_labels': None, 'move_last_seg_chanel_to_data': False, 'cascade_do_cascade_augmentations': False, 'cascade_random_binary_transform_p': 0.4, 'cascade_random_binary_transform_p_per_label': 1, 'cascade_random_binary_transform_size': (1, 8), 'cascade_remove_conn_comp_p': 0.2, 'cascade_remove_conn_comp_max_size_percent_threshold': 0.15, 'cascade_remove_conn_comp_fill_with_other_class_p': 0.0, 'do_additive_brightness': False, 'additive_brightness_p_per_sample': 0.15, 'additive_brightness_p_per_channel': 0.5, 'additive_brightness_mu': 0.0, 'additive_brightness_sigma': 0.1, 'num_threads': 12, 'num_cached_per_thread': 2, 'patch_size_for_spatialtransform': array([ 64, 192, 192])}", + "dataset_directory": "/opt/tiger/project/data/nnUNet_preprocessed/Task008_HepaticVessel", + "deep_supervision_scales": "[[1, 1, 1], [0.5, 0.5, 0.5], [0.25, 0.25, 0.25], [0.125, 0.125, 0.125], [0.0625, 0.0625, 0.0625]]", + "deterministic": "False", + "disable_ds": "True", + "distribute_batch_size": "False", + "dl_tr": "", + "dl_val": "", + "do_dummy_2D_aug": "False", + "ds_loss_weights": "[1. 0. 0. 0. 0.]", + "epoch": "71", + "experiment_name": "nnUNetTrainerV2_DDP", + "find_zero_weight_decay": "False", + "fold": "1", + "folder_with_preprocessed_data": "/opt/tiger/project/data/nnUNet_preprocessed/Task008_HepaticVessel/nnUNetData_plans_v2.1_stage1", + "fp16": "True", + "global_batch_size": "16", + "gt_niftis_folder": "/opt/tiger/project/data/nnUNet_preprocessed/Task008_HepaticVessel/gt_segmentations", + "inference_pad_border_mode": "constant", + "inference_pad_kwargs": "{'constant_values': 0}", + "init_args": "('/opt/tiger/project/data/nnUNet_preprocessed/Task008_HepaticVessel/nnUNetPlansv2.1_plans_3D.pkl', 1, 0, '/opt/tiger/project/data/nnUNet_trained_models/UNet_IN_NANFang/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32', '/opt/tiger/project/data/nnUNet_preprocessed/Task008_HepaticVessel', True, 1, True, False, False, True)", + "initial_lr": "0.0003", + "input_size": "[64, 192, 192]", + "is_spatial_aug_only": "False", + "is_ssl": "False", + "layer_decay": "1.0", + "local_rank": "0", + "log_file": "/opt/tiger/project/data/nnUNet_trained_models/UNet_IN_NANFang/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_1/training_log_2022_4_12_01_47_45.txt", + "loss": "None", + "lr_scheduler": "", + "lr_scheduler_eps": "0.001", + "lr_scheduler_name": "warmup_cosine", + "lr_scheduler_patience": "30", + "lr_threshold": "1e-06", + "max_num_epochs": "125", + "min_lr": "0", + "min_region_size_per_class": "None", + "min_size_per_class": "None", + "model": "Generic_TransUNet_max_ppbp", + "model_params": "{'is_masked_attn': True, 'max_dec_layers': 3, 'is_max_bottleneck_transformer': True, 'vit_depth': 1, 'max_msda': '', 'is_max_ms': True, 'max_ms_idxs': [-4, -3, -2], 'max_hidden_dim': 192, 'mw': 1.0, 'is_max_ds': True, 'is_masking': True, 'is_max_hungarian': True, 'num_queries': 20, 'is_max_cls': True, 'is_mhsa_float32': True}", + "net_conv_kernel_sizes": "[[3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3]]", + "net_num_pool_op_kernel_sizes": "[[2, 2, 2], [2, 2, 2], [2, 2, 2], [2, 2, 2], [1, 2, 2]]", + "net_pool_per_axis": "[4, 5, 5]", + "normalization_schemes": "OrderedDict([(0, 'CT')])", + "num_batches_per_epoch": "250", + "num_classes": "3", + "num_input_channels": "1", + "num_val_batches_per_epoch": "50", + "online_eval_fn": "[]", + "online_eval_foreground_dc": "[]", + "online_eval_fp": "[]", + "online_eval_tp": "[]", + "only_keep_largest_connected_component": "None", + "optim_name": "adamw", + "optimizer": "AdamW (\nParameter Group 0\n amsgrad: False\n betas: (0.9, 0.999)\n eps: 1e-08\n initial_lr: 0.0003\n lr: 0.00013567978905008846\n weight_decay: 3e-05\n)", + "output_folder": "/opt/tiger/project/data/nnUNet_trained_models/UNet_IN_NANFang/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_1", + "output_folder_base": "/opt/tiger/project/data/nnUNet_trained_models/UNet_IN_NANFang/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32", + "oversample_foreground_percent": "0.0", + "pad_all_sides": "None", + "patch_size": "[ 64 192 192]", + "patience": "50", + "pin_memory": "True", + "plans_file": "/opt/tiger/project/data/nnUNet_preprocessed/Task008_HepaticVessel/nnUNetPlansv2.1_plans_3D.pkl", + "reclip": "None", + "regions_class_order": "None", + "resume": "auto", + "save_best_checkpoint": "True", + "save_every": "1", + "save_final_checkpoint": "True", + "save_intermediate_checkpoints": "True", + "save_latest_only": "True", + "stage": "1", + "threeD": "True", + "tr_gen": "", + "train_loss_MA": "None", + "train_loss_MA_alpha": "0.93", + "train_loss_MA_eps": "0.0005", + "transpose_backward": "[0, 1, 2]", + "transpose_forward": "[0, 1, 2]", + "unpack_data": "True", + "use_mask_for_norm": "OrderedDict([(0, False)])", + "use_progress_bar": "False", + "val_eval_criterion_MA": "None", + "val_eval_criterion_alpha": "0.9", + "val_gen": "", + "warmup_epochs": "10", + "was_initialized": "True", + "weight_decay": "3e-05" +} \ No newline at end of file diff --git a/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_1/model_best.model b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_1/model_best.model new file mode 100644 index 0000000000000000000000000000000000000000..9751350d56ce7b4b6a861981ead8f000e9b6e507 --- /dev/null +++ b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_1/model_best.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8234e587470532959eed0bd70ede576a7799b69eb81123be03550133bd5581f0 +size 496529338 diff --git a/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_1/model_best.model.pkl b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_1/model_best.model.pkl new file mode 100644 index 0000000000000000000000000000000000000000..014c98a9a6354035352e573e86832fd2817affe9 --- /dev/null +++ b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_1/model_best.model.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d2097f69bb201473667d1842c13e15b59d686b7e6db4c94f0121336d326b6c24 +size 188314 diff --git a/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_1/model_final_checkpoint.model b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_1/model_final_checkpoint.model new file mode 100644 index 0000000000000000000000000000000000000000..bd5c184c6d6919236b078e4c00297dc98d2c0f72 --- /dev/null +++ b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_1/model_final_checkpoint.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:399f0625d42c8894614eb3bf5a03fdd14f3426ecf1bb509d38642d6bef46b528 +size 496529530 diff --git a/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_1/model_final_checkpoint.model.pkl b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_1/model_final_checkpoint.model.pkl new file mode 100644 index 0000000000000000000000000000000000000000..014c98a9a6354035352e573e86832fd2817affe9 --- /dev/null +++ b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_1/model_final_checkpoint.model.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d2097f69bb201473667d1842c13e15b59d686b7e6db4c94f0121336d326b6c24 +size 188314 diff --git a/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_1/model_latest.model b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_1/model_latest.model new file mode 100644 index 0000000000000000000000000000000000000000..8527d4da5d9fb7b386ace88e0e55827078d1d1ec --- /dev/null +++ b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_1/model_latest.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad2d3433793c557c8be722845c79ac21dc90b3061ffce24e9cda8262923fb819 +size 496529530 diff --git a/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_1/model_latest.model.pkl b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_1/model_latest.model.pkl new file mode 100644 index 0000000000000000000000000000000000000000..014c98a9a6354035352e573e86832fd2817affe9 --- /dev/null +++ b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_1/model_latest.model.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d2097f69bb201473667d1842c13e15b59d686b7e6db4c94f0121336d326b6c24 +size 188314 diff --git a/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_1/progress.png b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_1/progress.png new file mode 100644 index 0000000000000000000000000000000000000000..84c4645e846198d7e3394c1e9542705b44f5659f --- /dev/null +++ b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_1/progress.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0aa5c420a7f11adaadf914a1d684bab1ed9f6082d01fd4b7e20fb4094e15699f +size 304094 diff --git a/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_1/training_log_2022_4_12_01_47_45.txt b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_1/training_log_2022_4_12_01_47_45.txt new file mode 100644 index 0000000000000000000000000000000000000000..881e5bc9d73afbfd1bcf936bdb67e90afd30bccf --- /dev/null +++ b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_1/training_log_2022_4_12_01_47_45.txt @@ -0,0 +1,771 @@ +Starting... +2022-04-12 01:47:45.715590: Using splits from existing split file: /opt/tiger/project/data/nnUNet_preprocessed/Task008_HepaticVessel/splits_final.pkl +2022-04-12 01:47:45.716197: The split file contains 5 splits. +2022-04-12 01:47:45.716254: Desired fold for training: 1 +2022-04-12 01:47:45.716299: This split has 242 training and 61 validation cases. +2022-04-12 01:49:23.672634: TRAINING KEYS: + odict_keys(['hepaticvessel_001', 'hepaticvessel_002', 'hepaticvessel_004', 'hepaticvessel_005', 'hepaticvessel_007', 'hepaticvessel_008', 'hepaticvessel_010', 'hepaticvessel_011', 'hepaticvessel_013', 'hepaticvessel_018', 'hepaticvessel_019', 'hepaticvessel_020', 'hepaticvessel_022', 'hepaticvessel_026', 'hepaticvessel_028', 'hepaticvessel_030', 'hepaticvessel_031', 'hepaticvessel_032', 'hepaticvessel_033', 'hepaticvessel_039', 'hepaticvessel_040', 'hepaticvessel_042', 'hepaticvessel_046', 'hepaticvessel_049', 'hepaticvessel_050', 'hepaticvessel_051', 'hepaticvessel_052', 'hepaticvessel_053', 'hepaticvessel_057', 'hepaticvessel_058', 'hepaticvessel_059', 'hepaticvessel_061', 'hepaticvessel_066', 'hepaticvessel_067', 'hepaticvessel_068', 'hepaticvessel_070', 'hepaticvessel_072', 'hepaticvessel_075', 'hepaticvessel_077', 'hepaticvessel_078', 'hepaticvessel_080', 'hepaticvessel_081', 'hepaticvessel_082', 'hepaticvessel_084', 'hepaticvessel_085', 'hepaticvessel_086', 'hepaticvessel_087', 'hepaticvessel_088', 'hepaticvessel_089', 'hepaticvessel_090', 'hepaticvessel_091', 'hepaticvessel_092', 'hepaticvessel_093', 'hepaticvessel_094', 'hepaticvessel_096', 'hepaticvessel_097', 'hepaticvessel_098', 'hepaticvessel_100', 'hepaticvessel_102', 'hepaticvessel_108', 'hepaticvessel_110', 'hepaticvessel_111', 'hepaticvessel_112', 'hepaticvessel_113', 'hepaticvessel_115', 'hepaticvessel_116', 'hepaticvessel_117', 'hepaticvessel_119', 'hepaticvessel_121', 'hepaticvessel_123', 'hepaticvessel_124', 'hepaticvessel_125', 'hepaticvessel_127', 'hepaticvessel_128', 'hepaticvessel_129', 'hepaticvessel_133', 'hepaticvessel_136', 'hepaticvessel_137', 'hepaticvessel_138', 'hepaticvessel_139', 'hepaticvessel_140', 'hepaticvessel_141', 'hepaticvessel_144', 'hepaticvessel_146', 'hepaticvessel_147', 'hepaticvessel_148', 'hepaticvessel_150', 'hepaticvessel_151', 'hepaticvessel_154', 'hepaticvessel_157', 'hepaticvessel_158', 'hepaticvessel_159', 'hepaticvessel_160', 'hepaticvessel_161', 'hepaticvessel_163', 'hepaticvessel_164', 'hepaticvessel_166', 'hepaticvessel_167', 'hepaticvessel_171', 'hepaticvessel_172', 'hepaticvessel_173', 'hepaticvessel_174', 'hepaticvessel_175', 'hepaticvessel_177', 'hepaticvessel_178', 'hepaticvessel_179', 'hepaticvessel_181', 'hepaticvessel_183', 'hepaticvessel_184', 'hepaticvessel_189', 'hepaticvessel_193', 'hepaticvessel_194', 'hepaticvessel_195', 'hepaticvessel_197', 'hepaticvessel_198', 'hepaticvessel_199', 'hepaticvessel_200', 'hepaticvessel_201', 'hepaticvessel_203', 'hepaticvessel_206', 'hepaticvessel_207', 'hepaticvessel_208', 'hepaticvessel_209', 'hepaticvessel_211', 'hepaticvessel_213', 'hepaticvessel_214', 'hepaticvessel_215', 'hepaticvessel_217', 'hepaticvessel_218', 'hepaticvessel_221', 'hepaticvessel_222', 'hepaticvessel_223', 'hepaticvessel_224', 'hepaticvessel_227', 'hepaticvessel_229', 'hepaticvessel_233', 'hepaticvessel_234', 'hepaticvessel_236', 'hepaticvessel_237', 'hepaticvessel_238', 'hepaticvessel_244', 'hepaticvessel_245', 'hepaticvessel_246', 'hepaticvessel_248', 'hepaticvessel_259', 'hepaticvessel_262', 'hepaticvessel_263', 'hepaticvessel_265', 'hepaticvessel_266', 'hepaticvessel_268', 'hepaticvessel_269', 'hepaticvessel_270', 'hepaticvessel_271', 'hepaticvessel_272', 'hepaticvessel_274', 'hepaticvessel_275', 'hepaticvessel_278', 'hepaticvessel_279', 'hepaticvessel_280', 'hepaticvessel_281', 'hepaticvessel_282', 'hepaticvessel_284', 'hepaticvessel_286', 'hepaticvessel_289', 'hepaticvessel_290', 'hepaticvessel_291', 'hepaticvessel_293', 'hepaticvessel_294', 'hepaticvessel_296', 'hepaticvessel_300', 'hepaticvessel_305', 'hepaticvessel_307', 'hepaticvessel_308', 'hepaticvessel_309', 'hepaticvessel_314', 'hepaticvessel_318', 'hepaticvessel_319', 'hepaticvessel_320', 'hepaticvessel_322', 'hepaticvessel_323', 'hepaticvessel_325', 'hepaticvessel_327', 'hepaticvessel_329', 'hepaticvessel_333', 'hepaticvessel_335', 'hepaticvessel_336', 'hepaticvessel_338', 'hepaticvessel_344', 'hepaticvessel_350', 'hepaticvessel_353', 'hepaticvessel_358', 'hepaticvessel_359', 'hepaticvessel_361', 'hepaticvessel_363', 'hepaticvessel_367', 'hepaticvessel_368', 'hepaticvessel_369', 'hepaticvessel_371', 'hepaticvessel_372', 'hepaticvessel_373', 'hepaticvessel_374', 'hepaticvessel_375', 'hepaticvessel_378', 'hepaticvessel_379', 'hepaticvessel_382', 'hepaticvessel_385', 'hepaticvessel_386', 'hepaticvessel_393', 'hepaticvessel_396', 'hepaticvessel_397', 'hepaticvessel_398', 'hepaticvessel_400', 'hepaticvessel_401', 'hepaticvessel_402', 'hepaticvessel_404', 'hepaticvessel_411', 'hepaticvessel_412', 'hepaticvessel_416', 'hepaticvessel_419', 'hepaticvessel_420', 'hepaticvessel_422', 'hepaticvessel_423', 'hepaticvessel_424', 'hepaticvessel_425', 'hepaticvessel_427', 'hepaticvessel_429', 'hepaticvessel_431', 'hepaticvessel_432', 'hepaticvessel_433', 'hepaticvessel_434', 'hepaticvessel_437', 'hepaticvessel_438', 'hepaticvessel_441', 'hepaticvessel_442', 'hepaticvessel_444', 'hepaticvessel_445', 'hepaticvessel_446', 'hepaticvessel_447', 'hepaticvessel_453', 'hepaticvessel_454', 'hepaticvessel_455', 'hepaticvessel_458']) +2022-04-12 01:49:23.672823: VALIDATION KEYS: + odict_keys(['hepaticvessel_016', 'hepaticvessel_023', 'hepaticvessel_025', 'hepaticvessel_027', 'hepaticvessel_029', 'hepaticvessel_044', 'hepaticvessel_062', 'hepaticvessel_065', 'hepaticvessel_071', 'hepaticvessel_079', 'hepaticvessel_083', 'hepaticvessel_103', 'hepaticvessel_104', 'hepaticvessel_131', 'hepaticvessel_132', 'hepaticvessel_142', 'hepaticvessel_145', 'hepaticvessel_165', 'hepaticvessel_185', 'hepaticvessel_186', 'hepaticvessel_192', 'hepaticvessel_196', 'hepaticvessel_210', 'hepaticvessel_219', 'hepaticvessel_225', 'hepaticvessel_230', 'hepaticvessel_231', 'hepaticvessel_235', 'hepaticvessel_240', 'hepaticvessel_242', 'hepaticvessel_255', 'hepaticvessel_256', 'hepaticvessel_258', 'hepaticvessel_285', 'hepaticvessel_287', 'hepaticvessel_297', 'hepaticvessel_299', 'hepaticvessel_303', 'hepaticvessel_321', 'hepaticvessel_324', 'hepaticvessel_330', 'hepaticvessel_332', 'hepaticvessel_340', 'hepaticvessel_341', 'hepaticvessel_345', 'hepaticvessel_348', 'hepaticvessel_349', 'hepaticvessel_356', 'hepaticvessel_362', 'hepaticvessel_383', 'hepaticvessel_384', 'hepaticvessel_388', 'hepaticvessel_389', 'hepaticvessel_391', 'hepaticvessel_406', 'hepaticvessel_407', 'hepaticvessel_408', 'hepaticvessel_409', 'hepaticvessel_440', 'hepaticvessel_443', 'hepaticvessel_456']) +2022-04-12 01:49:37.964392: loading checkpoint /opt/tiger/project/data/nnUNet_trained_models/UNet_IN_NANFang/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_1/model_latest.model train= True +2022-04-12 01:49:38.804048: lr: 0.000132 +2022-04-12 01:49:53.151024: +epoch: 71 +2022-04-12 01:57:49.694312: train loss : 0.1032 +2022-04-12 01:58:16.156663: validation loss: 0.1305 +2022-04-12 01:58:16.157230: Average global foreground Dice: [0.643, 0.8416] +2022-04-12 01:58:16.157407: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 01:58:16.571024: lr: 0.000132 +2022-04-12 01:58:16.571239: saving scheduled checkpoint file... +2022-04-12 01:58:16.631706: saving checkpoint... +2022-04-12 01:58:17.447424: done, saving took 0.88 seconds +2022-04-12 01:58:17.454373: done +2022-04-12 01:58:47.022275: This epoch took 533.870604 s + +2022-04-12 01:58:47.022511: +epoch: 72 +2022-04-12 02:06:29.648321: train loss : 0.1011 +2022-04-12 02:06:55.986377: validation loss: 0.1333 +2022-04-12 02:06:55.987037: Average global foreground Dice: [0.6506, 0.8528] +2022-04-12 02:06:55.987121: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 02:06:56.392863: lr: 0.000128 +2022-04-12 02:06:56.393059: saving scheduled checkpoint file... +2022-04-12 02:06:56.443024: saving checkpoint... +2022-04-12 02:06:57.321468: done, saving took 0.93 seconds +2022-04-12 02:06:57.328844: done +2022-04-12 02:06:57.356558: saving checkpoint... +2022-04-12 02:06:57.917594: done, saving took 0.59 seconds +2022-04-12 02:07:23.173922: This epoch took 516.151274 s + +2022-04-12 02:07:23.174200: +epoch: 73 +2022-04-12 02:15:06.193495: train loss : 0.1096 +2022-04-12 02:15:32.526185: validation loss: 0.1035 +2022-04-12 02:15:32.526824: Average global foreground Dice: [0.6583, 0.8722] +2022-04-12 02:15:32.526909: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 02:15:32.995968: lr: 0.000124 +2022-04-12 02:15:32.996156: saving scheduled checkpoint file... +2022-04-12 02:15:33.046960: saving checkpoint... +2022-04-12 02:15:33.980390: done, saving took 0.98 seconds +2022-04-12 02:15:33.988710: done +2022-04-12 02:15:34.017741: saving checkpoint... +2022-04-12 02:15:34.889052: done, saving took 0.90 seconds +2022-04-12 02:16:02.025848: This epoch took 518.851540 s + +2022-04-12 02:16:02.026114: +epoch: 74 +2022-04-12 02:23:44.390225: train loss : 0.1066 +2022-04-12 02:24:10.791477: validation loss: 0.1189 +2022-04-12 02:24:10.792030: Average global foreground Dice: [0.6459, 0.8639] +2022-04-12 02:24:10.792114: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 02:24:11.200119: lr: 0.000119 +2022-04-12 02:24:11.200323: saving scheduled checkpoint file... +2022-04-12 02:24:11.250188: saving checkpoint... +2022-04-12 02:24:12.043696: done, saving took 0.84 seconds +2022-04-12 02:24:12.050386: done +2022-04-12 02:24:12.076408: saving checkpoint... +2022-04-12 02:24:12.850010: done, saving took 0.80 seconds +2022-04-12 02:24:35.732872: This epoch took 513.706657 s + +2022-04-12 02:24:35.733162: +epoch: 75 +2022-04-12 02:32:18.218835: train loss : 0.1074 +2022-04-12 02:32:44.560797: validation loss: 0.1117 +2022-04-12 02:32:44.561381: Average global foreground Dice: [0.6595, 0.8315] +2022-04-12 02:32:44.561466: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 02:32:44.964844: lr: 0.000115 +2022-04-12 02:32:44.965041: saving scheduled checkpoint file... +2022-04-12 02:32:45.014532: saving checkpoint... +2022-04-12 02:32:45.916269: done, saving took 0.95 seconds +2022-04-12 02:32:45.923709: done +2022-04-12 02:33:08.392709: This epoch took 512.659441 s + +2022-04-12 02:33:08.393205: +epoch: 76 +2022-04-12 02:40:50.204226: train loss : 0.1033 +2022-04-12 02:41:16.547214: validation loss: 0.1284 +2022-04-12 02:41:16.547741: Average global foreground Dice: [0.6446, 0.8595] +2022-04-12 02:41:16.547825: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 02:41:16.955729: lr: 0.000112 +2022-04-12 02:41:16.955919: saving scheduled checkpoint file... +2022-04-12 02:41:17.005730: saving checkpoint... +2022-04-12 02:41:17.905257: done, saving took 0.95 seconds +2022-04-12 02:41:17.913049: done +2022-04-12 02:41:17.938973: saving checkpoint... +2022-04-12 02:41:18.816704: done, saving took 0.90 seconds +2022-04-12 02:41:47.085027: This epoch took 518.691720 s + +2022-04-12 02:41:47.085310: +epoch: 77 +2022-04-12 02:49:29.633742: train loss : 0.0987 +2022-04-12 02:49:56.015526: validation loss: 0.1085 +2022-04-12 02:49:56.016048: Average global foreground Dice: [0.661, 0.8382] +2022-04-12 02:49:56.016127: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 02:49:56.423056: lr: 0.000108 +2022-04-12 02:49:56.423248: saving scheduled checkpoint file... +2022-04-12 02:49:56.472756: saving checkpoint... +2022-04-12 02:49:57.363767: done, saving took 0.94 seconds +2022-04-12 02:49:57.371142: done +2022-04-12 02:49:57.397474: saving checkpoint... +2022-04-12 02:49:58.250980: done, saving took 0.88 seconds +2022-04-12 02:50:17.277142: This epoch took 510.191730 s + +2022-04-12 02:50:17.277422: +epoch: 78 +2022-04-12 02:58:00.183569: train loss : 0.1005 +2022-04-12 02:58:26.547135: validation loss: 0.1207 +2022-04-12 02:58:26.547724: Average global foreground Dice: [0.642, 0.8506] +2022-04-12 02:58:26.547811: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 02:58:26.953531: lr: 0.000104 +2022-04-12 02:58:26.953735: saving scheduled checkpoint file... +2022-04-12 02:58:27.003320: saving checkpoint... +2022-04-12 02:58:27.795536: done, saving took 0.84 seconds +2022-04-12 02:58:27.802729: done +2022-04-12 02:58:47.164164: This epoch took 509.886633 s + +2022-04-12 02:58:47.164483: +epoch: 79 +2022-04-12 03:06:29.668583: train loss : 0.1029 +2022-04-12 03:06:56.021065: validation loss: 0.1178 +2022-04-12 03:06:56.021674: Average global foreground Dice: [0.6534, 0.8324] +2022-04-12 03:06:56.021758: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 03:06:56.429858: lr: 0.0001 +2022-04-12 03:06:56.430068: saving scheduled checkpoint file... +2022-04-12 03:06:56.480796: saving checkpoint... +2022-04-12 03:06:57.429707: done, saving took 1.00 seconds +2022-04-12 03:06:57.437105: done +2022-04-12 03:07:19.194026: This epoch took 512.029446 s + +2022-04-12 03:07:19.194331: +epoch: 80 +2022-04-12 03:15:01.648175: train loss : 0.1022 +2022-04-12 03:15:28.021894: validation loss: 0.1077 +2022-04-12 03:15:28.022485: Average global foreground Dice: [0.6583, 0.8729] +2022-04-12 03:15:28.022582: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 03:15:28.430223: lr: 9.6e-05 +2022-04-12 03:15:28.430423: saving scheduled checkpoint file... +2022-04-12 03:15:28.479786: saving checkpoint... +2022-04-12 03:15:29.378370: done, saving took 0.95 seconds +2022-04-12 03:15:29.385749: done +2022-04-12 03:15:29.411012: saving checkpoint... +2022-04-12 03:15:30.287992: done, saving took 0.90 seconds +2022-04-12 03:15:55.006488: This epoch took 515.812053 s + +2022-04-12 03:15:55.006775: +epoch: 81 +2022-04-12 03:23:37.473773: train loss : 0.0935 +2022-04-12 03:24:03.849096: validation loss: 0.1134 +2022-04-12 03:24:03.849644: Average global foreground Dice: [0.648, 0.7903] +2022-04-12 03:24:03.849741: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 03:24:04.256096: lr: 9.2e-05 +2022-04-12 03:24:04.256285: saving scheduled checkpoint file... +2022-04-12 03:24:04.305759: saving checkpoint... +2022-04-12 03:24:05.082351: done, saving took 0.83 seconds +2022-04-12 03:24:05.089416: done +2022-04-12 03:24:49.673854: This epoch took 534.666977 s + +2022-04-12 03:24:49.674152: +epoch: 82 +2022-04-12 03:32:32.153378: train loss : 0.0948 +2022-04-12 03:32:58.510216: validation loss: 0.1180 +2022-04-12 03:32:58.510844: Average global foreground Dice: [0.6668, 0.8711] +2022-04-12 03:32:58.510931: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 03:32:58.911346: lr: 8.8e-05 +2022-04-12 03:32:58.911533: saving scheduled checkpoint file... +2022-04-12 03:32:58.963757: saving checkpoint... +2022-04-12 03:32:59.879307: done, saving took 0.97 seconds +2022-04-12 03:32:59.886877: done +2022-04-12 03:33:20.449904: This epoch took 510.775651 s + +2022-04-12 03:33:20.450195: +epoch: 83 +2022-04-12 03:41:02.915692: train loss : 0.1014 +2022-04-12 03:41:29.292401: validation loss: 0.1061 +2022-04-12 03:41:29.293012: Average global foreground Dice: [0.6479, 0.8475] +2022-04-12 03:41:29.293097: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 03:41:29.691291: lr: 8.5e-05 +2022-04-12 03:41:29.691468: saving scheduled checkpoint file... +2022-04-12 03:41:29.740707: saving checkpoint... +2022-04-12 03:41:30.624274: done, saving took 0.93 seconds +2022-04-12 03:41:30.632542: done +2022-04-12 03:41:50.862592: This epoch took 510.412282 s + +2022-04-12 03:41:50.862871: +epoch: 84 +2022-04-12 03:49:33.934120: train loss : 0.1059 +2022-04-12 03:50:00.304400: validation loss: 0.1166 +2022-04-12 03:50:00.304986: Average global foreground Dice: [0.6326, 0.8662] +2022-04-12 03:50:00.305071: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 03:50:00.776315: lr: 8.1e-05 +2022-04-12 03:50:00.776498: saving scheduled checkpoint file... +2022-04-12 03:50:00.825917: saving checkpoint... +2022-04-12 03:50:01.626990: done, saving took 0.85 seconds +2022-04-12 03:50:01.657367: done +2022-04-12 03:50:39.177516: This epoch took 528.314546 s + +2022-04-12 03:50:39.177794: +epoch: 85 +2022-04-12 03:58:21.691803: train loss : 0.0956 +2022-04-12 03:58:48.036259: validation loss: 0.1199 +2022-04-12 03:58:48.036843: Average global foreground Dice: [0.6538, 0.8629] +2022-04-12 03:58:48.036927: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 03:58:48.438333: lr: 7.7e-05 +2022-04-12 03:58:48.438528: saving scheduled checkpoint file... +2022-04-12 03:58:48.488003: saving checkpoint... +2022-04-12 03:58:49.389500: done, saving took 0.95 seconds +2022-04-12 03:58:49.397269: done +2022-04-12 03:58:49.433043: saving checkpoint... +2022-04-12 03:58:50.288560: done, saving took 0.89 seconds +2022-04-12 03:59:11.186197: This epoch took 512.008299 s + +2022-04-12 03:59:11.186495: +epoch: 86 +2022-04-12 04:06:53.730433: train loss : 0.1128 +2022-04-12 04:07:20.065057: validation loss: 0.1089 +2022-04-12 04:07:20.065630: Average global foreground Dice: [0.6651, 0.871] +2022-04-12 04:07:20.065715: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 04:07:20.475675: lr: 7.4e-05 +2022-04-12 04:07:20.475862: saving scheduled checkpoint file... +2022-04-12 04:07:20.525327: saving checkpoint... +2022-04-12 04:07:21.429981: done, saving took 0.95 seconds +2022-04-12 04:07:21.437414: done +2022-04-12 04:07:21.463322: saving checkpoint... +2022-04-12 04:07:22.345131: done, saving took 0.91 seconds +2022-04-12 04:08:12.194013: This epoch took 541.007360 s + +2022-04-12 04:08:12.194272: +epoch: 87 +2022-04-12 04:15:55.084049: train loss : 0.1040 +2022-04-12 04:16:21.452374: validation loss: 0.1230 +2022-04-12 04:16:21.453017: Average global foreground Dice: [0.6493, 0.8623] +2022-04-12 04:16:21.453100: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 04:16:21.854911: lr: 7e-05 +2022-04-12 04:16:21.855108: saving scheduled checkpoint file... +2022-04-12 04:16:21.905278: saving checkpoint... +2022-04-12 04:16:22.659439: done, saving took 0.80 seconds +2022-04-12 04:16:22.666493: done +2022-04-12 04:16:22.691724: saving checkpoint... +2022-04-12 04:16:23.472085: done, saving took 0.81 seconds +2022-04-12 04:16:45.433445: This epoch took 513.239073 s + +2022-04-12 04:16:45.433738: +epoch: 88 +2022-04-12 04:24:28.149167: train loss : 0.1043 +2022-04-12 04:24:54.479588: validation loss: 0.1004 +2022-04-12 04:24:54.480131: Average global foreground Dice: [0.6462, 0.8859] +2022-04-12 04:24:54.480213: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 04:24:54.885033: lr: 6.7e-05 +2022-04-12 04:24:54.885221: saving scheduled checkpoint file... +2022-04-12 04:24:54.934930: saving checkpoint... +2022-04-12 04:24:55.815455: done, saving took 0.93 seconds +2022-04-12 04:24:55.823053: done +2022-04-12 04:24:55.850444: saving checkpoint... +2022-04-12 04:24:56.753325: done, saving took 0.93 seconds +2022-04-12 04:25:16.436562: This epoch took 511.002717 s + +2022-04-12 04:25:16.436848: +epoch: 89 +2022-04-12 04:32:59.019767: train loss : 0.1034 +2022-04-12 04:33:25.358483: validation loss: 0.1032 +2022-04-12 04:33:25.359166: Average global foreground Dice: [0.6397, 0.8846] +2022-04-12 04:33:25.359251: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 04:33:25.760066: lr: 6.3e-05 +2022-04-12 04:33:25.760276: saving scheduled checkpoint file... +2022-04-12 04:33:25.809910: saving checkpoint... +2022-04-12 04:33:26.703683: done, saving took 0.94 seconds +2022-04-12 04:33:26.712762: done +2022-04-12 04:33:26.745128: saving checkpoint... +2022-04-12 04:33:27.636033: done, saving took 0.92 seconds +2022-04-12 04:33:50.735580: This epoch took 514.298618 s + +2022-04-12 04:33:50.735859: +epoch: 90 +2022-04-12 04:41:33.113122: train loss : 0.1008 +2022-04-12 04:41:59.475420: validation loss: 0.0984 +2022-04-12 04:41:59.475977: Average global foreground Dice: [0.6269, 0.884] +2022-04-12 04:41:59.476137: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 04:41:59.888294: lr: 6e-05 +2022-04-12 04:41:59.888487: saving scheduled checkpoint file... +2022-04-12 04:41:59.939038: saving checkpoint... +2022-04-12 04:42:00.744348: done, saving took 0.86 seconds +2022-04-12 04:42:00.751153: done +2022-04-12 04:42:00.776982: saving checkpoint... +2022-04-12 04:42:01.575939: done, saving took 0.82 seconds +2022-04-12 04:42:18.097639: This epoch took 507.361677 s + +2022-04-12 04:42:18.097926: +epoch: 91 +2022-04-12 04:50:01.081358: train loss : 0.0977 +2022-04-12 04:50:27.449257: validation loss: 0.1181 +2022-04-12 04:50:27.449831: Average global foreground Dice: [0.6423, 0.8551] +2022-04-12 04:50:27.449915: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 04:50:27.860837: lr: 5.7e-05 +2022-04-12 04:50:27.861039: saving scheduled checkpoint file... +2022-04-12 04:50:27.910363: saving checkpoint... +2022-04-12 04:50:28.805149: done, saving took 0.94 seconds +2022-04-12 04:50:28.813156: done +2022-04-12 04:51:00.700607: This epoch took 522.602580 s + +2022-04-12 04:51:00.700882: +epoch: 92 +2022-04-12 04:58:43.172932: train loss : 0.0993 +2022-04-12 04:59:09.516165: validation loss: 0.1132 +2022-04-12 04:59:09.516814: Average global foreground Dice: [0.6483, 0.8695] +2022-04-12 04:59:09.516901: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 04:59:09.920715: lr: 5.4e-05 +2022-04-12 04:59:09.920972: saving scheduled checkpoint file... +2022-04-12 04:59:09.970634: saving checkpoint... +2022-04-12 04:59:10.865305: done, saving took 0.94 seconds +2022-04-12 04:59:10.873847: done +2022-04-12 04:59:10.909582: saving checkpoint... +2022-04-12 04:59:11.781749: done, saving took 0.91 seconds +2022-04-12 04:59:33.127198: This epoch took 512.426207 s + +2022-04-12 04:59:33.127510: +epoch: 93 +2022-04-12 05:07:15.831490: train loss : 0.0940 +2022-04-12 05:07:42.200351: validation loss: 0.1109 +2022-04-12 05:07:42.200903: Average global foreground Dice: [0.6617, 0.8547] +2022-04-12 05:07:42.200985: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 05:07:42.599155: lr: 5.1e-05 +2022-04-12 05:07:42.599353: saving scheduled checkpoint file... +2022-04-12 05:07:42.648796: saving checkpoint... +2022-04-12 05:07:43.536121: done, saving took 0.94 seconds +2022-04-12 05:07:43.544144: done +2022-04-12 05:07:43.572967: saving checkpoint... +2022-04-12 05:07:44.447019: done, saving took 0.90 seconds +2022-04-12 05:08:10.215270: This epoch took 517.087635 s + +2022-04-12 05:08:10.215595: +epoch: 94 +2022-04-12 05:15:52.855393: train loss : 0.1015 +2022-04-12 05:16:19.268202: validation loss: 0.1099 +2022-04-12 05:16:19.268808: Average global foreground Dice: [0.6554, 0.8489] +2022-04-12 05:16:19.268892: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 05:16:19.672113: lr: 4.8e-05 +2022-04-12 05:16:19.672306: saving scheduled checkpoint file... +2022-04-12 05:16:19.721741: saving checkpoint... +2022-04-12 05:16:20.529330: done, saving took 0.86 seconds +2022-04-12 05:16:20.558667: done +2022-04-12 05:16:58.649115: This epoch took 528.433414 s + +2022-04-12 05:16:58.649410: +epoch: 95 +2022-04-12 05:24:40.946638: train loss : 0.0991 +2022-04-12 05:25:07.301243: validation loss: 0.0969 +2022-04-12 05:25:07.301813: Average global foreground Dice: [0.6551, 0.8487] +2022-04-12 05:25:07.301899: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 05:25:07.703476: lr: 4.5e-05 +2022-04-12 05:25:07.703681: saving scheduled checkpoint file... +2022-04-12 05:25:07.753533: saving checkpoint... +2022-04-12 05:25:08.661253: done, saving took 0.96 seconds +2022-04-12 05:25:08.668943: done +2022-04-12 05:25:30.086483: This epoch took 511.436973 s + +2022-04-12 05:25:30.086766: +epoch: 96 +2022-04-12 05:33:12.843010: train loss : 0.0983 +2022-04-12 05:33:39.214464: validation loss: 0.1159 +2022-04-12 05:33:39.215133: Average global foreground Dice: [0.6455, 0.8688] +2022-04-12 05:33:39.215220: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 05:33:39.631214: lr: 4.2e-05 +2022-04-12 05:33:39.631405: saving scheduled checkpoint file... +2022-04-12 05:33:39.681313: saving checkpoint... +2022-04-12 05:33:40.582208: done, saving took 0.95 seconds +2022-04-12 05:33:40.590673: done +2022-04-12 05:34:24.072500: This epoch took 533.985631 s + +2022-04-12 05:34:24.072798: +epoch: 97 +2022-04-12 05:42:06.595031: train loss : 0.0979 +2022-04-12 05:42:32.949392: validation loss: 0.1080 +2022-04-12 05:42:32.950503: Average global foreground Dice: [0.6516, 0.8672] +2022-04-12 05:42:32.950598: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 05:42:33.356525: lr: 3.9e-05 +2022-04-12 05:42:33.356733: saving scheduled checkpoint file... +2022-04-12 05:42:33.406365: saving checkpoint... +2022-04-12 05:42:34.291275: done, saving took 0.93 seconds +2022-04-12 05:42:34.298696: done +2022-04-12 05:42:34.324865: saving checkpoint... +2022-04-12 05:42:35.196396: done, saving took 0.90 seconds +2022-04-12 05:43:01.950181: This epoch took 517.877281 s + +2022-04-12 05:43:01.950459: +epoch: 98 +2022-04-12 05:50:44.144110: train loss : 0.0967 +2022-04-12 05:51:10.501491: validation loss: 0.1040 +2022-04-12 05:51:10.502162: Average global foreground Dice: [0.6688, 0.8406] +2022-04-12 05:51:10.502246: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 05:51:10.904269: lr: 3.6e-05 +2022-04-12 05:51:10.904446: saving scheduled checkpoint file... +2022-04-12 05:51:10.953789: saving checkpoint... +2022-04-12 05:51:11.864342: done, saving took 0.96 seconds +2022-04-12 05:51:11.872040: done +2022-04-12 05:51:36.761628: This epoch took 514.811064 s + +2022-04-12 05:51:36.761946: +epoch: 99 +2022-04-12 05:59:19.299343: train loss : 0.0981 +2022-04-12 05:59:45.656408: validation loss: 0.1176 +2022-04-12 05:59:45.656996: Average global foreground Dice: [0.6576, 0.8719] +2022-04-12 05:59:45.657082: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 05:59:46.057636: lr: 3.4e-05 +2022-04-12 05:59:46.057829: saving scheduled checkpoint file... +2022-04-12 05:59:46.107279: saving checkpoint... +2022-04-12 05:59:46.958303: done, saving took 0.90 seconds +2022-04-12 05:59:46.966263: done +2022-04-12 05:59:46.993814: saving checkpoint... +2022-04-12 05:59:47.861776: done, saving took 0.90 seconds +2022-04-12 06:00:07.657879: This epoch took 510.895798 s + +2022-04-12 06:00:07.658240: +epoch: 100 +2022-04-12 06:07:50.112843: train loss : 0.1014 +2022-04-12 06:08:16.516289: validation loss: 0.1085 +2022-04-12 06:08:16.516893: Average global foreground Dice: [0.6732, 0.882] +2022-04-12 06:08:16.516977: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 06:08:16.930567: lr: 3.1e-05 +2022-04-12 06:08:16.930767: saving scheduled checkpoint file... +2022-04-12 06:08:16.980258: saving checkpoint... +2022-04-12 06:08:17.801977: done, saving took 0.87 seconds +2022-04-12 06:08:17.809429: done +2022-04-12 06:08:17.836146: saving checkpoint... +2022-04-12 06:08:18.598226: done, saving took 0.79 seconds +2022-04-12 06:08:39.009843: This epoch took 511.351490 s + +2022-04-12 06:08:39.010150: +epoch: 101 +2022-04-12 06:16:21.644188: train loss : 0.0946 +2022-04-12 06:16:48.017226: validation loss: 0.1126 +2022-04-12 06:16:48.017861: Average global foreground Dice: [0.6545, 0.8668] +2022-04-12 06:16:48.017947: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 06:16:48.422889: lr: 2.9e-05 +2022-04-12 06:16:48.423091: saving scheduled checkpoint file... +2022-04-12 06:16:48.472787: saving checkpoint... +2022-04-12 06:16:49.352198: done, saving took 0.93 seconds +2022-04-12 06:16:49.360469: done +2022-04-12 06:16:49.389912: saving checkpoint... +2022-04-12 06:16:50.275704: done, saving took 0.92 seconds +2022-04-12 06:17:13.655284: This epoch took 514.645032 s + +2022-04-12 06:17:13.655623: +epoch: 102 +2022-04-12 06:24:56.514107: train loss : 0.0948 +2022-04-12 06:25:22.929111: validation loss: 0.1128 +2022-04-12 06:25:22.929702: Average global foreground Dice: [0.6534, 0.8651] +2022-04-12 06:25:22.929787: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 06:25:23.345099: lr: 2.6e-05 +2022-04-12 06:25:23.345296: saving scheduled checkpoint file... +2022-04-12 06:25:23.394982: saving checkpoint... +2022-04-12 06:25:24.291731: done, saving took 0.95 seconds +2022-04-12 06:25:24.299320: done +2022-04-12 06:25:24.325481: saving checkpoint... +2022-04-12 06:25:25.198053: done, saving took 0.90 seconds +2022-04-12 06:26:04.287975: This epoch took 530.632249 s + +2022-04-12 06:26:04.288282: +epoch: 103 +2022-04-12 06:33:47.260315: train loss : 0.0910 +2022-04-12 06:34:13.618779: validation loss: 0.1122 +2022-04-12 06:34:13.619409: Average global foreground Dice: [0.6486, 0.8714] +2022-04-12 06:34:13.619493: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 06:34:14.021538: lr: 2.4e-05 +2022-04-12 06:34:14.021744: saving scheduled checkpoint file... +2022-04-12 06:34:14.071274: saving checkpoint... +2022-04-12 06:34:14.890485: done, saving took 0.87 seconds +2022-04-12 06:34:14.897752: done +2022-04-12 06:34:14.923895: saving checkpoint... +2022-04-12 06:34:15.695315: done, saving took 0.80 seconds +2022-04-12 06:34:35.904371: This epoch took 511.615988 s + +2022-04-12 06:34:35.904716: +epoch: 104 +2022-04-12 06:42:18.150730: train loss : 0.0937 +2022-04-12 06:42:44.539033: validation loss: 0.1077 +2022-04-12 06:42:44.539643: Average global foreground Dice: [0.6593, 0.8761] +2022-04-12 06:42:44.539728: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 06:42:44.946461: lr: 2.2e-05 +2022-04-12 06:42:44.946668: saving scheduled checkpoint file... +2022-04-12 06:42:44.996276: saving checkpoint... +2022-04-12 06:42:45.880019: done, saving took 0.93 seconds +2022-04-12 06:42:45.887694: done +2022-04-12 06:42:45.926809: saving checkpoint... +2022-04-12 06:42:46.787819: done, saving took 0.90 seconds +2022-04-12 06:43:03.784698: This epoch took 507.879827 s + +2022-04-12 06:43:03.784977: +epoch: 105 +2022-04-12 06:50:46.285807: train loss : 0.0888 +2022-04-12 06:51:12.650251: validation loss: 0.1235 +2022-04-12 06:51:12.650828: Average global foreground Dice: [0.647, 0.8546] +2022-04-12 06:51:12.650913: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 06:51:13.055029: lr: 2e-05 +2022-04-12 06:51:13.055219: saving scheduled checkpoint file... +2022-04-12 06:51:13.105167: saving checkpoint... +2022-04-12 06:51:14.011938: done, saving took 0.96 seconds +2022-04-12 06:51:14.020160: done +2022-04-12 06:51:30.358945: This epoch took 506.573867 s + +2022-04-12 06:51:30.359259: +epoch: 106 +2022-04-12 06:59:12.830997: train loss : 0.0939 +2022-04-12 06:59:39.170909: validation loss: 0.1194 +2022-04-12 06:59:39.171612: Average global foreground Dice: [0.6563, 0.8687] +2022-04-12 06:59:39.171698: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 06:59:39.576355: lr: 1.8e-05 +2022-04-12 06:59:39.576545: saving scheduled checkpoint file... +2022-04-12 06:59:39.628755: saving checkpoint... +2022-04-12 06:59:40.511643: done, saving took 0.93 seconds +2022-04-12 06:59:40.519433: done +2022-04-12 07:00:05.455869: This epoch took 515.096512 s + +2022-04-12 07:00:05.456148: +epoch: 107 +2022-04-12 07:07:48.161068: train loss : 0.0921 +2022-04-12 07:08:14.469932: validation loss: 0.1120 +2022-04-12 07:08:14.470494: Average global foreground Dice: [0.6492, 0.8366] +2022-04-12 07:08:14.470588: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 07:08:14.874798: lr: 1.6e-05 +2022-04-12 07:08:14.874981: saving scheduled checkpoint file... +2022-04-12 07:08:14.924342: saving checkpoint... +2022-04-12 07:08:15.680308: done, saving took 0.81 seconds +2022-04-12 07:08:15.687589: done +2022-04-12 07:08:41.636631: This epoch took 516.180385 s + +2022-04-12 07:08:41.636923: +epoch: 108 +2022-04-12 07:16:24.231153: train loss : 0.0926 +2022-04-12 07:16:50.641657: validation loss: 0.1105 +2022-04-12 07:16:50.642201: Average global foreground Dice: [0.6579, 0.8562] +2022-04-12 07:16:50.642281: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 07:16:51.045394: lr: 1.4e-05 +2022-04-12 07:16:51.045588: saving scheduled checkpoint file... +2022-04-12 07:16:51.094967: saving checkpoint... +2022-04-12 07:16:51.972413: done, saving took 0.93 seconds +2022-04-12 07:16:51.979677: done +2022-04-12 07:17:13.522203: This epoch took 511.885176 s + +2022-04-12 07:17:13.522489: +epoch: 109 +2022-04-12 07:24:55.869624: train loss : 0.0985 +2022-04-12 07:25:22.211050: validation loss: 0.1209 +2022-04-12 07:25:22.211686: Average global foreground Dice: [0.6453, 0.856] +2022-04-12 07:25:22.211768: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 07:25:22.622737: lr: 1.2e-05 +2022-04-12 07:25:22.622922: saving scheduled checkpoint file... +2022-04-12 07:25:22.672471: saving checkpoint... +2022-04-12 07:25:23.581385: done, saving took 0.96 seconds +2022-04-12 07:25:23.588823: done +2022-04-12 07:25:48.872918: This epoch took 515.350323 s + +2022-04-12 07:25:48.873195: +epoch: 110 +2022-04-12 07:33:32.551534: train loss : 0.0973 +2022-04-12 07:33:58.899036: validation loss: 0.1142 +2022-04-12 07:33:58.899562: Average global foreground Dice: [0.6618, 0.8648] +2022-04-12 07:33:58.899647: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 07:33:59.307953: lr: 1.1e-05 +2022-04-12 07:33:59.308133: saving scheduled checkpoint file... +2022-04-12 07:33:59.357507: saving checkpoint... +2022-04-12 07:34:00.162388: done, saving took 0.85 seconds +2022-04-12 07:34:00.169318: done +2022-04-12 07:34:18.244838: This epoch took 509.371542 s + +2022-04-12 07:34:18.245121: +epoch: 111 +2022-04-12 07:42:00.635880: train loss : 0.0909 +2022-04-12 07:42:27.000191: validation loss: 0.1107 +2022-04-12 07:42:27.000841: Average global foreground Dice: [0.6604, 0.8594] +2022-04-12 07:42:27.000930: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 07:42:27.404088: lr: 9e-06 +2022-04-12 07:42:27.404283: saving scheduled checkpoint file... +2022-04-12 07:42:27.453604: saving checkpoint... +2022-04-12 07:42:28.342470: done, saving took 0.94 seconds +2022-04-12 07:42:28.350491: done +2022-04-12 07:42:50.313202: This epoch took 512.067981 s + +2022-04-12 07:42:50.313535: +epoch: 112 +2022-04-12 07:50:32.778379: train loss : 0.1013 +2022-04-12 07:50:59.161238: validation loss: 0.1168 +2022-04-12 07:50:59.161930: Average global foreground Dice: [0.6706, 0.8804] +2022-04-12 07:50:59.162031: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 07:50:59.578756: lr: 8e-06 +2022-04-12 07:50:59.578949: saving scheduled checkpoint file... +2022-04-12 07:50:59.628338: saving checkpoint... +2022-04-12 07:51:00.521036: done, saving took 0.94 seconds +2022-04-12 07:51:00.529813: done +2022-04-12 07:51:20.023951: This epoch took 509.710284 s + +2022-04-12 07:51:20.024251: +epoch: 113 +2022-04-12 07:59:02.259558: train loss : 0.0952 +2022-04-12 07:59:28.574529: validation loss: 0.1126 +2022-04-12 07:59:28.575164: Average global foreground Dice: [0.6641, 0.8765] +2022-04-12 07:59:28.575247: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 07:59:28.994439: lr: 7e-06 +2022-04-12 07:59:28.994648: saving scheduled checkpoint file... +2022-04-12 07:59:29.044352: saving checkpoint... +2022-04-12 07:59:29.917429: done, saving took 0.92 seconds +2022-04-12 07:59:29.924901: done +2022-04-12 07:59:29.951155: saving checkpoint... +2022-04-12 07:59:30.825511: done, saving took 0.90 seconds +2022-04-12 08:00:04.223475: This epoch took 524.199126 s + +2022-04-12 08:00:04.223759: +epoch: 114 +2022-04-12 08:07:47.044150: train loss : 0.0915 +2022-04-12 08:08:13.393082: validation loss: 0.0974 +2022-04-12 08:08:13.393772: Average global foreground Dice: [0.6534, 0.8701] +2022-04-12 08:08:13.393862: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 08:08:13.804751: lr: 6e-06 +2022-04-12 08:08:13.804959: saving scheduled checkpoint file... +2022-04-12 08:08:13.855244: saving checkpoint... +2022-04-12 08:08:14.610067: done, saving took 0.81 seconds +2022-04-12 08:08:14.618208: done +2022-04-12 08:08:14.645764: saving checkpoint... +2022-04-12 08:08:15.449506: done, saving took 0.83 seconds +2022-04-12 08:08:31.929602: This epoch took 507.705730 s + +2022-04-12 08:08:31.929906: +epoch: 115 +2022-04-12 08:16:15.132812: train loss : 0.0947 +2022-04-12 08:16:41.487801: validation loss: 0.1080 +2022-04-12 08:16:41.488466: Average global foreground Dice: [0.6604, 0.8744] +2022-04-12 08:16:41.488559: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 08:16:41.899711: lr: 5e-06 +2022-04-12 08:16:41.899907: saving scheduled checkpoint file... +2022-04-12 08:16:41.949359: saving checkpoint... +2022-04-12 08:16:42.869321: done, saving took 0.97 seconds +2022-04-12 08:16:42.877380: done +2022-04-12 08:16:42.906241: saving checkpoint... +2022-04-12 08:16:43.814054: done, saving took 0.94 seconds +2022-04-12 08:17:08.953405: This epoch took 517.023400 s + +2022-04-12 08:17:08.953717: +epoch: 116 +2022-04-12 08:24:52.541597: train loss : 0.0937 +2022-04-12 08:25:18.884503: validation loss: 0.1244 +2022-04-12 08:25:18.885070: Average global foreground Dice: [0.6509, 0.8797] +2022-04-12 08:25:18.885158: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 08:25:19.299258: lr: 4e-06 +2022-04-12 08:25:19.299438: saving scheduled checkpoint file... +2022-04-12 08:25:19.348815: saving checkpoint... +2022-04-12 08:25:20.245363: done, saving took 0.95 seconds +2022-04-12 08:25:20.253742: done +2022-04-12 08:25:20.282433: saving checkpoint... +2022-04-12 08:25:21.142138: done, saving took 0.89 seconds +2022-04-12 08:25:45.637394: This epoch took 516.683575 s + +2022-04-12 08:25:45.637714: +epoch: 117 +2022-04-12 08:33:28.370584: train loss : 0.0921 +2022-04-12 08:33:54.730809: validation loss: 0.1215 +2022-04-12 08:33:54.731369: Average global foreground Dice: [0.6466, 0.8382] +2022-04-12 08:33:54.731454: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 08:33:55.139373: lr: 3e-06 +2022-04-12 08:33:55.139580: saving scheduled checkpoint file... +2022-04-12 08:33:55.189398: saving checkpoint... +2022-04-12 08:33:56.087631: done, saving took 0.95 seconds +2022-04-12 08:33:56.095191: done +2022-04-12 08:34:26.783720: This epoch took 521.145902 s + +2022-04-12 08:34:26.784005: +epoch: 118 +2022-04-12 08:42:09.272538: train loss : 0.0911 +2022-04-12 08:42:35.604627: validation loss: 0.1132 +2022-04-12 08:42:35.605283: Average global foreground Dice: [0.653, 0.8752] +2022-04-12 08:42:35.605367: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 08:42:36.014820: lr: 2e-06 +2022-04-12 08:42:36.015008: saving scheduled checkpoint file... +2022-04-12 08:42:36.065012: saving checkpoint... +2022-04-12 08:42:36.967463: done, saving took 0.95 seconds +2022-04-12 08:42:36.974240: done +2022-04-12 08:43:25.452103: This epoch took 538.667975 s + +2022-04-12 08:43:25.452703: +epoch: 119 +2022-04-12 08:51:08.222655: train loss : 0.0946 +2022-04-12 08:51:34.558525: validation loss: 0.1305 +2022-04-12 08:51:34.559131: Average global foreground Dice: [0.6626, 0.8591] +2022-04-12 08:51:34.559214: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 08:51:34.983504: lr: 1e-06 +2022-04-12 08:51:34.983703: saving scheduled checkpoint file... +2022-04-12 08:51:35.033106: saving checkpoint... +2022-04-12 08:51:35.926189: done, saving took 0.94 seconds +2022-04-12 08:51:35.933503: done +2022-04-12 08:51:58.995100: This epoch took 513.542206 s + +2022-04-12 08:51:58.995444: +epoch: 120 +2022-04-12 08:59:41.557760: train loss : 0.0925 +2022-04-12 09:00:07.862391: validation loss: 0.0994 +2022-04-12 09:00:07.863036: Average global foreground Dice: [0.6634, 0.885] +2022-04-12 09:00:07.863123: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 09:00:08.269758: lr: 1e-06 +2022-04-12 09:00:08.269943: saving scheduled checkpoint file... +2022-04-12 09:00:08.319494: saving checkpoint... +2022-04-12 09:00:09.144474: done, saving took 0.87 seconds +2022-04-12 09:00:09.151906: done +2022-04-12 09:00:39.517333: This epoch took 520.521788 s + +2022-04-12 09:00:39.517647: +epoch: 121 +2022-04-12 09:08:22.503497: train loss : 0.0988 +2022-04-12 09:08:48.851639: validation loss: 0.1012 +2022-04-12 09:08:48.852184: Average global foreground Dice: [0.664, 0.8654] +2022-04-12 09:08:48.852267: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 09:08:49.256476: lr: 1e-06 +2022-04-12 09:08:49.256670: saving scheduled checkpoint file... +2022-04-12 09:08:49.305947: saving checkpoint... +2022-04-12 09:08:50.180798: done, saving took 0.92 seconds +2022-04-12 09:08:50.189090: done +2022-04-12 09:08:50.218187: saving checkpoint... +2022-04-12 09:08:51.111200: done, saving took 0.92 seconds +2022-04-12 09:09:13.634463: This epoch took 514.116712 s + +2022-04-12 09:09:13.634751: +epoch: 122 +2022-04-12 09:16:56.053530: train loss : 0.0965 +2022-04-12 09:17:22.378980: validation loss: 0.1227 +2022-04-12 09:17:22.379484: Average global foreground Dice: [0.6516, 0.881] +2022-04-12 09:17:22.379575: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 09:17:22.788263: lr: 0.0 +2022-04-12 09:17:22.788441: saving scheduled checkpoint file... +2022-04-12 09:17:22.837835: saving checkpoint... +2022-04-12 09:17:23.742558: done, saving took 0.95 seconds +2022-04-12 09:17:23.750092: done +2022-04-12 09:17:23.776498: saving checkpoint... +2022-04-12 09:17:24.676388: done, saving took 0.93 seconds +2022-04-12 09:17:53.596542: This epoch took 519.961684 s + +2022-04-12 09:17:53.596852: +epoch: 123 +2022-04-12 09:25:36.507234: train loss : 0.0970 +2022-04-12 09:26:02.863523: validation loss: 0.1171 +2022-04-12 09:26:02.864114: Average global foreground Dice: [0.6551, 0.8827] +2022-04-12 09:26:02.864197: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 09:26:03.272641: lr: 0.0 +2022-04-12 09:26:03.272827: saving scheduled checkpoint file... +2022-04-12 09:26:03.322485: saving checkpoint... +2022-04-12 09:26:04.107136: done, saving took 0.83 seconds +2022-04-12 09:26:04.128266: done +2022-04-12 09:26:04.154564: saving checkpoint... +2022-04-12 09:26:04.900265: done, saving took 0.77 seconds +2022-04-12 09:26:25.513794: This epoch took 511.916837 s + +2022-04-12 09:26:25.514117: +epoch: 124 +2022-04-12 09:34:08.067902: train loss : 0.0985 +2022-04-12 09:34:34.424941: validation loss: 0.1139 +2022-04-12 09:34:34.425489: Average global foreground Dice: [0.647, 0.8681] +2022-04-12 09:34:34.425583: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 09:34:34.832934: lr: 0.0 +2022-04-12 09:34:34.833128: saving scheduled checkpoint file... +2022-04-12 09:34:34.882842: saving checkpoint... +2022-04-12 09:34:35.771612: done, saving took 0.94 seconds +2022-04-12 09:34:35.779083: done +2022-04-12 09:35:15.461941: This epoch took 529.947685 s + +2022-04-12 09:35:15.520846: saving checkpoint... +2022-04-12 09:35:16.280211: done, saving took 0.82 seconds diff --git a/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_2/debug.json b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_2/debug.json new file mode 100644 index 0000000000000000000000000000000000000000..51b06e2c1fbe53f97c99a83ad468b12e514aaf56 --- /dev/null +++ b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_2/debug.json @@ -0,0 +1,106 @@ +{ + "all_tr_losses": "[0.3365588, 0.23576154, 0.20644811, 0.1925307, 0.17650223, 0.17113653, 0.15389562, 0.15411775, 0.14826968, 0.15403421, 0.1440485, 0.15402897, 0.14373074, 0.14662163, 0.14329961, 0.13333677, 0.14521024, 0.13089782, 0.13481817, 0.12698181, 0.13424017, 0.12435664, 0.12738469, 0.13200112, 0.1282572, 0.13086745, 0.12717266, 0.13209039, 0.11893695, 0.12243814, 0.12271249, 0.1244644, 0.120372586, 0.1278699, 0.1222408, 0.12434122, 0.121071704, 0.115636334, 0.12998971, 0.12240019, 0.117289364, 0.11434085, 0.1206092, 0.11047681, 0.11596647, 0.12526226, 0.116367556, 0.113776915, 0.11709769, 0.111778274, 0.11058851, 0.11627464, 0.11668352, 0.11725866, 0.114777334, 0.10902945, 0.11752333, 0.105974056, 0.11599809, 0.11460744, 0.11100236, 0.11232076, 0.11299875, 0.11548251, 0.105052, 0.10675668, 0.10548254, 0.11169742, 0.1080365, 0.11105766, 0.09920337, 0.103396885, 0.10165332, 0.10527045, 0.10346537, 0.108782746, 0.1044361, 0.09820995, 0.09989239, 0.09823276, 0.10304799, 0.100780904, 0.103387326, 0.09280554, 0.10075345, 0.09364079, 0.09474833, 0.097541355, 0.096883245, 0.09801272, 0.08907745, 0.094783954, 0.0900567]", + "all_val_eval_metrics": "[0.22362134851066784, 0.4681743519413855, 0.4763226412707299, 0.5314078850580991, 0.5855597511427186, 0.5573378363123547, 0.6144453233639567, 0.6361954419336993, 0.6678669186170278, 0.6636059055274977, 0.6609184350362479, 0.4689656073727865, 0.6380004375952435, 0.6937566438448917, 0.7016893401201112, 0.6953145003294778, 0.7078793778405844, 0.6964784489636958, 0.7039896611165055, 0.6837297054472242, 0.6976557239425736, 0.6623992889203882, 0.6938693226127461, 0.7319395860419855, 0.7048480263902688, 0.7034690476808928, 0.729154117172781, 0.7219168182442416, 0.7138226212201069, 0.7265563622597072, 0.7260280876057625, 0.7245606878544991, 0.7210457545958671, 0.7215736167545345, 0.7280911293911866, 0.7236171709093452, 0.7357761250722543, 0.7148970244206554, 0.7020775640464304, 0.7184920115418894, 0.7400005779958321, 0.7364482643579509, 0.7460976597333615, 0.7447956048099591, 0.7277171365864633, 0.7174136244776235, 0.7332727298428101, 0.7119007579894789, 0.728118948650897, 0.7250240898958595, 0.7264547569968802, 0.7479807567826615, 0.7271300484412927, 0.7364948036735464, 0.7507829211833852, 0.7431158993354015, 0.7276440741421553, 0.7441998454281455, 0.7348914376561888, 0.7441568384881951, 0.7417610191775222, 0.7325026434681247, 0.7512153099450354, 0.7530621122398669, 0.7566591009206811, 0.7543712305846757, 0.7504239165067992, 0.739429560895169, 0.7417044956448609, 0.7428402452453265, 0.7254917800449425, 0.7510673725543584, 0.7486325422270188, 0.7520073425038944, 0.7528195565482716, 0.7385722561634516, 0.7417615219162972, 0.7589276364374139, 0.7501715194456285, 0.7511627793027655, 0.7336959785489074, 0.7346514273128679, 0.738454071100811, 0.7210844771792085, 0.7519652776191077, 0.7551270491149819, 0.7525788386506502, 0.7324255515490765, 0.7332737453884888, 0.7389992681438778, 0.719622482794996, 0.7474999850059756, 0.7512417061933725]", + "all_val_losses": "[0.27353683, 0.21563168, 0.22545616, 0.17471375, 0.16070622, 0.16916835, 0.14368363, 0.14897588, 0.14481923, 0.14669828, 0.16085072, 0.17799395, 0.13749734, 0.1578951, 0.1339963, 0.13933295, 0.13721834, 0.14562169, 0.13580345, 0.14007284, 0.12492545, 0.15388352, 0.12720071, 0.13041332, 0.1352024, 0.12542722, 0.12870632, 0.14012887, 0.11813672, 0.11324752, 0.123189956, 0.12444647, 0.13844359, 0.12112669, 0.119424894, 0.13957475, 0.11372627, 0.13046922, 0.13683175, 0.13842992, 0.122694634, 0.12575035, 0.10276161, 0.12905096, 0.13321823, 0.12823167, 0.1158135, 0.14339434, 0.122966096, 0.121158324, 0.12623544, 0.11132485, 0.10824719, 0.12059457, 0.12660325, 0.13544, 0.13883731, 0.12793529, 0.13314013, 0.116851464, 0.11731087, 0.14206949, 0.13704526, 0.115740165, 0.10180616, 0.11520993, 0.1501946, 0.1139482, 0.12092621, 0.1256443, 0.11076236, 0.10439913, 0.107668, 0.11303625, 0.12650496, 0.11198064, 0.13196927, 0.10110061, 0.113442555, 0.11976511, 0.11249426, 0.109426945, 0.12533823, 0.11059702, 0.10330965, 0.11261508, 0.11954655, 0.11479926, 0.109553464, 0.12314592, 0.11573183, 0.1017501, 0.100643724]", + "all_val_losses_tr_mode": "[]", + "also_val_in_tr_mode": "False", + "amp_grad_scaler": "", + "args": "Namespace(batch_size=2, config='./configs/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32.yaml', continue_training=False, crop_size=[64, 192, 192], dbs=False, deep_supervision_scales=[], deterministic=False, disable_decoder=False, disable_ds=True, disable_postprocessing_on_folds=False, disable_saving=False, drop_path=0.0, find_lr=False, find_zero_weight_decay=False, fix_ds_net_numpool=False, fold='2', fp32=False, hdfs_base='GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32', initial_lr=0.0003, is_sigmoid=False, is_spatial_aug_only=False, is_ssl=False, layer_decay=1.0, local_rank=0, loss_name='', lrschedule='warmup_cosine', mask_ratio=0.75, max_loss_cal='v1', max_num_epochs=125, merge_femur=False, min_lr=0, model='Generic_TransUNet_max_ppbp', model_params={'is_masked_attn': True, 'max_dec_layers': 3, 'is_max_bottleneck_transformer': True, 'vit_depth': 1, 'max_msda': '', 'is_max_ms': True, 'max_ms_idxs': [-4, -3, -2], 'max_hidden_dim': 192, 'mw': 1.0, 'is_max_ds': True, 'is_masking': True, 'is_max_hungarian': True, 'num_queries': 20, 'is_max_cls': True, 'is_mhsa_float32': True}, n_class=17, network='3d_fullres', network_trainer='nnUNetTrainerV2_DDP', npz=False, opt_betas=None, opt_eps=1e-08, optim_name='adamw', p='nnUNetPlansv2.1', plan_update='', pretrained=False, pretrained_config=None, pretrained_weights=None, rank=0, reclip=None, resume='auto', skip_grad_nan=False, task='Task008_HepaticVessel', task_pretrained='Task801_WORD', total_batch_size=None, use_compressed_data=False, val_final=False, val_folder='validation_raw', valbest=False, validation_only=False, vallatest=False, warmup_epochs=10, weight_decay=3e-05, world_size=8)", + "base_num_features": "32", + "basic_generator_patch_size": "[178 308 233]", + "batch_dice": "True", + "batch_size": "2", + "best_MA_tr_loss_for_patience": "None", + "best_epoch_based_on_MA_tr_loss": "None", + "best_val_eval_criterion_MA": "None", + "classes": "[1, 2]", + "conv_per_stage": "2", + "data_aug_params": "{'selected_data_channels': None, 'selected_seg_channels': [0], 'do_elastic': False, 'elastic_deform_alpha': (0.0, 900.0), 'elastic_deform_sigma': (9.0, 13.0), 'p_eldef': 0.2, 'do_scaling': True, 'scale_range': (0.7, 1.4), 'independent_scale_factor_for_each_axis': False, 'p_independent_scale_per_axis': 1, 'p_scale': 0.2, 'do_rotation': True, 'rotation_x': (-0.5235987755982988, 0.5235987755982988), 'rotation_y': (-0.5235987755982988, 0.5235987755982988), 'rotation_z': (-0.5235987755982988, 0.5235987755982988), 'rotation_p_per_axis': 1, 'p_rot': 0.2, 'random_crop': False, 'random_crop_dist_to_border': None, 'do_gamma': True, 'gamma_retain_stats': True, 'gamma_range': (0.7, 1.5), 'p_gamma': 0.3, 'do_mirror': True, 'mirror_axes': (0, 1, 2), 'dummy_2D': False, 'mask_was_used_for_normalization': OrderedDict([(0, False)]), 'border_mode_data': 'constant', 'all_segmentation_labels': None, 'move_last_seg_chanel_to_data': False, 'cascade_do_cascade_augmentations': False, 'cascade_random_binary_transform_p': 0.4, 'cascade_random_binary_transform_p_per_label': 1, 'cascade_random_binary_transform_size': (1, 8), 'cascade_remove_conn_comp_p': 0.2, 'cascade_remove_conn_comp_max_size_percent_threshold': 0.15, 'cascade_remove_conn_comp_fill_with_other_class_p': 0.0, 'do_additive_brightness': False, 'additive_brightness_p_per_sample': 0.15, 'additive_brightness_p_per_channel': 0.5, 'additive_brightness_mu': 0.0, 'additive_brightness_sigma': 0.1, 'num_threads': 12, 'num_cached_per_thread': 2, 'patch_size_for_spatialtransform': array([ 64, 192, 192])}", + "dataset_directory": "/opt/tiger/project/data/nnUNet_preprocessed/Task008_HepaticVessel", + "deep_supervision_scales": "[[1, 1, 1], [0.5, 0.5, 0.5], [0.25, 0.25, 0.25], [0.125, 0.125, 0.125], [0.0625, 0.0625, 0.0625]]", + "deterministic": "False", + "disable_ds": "True", + "distribute_batch_size": "False", + "dl_tr": "", + "dl_val": "", + "do_dummy_2D_aug": "False", + "ds_loss_weights": "[1. 0. 0. 0. 0.]", + "epoch": "93", + "experiment_name": "nnUNetTrainerV2_DDP", + "find_zero_weight_decay": "False", + "fold": "2", + "folder_with_preprocessed_data": "/opt/tiger/project/data/nnUNet_preprocessed/Task008_HepaticVessel/nnUNetData_plans_v2.1_stage1", + "fp16": "True", + "global_batch_size": "16", + "gt_niftis_folder": "/opt/tiger/project/data/nnUNet_preprocessed/Task008_HepaticVessel/gt_segmentations", + "inference_pad_border_mode": "constant", + "inference_pad_kwargs": "{'constant_values': 0}", + "init_args": "('/opt/tiger/project/data/nnUNet_preprocessed/Task008_HepaticVessel/nnUNetPlansv2.1_plans_3D.pkl', 2, 0, '/opt/tiger/project/data/nnUNet_trained_models/UNet_IN_NANFang/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32', '/opt/tiger/project/data/nnUNet_preprocessed/Task008_HepaticVessel', True, 1, True, False, False, True)", + "initial_lr": "0.0003", + "input_size": "[64, 192, 192]", + "is_spatial_aug_only": "False", + "is_ssl": "False", + "layer_decay": "1.0", + "local_rank": "0", + "log_file": "/opt/tiger/project/data/nnUNet_trained_models/UNet_IN_NANFang/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_2/training_log_2022_5_27_13_08_39.txt", + "loss": "None", + "lr_scheduler": "", + "lr_scheduler_eps": "0.001", + "lr_scheduler_name": "warmup_cosine", + "lr_scheduler_patience": "30", + "lr_threshold": "1e-06", + "max_num_epochs": "125", + "min_lr": "0", + "min_region_size_per_class": "None", + "min_size_per_class": "None", + "model": "Generic_TransUNet_max_ppbp", + "model_params": "{'is_masked_attn': True, 'max_dec_layers': 3, 'is_max_bottleneck_transformer': True, 'vit_depth': 1, 'max_msda': '', 'is_max_ms': True, 'max_ms_idxs': [-4, -3, -2], 'max_hidden_dim': 192, 'mw': 1.0, 'is_max_ds': True, 'is_masking': True, 'is_max_hungarian': True, 'num_queries': 20, 'is_max_cls': True, 'is_mhsa_float32': True}", + "net_conv_kernel_sizes": "[[3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3]]", + "net_num_pool_op_kernel_sizes": "[[2, 2, 2], [2, 2, 2], [2, 2, 2], [2, 2, 2], [1, 2, 2]]", + "net_pool_per_axis": "[4, 5, 5]", + "normalization_schemes": "OrderedDict([(0, 'CT')])", + "num_batches_per_epoch": "250", + "num_classes": "3", + "num_input_channels": "1", + "num_val_batches_per_epoch": "50", + "online_eval_fn": "[]", + "online_eval_foreground_dc": "[]", + "online_eval_fp": "[]", + "online_eval_tp": "[]", + "only_keep_largest_connected_component": "None", + "optim_name": "adamw", + "optimizer": "AdamW (\nParameter Group 0\n amsgrad: False\n betas: (0.9, 0.999)\n eps: 1e-08\n initial_lr: 0.0003\n lr: 5.375636079717583e-05\n weight_decay: 3e-05\n)", + "output_folder": "/opt/tiger/project/data/nnUNet_trained_models/UNet_IN_NANFang/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_2", + "output_folder_base": "/opt/tiger/project/data/nnUNet_trained_models/UNet_IN_NANFang/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32", + "oversample_foreground_percent": "0.0", + "pad_all_sides": "None", + "patch_size": "[ 64 192 192]", + "patience": "50", + "pin_memory": "True", + "plans_file": "/opt/tiger/project/data/nnUNet_preprocessed/Task008_HepaticVessel/nnUNetPlansv2.1_plans_3D.pkl", + "reclip": "None", + "regions_class_order": "None", + "resume": "auto", + "save_best_checkpoint": "True", + "save_every": "1", + "save_final_checkpoint": "True", + "save_intermediate_checkpoints": "True", + "save_latest_only": "True", + "stage": "1", + "threeD": "True", + "tr_gen": "", + "train_loss_MA": "None", + "train_loss_MA_alpha": "0.93", + "train_loss_MA_eps": "0.0005", + "transpose_backward": "[0, 1, 2]", + "transpose_forward": "[0, 1, 2]", + "unpack_data": "True", + "use_mask_for_norm": "OrderedDict([(0, False)])", + "use_progress_bar": "False", + "val_eval_criterion_MA": "None", + "val_eval_criterion_alpha": "0.9", + "val_gen": "", + "warmup_epochs": "10", + "was_initialized": "True", + "weight_decay": "3e-05" +} \ No newline at end of file diff --git a/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_2/model_best.model b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_2/model_best.model new file mode 100644 index 0000000000000000000000000000000000000000..b4c3434e7b8e323b158fa286f33e4b78bb6ace89 --- /dev/null +++ b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_2/model_best.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6fc7152cc5e27f4e249045e3e98919a5c20655607eb191d22fca620da37b48dd +size 496524090 diff --git a/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_2/model_best.model.pkl b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_2/model_best.model.pkl new file mode 100644 index 0000000000000000000000000000000000000000..61054c44139ee7cd51203fe7a4168c9ea94b610a --- /dev/null +++ b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_2/model_best.model.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a492b355d6b010096d22ee9fd14ac1404f535476bae2db246e26e0d73067aa2f +size 188932 diff --git a/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_2/model_final_checkpoint.model b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_2/model_final_checkpoint.model new file mode 100644 index 0000000000000000000000000000000000000000..195b53b0db9b7b7ebec9acf1f40007cfc8916671 --- /dev/null +++ b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_2/model_final_checkpoint.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1e02586e63084544c716d993cd695a3a03341818bd602b274da8d84e2b9fb31e +size 496529466 diff --git a/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_2/model_final_checkpoint.model.pkl b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_2/model_final_checkpoint.model.pkl new file mode 100644 index 0000000000000000000000000000000000000000..61054c44139ee7cd51203fe7a4168c9ea94b610a --- /dev/null +++ b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_2/model_final_checkpoint.model.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a492b355d6b010096d22ee9fd14ac1404f535476bae2db246e26e0d73067aa2f +size 188932 diff --git a/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_2/model_latest.model b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_2/model_latest.model new file mode 100644 index 0000000000000000000000000000000000000000..04d92de4c5e633603123dee30ffabac5db5c10a8 --- /dev/null +++ b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_2/model_latest.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f3d23b0960aaccc552a5d98571138a4a0bcb004acbd1046712386270088d21e +size 496529466 diff --git a/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_2/model_latest.model.pkl b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_2/model_latest.model.pkl new file mode 100644 index 0000000000000000000000000000000000000000..61054c44139ee7cd51203fe7a4168c9ea94b610a --- /dev/null +++ b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_2/model_latest.model.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a492b355d6b010096d22ee9fd14ac1404f535476bae2db246e26e0d73067aa2f +size 188932 diff --git a/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_2/progress.png b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_2/progress.png new file mode 100644 index 0000000000000000000000000000000000000000..4f17f17dd967ae56c3048fdc0d7a9dc94375116a --- /dev/null +++ b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_2/progress.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4d0c78175c33c37fe1990847537111280a5da372080157fd972fb255d5c901ea +size 299572 diff --git a/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_2/training_log_2022_5_27_13_08_39.txt b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_2/training_log_2022_5_27_13_08_39.txt new file mode 100644 index 0000000000000000000000000000000000000000..486a8b55bc7e68b4e89e16f7372942fd1d9eac2d --- /dev/null +++ b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_2/training_log_2022_5_27_13_08_39.txt @@ -0,0 +1,433 @@ +Starting... +2022-05-27 13:08:39.986044: Using splits from existing split file: /opt/tiger/project/data/nnUNet_preprocessed/Task008_HepaticVessel/splits_final.pkl +2022-05-27 13:08:39.986553: The split file contains 5 splits. +2022-05-27 13:08:39.986617: Desired fold for training: 2 +2022-05-27 13:08:39.986657: This split has 242 training and 61 validation cases. +2022-05-27 13:10:18.506199: TRAINING KEYS: + odict_keys(['hepaticvessel_001', 'hepaticvessel_005', 'hepaticvessel_008', 'hepaticvessel_010', 'hepaticvessel_011', 'hepaticvessel_013', 'hepaticvessel_016', 'hepaticvessel_018', 'hepaticvessel_019', 'hepaticvessel_020', 'hepaticvessel_023', 'hepaticvessel_025', 'hepaticvessel_026', 'hepaticvessel_027', 'hepaticvessel_028', 'hepaticvessel_029', 'hepaticvessel_030', 'hepaticvessel_031', 'hepaticvessel_032', 'hepaticvessel_033', 'hepaticvessel_044', 'hepaticvessel_049', 'hepaticvessel_050', 'hepaticvessel_052', 'hepaticvessel_053', 'hepaticvessel_057', 'hepaticvessel_058', 'hepaticvessel_059', 'hepaticvessel_061', 'hepaticvessel_062', 'hepaticvessel_065', 'hepaticvessel_066', 'hepaticvessel_067', 'hepaticvessel_070', 'hepaticvessel_071', 'hepaticvessel_072', 'hepaticvessel_075', 'hepaticvessel_077', 'hepaticvessel_079', 'hepaticvessel_080', 'hepaticvessel_082', 'hepaticvessel_083', 'hepaticvessel_084', 'hepaticvessel_085', 'hepaticvessel_087', 'hepaticvessel_088', 'hepaticvessel_089', 'hepaticvessel_090', 'hepaticvessel_092', 'hepaticvessel_093', 'hepaticvessel_094', 'hepaticvessel_096', 'hepaticvessel_098', 'hepaticvessel_100', 'hepaticvessel_102', 'hepaticvessel_103', 'hepaticvessel_104', 'hepaticvessel_108', 'hepaticvessel_110', 'hepaticvessel_111', 'hepaticvessel_112', 'hepaticvessel_113', 'hepaticvessel_115', 'hepaticvessel_119', 'hepaticvessel_121', 'hepaticvessel_123', 'hepaticvessel_124', 'hepaticvessel_127', 'hepaticvessel_129', 'hepaticvessel_131', 'hepaticvessel_132', 'hepaticvessel_133', 'hepaticvessel_136', 'hepaticvessel_142', 'hepaticvessel_145', 'hepaticvessel_146', 'hepaticvessel_147', 'hepaticvessel_148', 'hepaticvessel_150', 'hepaticvessel_154', 'hepaticvessel_157', 'hepaticvessel_158', 'hepaticvessel_159', 'hepaticvessel_161', 'hepaticvessel_163', 'hepaticvessel_164', 'hepaticvessel_165', 'hepaticvessel_166', 'hepaticvessel_167', 'hepaticvessel_171', 'hepaticvessel_173', 'hepaticvessel_174', 'hepaticvessel_175', 'hepaticvessel_177', 'hepaticvessel_179', 'hepaticvessel_181', 'hepaticvessel_183', 'hepaticvessel_185', 'hepaticvessel_186', 'hepaticvessel_189', 'hepaticvessel_192', 'hepaticvessel_194', 'hepaticvessel_195', 'hepaticvessel_196', 'hepaticvessel_198', 'hepaticvessel_199', 'hepaticvessel_200', 'hepaticvessel_206', 'hepaticvessel_207', 'hepaticvessel_208', 'hepaticvessel_210', 'hepaticvessel_211', 'hepaticvessel_213', 'hepaticvessel_215', 'hepaticvessel_217', 'hepaticvessel_218', 'hepaticvessel_219', 'hepaticvessel_221', 'hepaticvessel_222', 'hepaticvessel_223', 'hepaticvessel_224', 'hepaticvessel_225', 'hepaticvessel_227', 'hepaticvessel_230', 'hepaticvessel_231', 'hepaticvessel_234', 'hepaticvessel_235', 'hepaticvessel_237', 'hepaticvessel_240', 'hepaticvessel_242', 'hepaticvessel_244', 'hepaticvessel_245', 'hepaticvessel_246', 'hepaticvessel_248', 'hepaticvessel_255', 'hepaticvessel_256', 'hepaticvessel_258', 'hepaticvessel_259', 'hepaticvessel_262', 'hepaticvessel_266', 'hepaticvessel_269', 'hepaticvessel_270', 'hepaticvessel_271', 'hepaticvessel_272', 'hepaticvessel_274', 'hepaticvessel_275', 'hepaticvessel_278', 'hepaticvessel_279', 'hepaticvessel_281', 'hepaticvessel_282', 'hepaticvessel_284', 'hepaticvessel_285', 'hepaticvessel_287', 'hepaticvessel_290', 'hepaticvessel_291', 'hepaticvessel_293', 'hepaticvessel_294', 'hepaticvessel_296', 'hepaticvessel_297', 'hepaticvessel_299', 'hepaticvessel_300', 'hepaticvessel_303', 'hepaticvessel_305', 'hepaticvessel_308', 'hepaticvessel_309', 'hepaticvessel_314', 'hepaticvessel_319', 'hepaticvessel_320', 'hepaticvessel_321', 'hepaticvessel_323', 'hepaticvessel_324', 'hepaticvessel_325', 'hepaticvessel_327', 'hepaticvessel_329', 'hepaticvessel_330', 'hepaticvessel_332', 'hepaticvessel_333', 'hepaticvessel_335', 'hepaticvessel_336', 'hepaticvessel_340', 'hepaticvessel_341', 'hepaticvessel_344', 'hepaticvessel_345', 'hepaticvessel_348', 'hepaticvessel_349', 'hepaticvessel_350', 'hepaticvessel_356', 'hepaticvessel_358', 'hepaticvessel_359', 'hepaticvessel_361', 'hepaticvessel_362', 'hepaticvessel_363', 'hepaticvessel_368', 'hepaticvessel_369', 'hepaticvessel_371', 'hepaticvessel_372', 'hepaticvessel_373', 'hepaticvessel_374', 'hepaticvessel_375', 'hepaticvessel_378', 'hepaticvessel_379', 'hepaticvessel_383', 'hepaticvessel_384', 'hepaticvessel_386', 'hepaticvessel_388', 'hepaticvessel_389', 'hepaticvessel_391', 'hepaticvessel_393', 'hepaticvessel_396', 'hepaticvessel_397', 'hepaticvessel_400', 'hepaticvessel_401', 'hepaticvessel_402', 'hepaticvessel_404', 'hepaticvessel_406', 'hepaticvessel_407', 'hepaticvessel_408', 'hepaticvessel_409', 'hepaticvessel_411', 'hepaticvessel_412', 'hepaticvessel_416', 'hepaticvessel_422', 'hepaticvessel_424', 'hepaticvessel_425', 'hepaticvessel_427', 'hepaticvessel_429', 'hepaticvessel_432', 'hepaticvessel_434', 'hepaticvessel_437', 'hepaticvessel_438', 'hepaticvessel_440', 'hepaticvessel_441', 'hepaticvessel_443', 'hepaticvessel_444', 'hepaticvessel_445', 'hepaticvessel_446', 'hepaticvessel_447', 'hepaticvessel_453', 'hepaticvessel_454', 'hepaticvessel_455', 'hepaticvessel_456', 'hepaticvessel_458']) +2022-05-27 13:10:18.506382: VALIDATION KEYS: + odict_keys(['hepaticvessel_002', 'hepaticvessel_004', 'hepaticvessel_007', 'hepaticvessel_022', 'hepaticvessel_039', 'hepaticvessel_040', 'hepaticvessel_042', 'hepaticvessel_046', 'hepaticvessel_051', 'hepaticvessel_068', 'hepaticvessel_078', 'hepaticvessel_081', 'hepaticvessel_086', 'hepaticvessel_091', 'hepaticvessel_097', 'hepaticvessel_116', 'hepaticvessel_117', 'hepaticvessel_125', 'hepaticvessel_128', 'hepaticvessel_137', 'hepaticvessel_138', 'hepaticvessel_139', 'hepaticvessel_140', 'hepaticvessel_141', 'hepaticvessel_144', 'hepaticvessel_151', 'hepaticvessel_160', 'hepaticvessel_172', 'hepaticvessel_178', 'hepaticvessel_184', 'hepaticvessel_193', 'hepaticvessel_197', 'hepaticvessel_201', 'hepaticvessel_203', 'hepaticvessel_209', 'hepaticvessel_214', 'hepaticvessel_229', 'hepaticvessel_233', 'hepaticvessel_236', 'hepaticvessel_238', 'hepaticvessel_263', 'hepaticvessel_265', 'hepaticvessel_268', 'hepaticvessel_280', 'hepaticvessel_286', 'hepaticvessel_289', 'hepaticvessel_307', 'hepaticvessel_318', 'hepaticvessel_322', 'hepaticvessel_338', 'hepaticvessel_353', 'hepaticvessel_367', 'hepaticvessel_382', 'hepaticvessel_385', 'hepaticvessel_398', 'hepaticvessel_419', 'hepaticvessel_420', 'hepaticvessel_423', 'hepaticvessel_431', 'hepaticvessel_433', 'hepaticvessel_442']) +2022-05-27 13:10:26.430544: loading checkpoint /opt/tiger/project/data/nnUNet_trained_models/UNet_IN_NANFang/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_2/model_latest.model train= True +2022-05-27 13:10:27.457920: lr: 5.1e-05 +2022-05-27 13:10:40.519020: +epoch: 93 +2022-05-27 13:18:19.066530: train loss : 0.0870 +2022-05-27 13:18:43.959040: validation loss: 0.1025 +2022-05-27 13:18:43.959574: Average global foreground Dice: [0.6451, 0.8471] +2022-05-27 13:18:43.959681: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-27 13:18:44.396659: lr: 5.1e-05 +2022-05-27 13:18:44.396825: saving scheduled checkpoint file... +2022-05-27 13:18:44.469290: saving checkpoint... +2022-05-27 13:18:45.371220: done, saving took 0.97 seconds +2022-05-27 13:18:45.380165: done +2022-05-27 13:18:54.655742: This epoch took 494.134558 s + +2022-05-27 13:18:54.655977: +epoch: 94 +2022-05-27 13:26:21.435696: train loss : 0.0894 +2022-05-27 13:26:46.319200: validation loss: 0.1118 +2022-05-27 13:26:46.319757: Average global foreground Dice: [0.641, 0.8675] +2022-05-27 13:26:46.319873: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-27 13:26:46.747636: lr: 4.8e-05 +2022-05-27 13:26:46.747775: saving scheduled checkpoint file... +2022-05-27 13:26:46.807386: saving checkpoint... +2022-05-27 13:26:47.676966: done, saving took 0.93 seconds +2022-05-27 13:26:47.684530: done +2022-05-27 13:26:47.703182: saving checkpoint... +2022-05-27 13:26:48.314137: done, saving took 0.63 seconds +2022-05-27 13:27:26.059867: This epoch took 511.403738 s + +2022-05-27 13:27:26.060159: +epoch: 95 +2022-05-27 13:34:52.701140: train loss : 0.0913 +2022-05-27 13:35:17.599377: validation loss: 0.1196 +2022-05-27 13:35:17.600002: Average global foreground Dice: [0.6529, 0.8528] +2022-05-27 13:35:17.600099: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-27 13:35:18.026276: lr: 4.5e-05 +2022-05-27 13:35:18.026423: saving scheduled checkpoint file... +2022-05-27 13:35:18.085472: saving checkpoint... +2022-05-27 13:35:18.935143: done, saving took 0.91 seconds +2022-05-27 13:35:18.942611: done +2022-05-27 13:35:18.959168: saving checkpoint... +2022-05-27 13:35:19.768434: done, saving took 0.83 seconds +2022-05-27 13:35:42.514996: This epoch took 496.454692 s + +2022-05-27 13:35:42.515265: +epoch: 96 +2022-05-27 13:43:09.050013: train loss : 0.0932 +2022-05-27 13:43:33.913221: validation loss: 0.1126 +2022-05-27 13:43:33.913707: Average global foreground Dice: [0.6313, 0.8374] +2022-05-27 13:43:33.913803: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-27 13:43:34.343753: lr: 4.2e-05 +2022-05-27 13:43:34.343904: saving scheduled checkpoint file... +2022-05-27 13:43:34.402983: saving checkpoint... +2022-05-27 13:43:35.347799: done, saving took 1.00 seconds +2022-05-27 13:43:35.356315: done +2022-05-27 13:44:00.653875: This epoch took 498.138493 s + +2022-05-27 13:44:00.654253: +epoch: 97 +2022-05-27 13:51:27.551327: train loss : 0.0927 +2022-05-27 13:51:52.410709: validation loss: 0.1108 +2022-05-27 13:51:52.411217: Average global foreground Dice: [0.6421, 0.8218] +2022-05-27 13:51:52.411311: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-27 13:51:52.840672: lr: 3.9e-05 +2022-05-27 13:51:52.840822: saving scheduled checkpoint file... +2022-05-27 13:51:52.900112: saving checkpoint... +2022-05-27 13:51:53.833024: done, saving took 0.99 seconds +2022-05-27 13:51:53.841214: done +2022-05-27 13:52:29.609810: This epoch took 508.955400 s + +2022-05-27 13:52:29.610171: +epoch: 98 +2022-05-27 13:59:56.230547: train loss : 0.0936 +2022-05-27 14:00:21.135344: validation loss: 0.1229 +2022-05-27 14:00:21.135980: Average global foreground Dice: [0.638, 0.8422] +2022-05-27 14:00:21.136074: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-27 14:00:21.565876: lr: 3.6e-05 +2022-05-27 14:00:21.566022: saving scheduled checkpoint file... +2022-05-27 14:00:21.625840: saving checkpoint... +2022-05-27 14:00:22.549603: done, saving took 0.98 seconds +2022-05-27 14:00:22.557997: done +2022-05-27 14:00:43.473881: This epoch took 493.863556 s + +2022-05-27 14:00:43.474148: +epoch: 99 +2022-05-27 14:08:10.037102: train loss : 0.0854 +2022-05-27 14:08:34.928749: validation loss: 0.1092 +2022-05-27 14:08:34.929256: Average global foreground Dice: [0.6473, 0.8002] +2022-05-27 14:08:34.929346: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-27 14:08:35.356624: lr: 3.4e-05 +2022-05-27 14:08:35.356843: saving scheduled checkpoint file... +2022-05-27 14:08:35.416116: saving checkpoint... +2022-05-27 14:08:36.249519: done, saving took 0.89 seconds +2022-05-27 14:08:36.258090: done +2022-05-27 14:09:10.303372: This epoch took 506.829124 s + +2022-05-27 14:09:10.303654: +epoch: 100 +2022-05-27 14:16:37.200175: train loss : 0.0900 +2022-05-27 14:17:02.112207: validation loss: 0.1010 +2022-05-27 14:17:02.112753: Average global foreground Dice: [0.6444, 0.8565] +2022-05-27 14:17:02.112850: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-27 14:17:02.541251: lr: 3.1e-05 +2022-05-27 14:17:02.541404: saving scheduled checkpoint file... +2022-05-27 14:17:02.601057: saving checkpoint... +2022-05-27 14:17:03.433770: done, saving took 0.89 seconds +2022-05-27 14:17:03.441698: done +2022-05-27 14:17:57.787756: This epoch took 527.483940 s + +2022-05-27 14:17:57.788044: +epoch: 101 +2022-05-27 14:25:24.351850: train loss : 0.0935 +2022-05-27 14:25:49.254310: validation loss: 0.1036 +2022-05-27 14:25:49.254805: Average global foreground Dice: [0.6424, 0.8513] +2022-05-27 14:25:49.254899: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-27 14:25:49.694954: lr: 2.9e-05 +2022-05-27 14:25:49.695111: saving scheduled checkpoint file... +2022-05-27 14:25:49.754292: saving checkpoint... +2022-05-27 14:25:50.732259: done, saving took 1.04 seconds +2022-05-27 14:25:50.740791: done +2022-05-27 14:26:16.473124: This epoch took 498.684963 s + +2022-05-27 14:26:16.473398: +epoch: 102 +2022-05-27 14:33:43.073627: train loss : 0.0956 +2022-05-27 14:34:07.982656: validation loss: 0.1130 +2022-05-27 14:34:07.983187: Average global foreground Dice: [0.6481, 0.8496] +2022-05-27 14:34:07.983282: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-27 14:34:08.414323: lr: 2.6e-05 +2022-05-27 14:34:08.414468: saving scheduled checkpoint file... +2022-05-27 14:34:08.474471: saving checkpoint... +2022-05-27 14:34:09.396829: done, saving took 0.98 seconds +2022-05-27 14:34:09.405622: done +2022-05-27 14:34:25.851990: This epoch took 489.378482 s + +2022-05-27 14:34:25.852248: +epoch: 103 +2022-05-27 14:41:52.325432: train loss : 0.0935 +2022-05-27 14:42:17.229959: validation loss: 0.1222 +2022-05-27 14:42:17.230487: Average global foreground Dice: [0.6314, 0.8707] +2022-05-27 14:42:17.230581: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-27 14:42:17.663993: lr: 2.4e-05 +2022-05-27 14:42:17.664150: saving scheduled checkpoint file... +2022-05-27 14:42:17.723634: saving checkpoint... +2022-05-27 14:42:18.676801: done, saving took 1.01 seconds +2022-05-27 14:42:18.685125: done +2022-05-27 14:42:38.120981: This epoch took 492.268629 s + +2022-05-27 14:42:38.121239: +epoch: 104 +2022-05-27 14:50:05.018342: train loss : 0.0996 +2022-05-27 14:50:29.892182: validation loss: 0.1046 +2022-05-27 14:50:29.892713: Average global foreground Dice: [0.6237, 0.8687] +2022-05-27 14:50:29.892809: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-27 14:50:30.323799: lr: 2.2e-05 +2022-05-27 14:50:30.323951: saving scheduled checkpoint file... +2022-05-27 14:50:30.383751: saving checkpoint... +2022-05-27 14:50:31.352848: done, saving took 1.03 seconds +2022-05-27 14:50:31.362734: done +2022-05-27 14:50:54.676013: This epoch took 496.554671 s + +2022-05-27 14:50:54.676284: +epoch: 105 +2022-05-27 14:58:21.394512: train loss : 0.0953 +2022-05-27 14:58:46.286985: validation loss: 0.1110 +2022-05-27 14:58:46.287517: Average global foreground Dice: [0.6502, 0.8511] +2022-05-27 14:58:46.287616: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-27 14:58:46.716838: lr: 2e-05 +2022-05-27 14:58:46.716985: saving scheduled checkpoint file... +2022-05-27 14:58:46.776605: saving checkpoint... +2022-05-27 14:58:47.710517: done, saving took 0.99 seconds +2022-05-27 14:58:47.719355: done +2022-05-27 14:59:14.278253: This epoch took 499.601860 s + +2022-05-27 14:59:14.278527: +epoch: 106 +2022-05-27 15:06:41.096495: train loss : 0.0979 +2022-05-27 15:07:06.004962: validation loss: 0.1237 +2022-05-27 15:07:06.005507: Average global foreground Dice: [0.6348, 0.8571] +2022-05-27 15:07:06.005607: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-27 15:07:06.434954: lr: 1.8e-05 +2022-05-27 15:07:06.435095: saving scheduled checkpoint file... +2022-05-27 15:07:06.494618: saving checkpoint... +2022-05-27 15:07:07.418174: done, saving took 0.98 seconds +2022-05-27 15:07:07.426654: done +2022-05-27 15:07:32.592615: This epoch took 498.313931 s + +2022-05-27 15:07:32.592895: +epoch: 107 +2022-05-27 15:14:59.477870: train loss : 0.0961 +2022-05-27 15:15:24.370386: validation loss: 0.1084 +2022-05-27 15:15:24.370879: Average global foreground Dice: [0.6438, 0.8656] +2022-05-27 15:15:24.370977: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-27 15:15:24.805041: lr: 1.6e-05 +2022-05-27 15:15:24.805193: saving scheduled checkpoint file... +2022-05-27 15:15:24.865577: saving checkpoint... +2022-05-27 15:15:25.822587: done, saving took 1.02 seconds +2022-05-27 15:15:25.831582: done +2022-05-27 15:15:58.220585: This epoch took 505.627577 s + +2022-05-27 15:15:58.220893: +epoch: 108 +2022-05-27 15:23:25.215914: train loss : 0.0928 +2022-05-27 15:23:50.114080: validation loss: 0.1173 +2022-05-27 15:23:50.114579: Average global foreground Dice: [0.6301, 0.816] +2022-05-27 15:23:50.114680: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-27 15:23:50.550763: lr: 1.4e-05 +2022-05-27 15:23:50.550915: saving scheduled checkpoint file... +2022-05-27 15:23:50.610373: saving checkpoint... +2022-05-27 15:23:51.546987: done, saving took 1.00 seconds +2022-05-27 15:23:51.555466: done +2022-05-27 15:24:08.393326: This epoch took 490.172274 s + +2022-05-27 15:24:08.393613: +epoch: 109 +2022-05-27 15:31:35.154878: train loss : 0.1009 +2022-05-27 15:32:00.058503: validation loss: 0.1079 +2022-05-27 15:32:00.059041: Average global foreground Dice: [0.636, 0.8623] +2022-05-27 15:32:00.059134: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-27 15:32:00.486300: lr: 1.2e-05 +2022-05-27 15:32:00.486473: saving scheduled checkpoint file... +2022-05-27 15:32:00.545723: saving checkpoint... +2022-05-27 15:32:01.397580: done, saving took 0.91 seconds +2022-05-27 15:32:01.404934: done +2022-05-27 15:32:21.368201: This epoch took 492.974409 s + +2022-05-27 15:32:21.368479: +epoch: 110 +2022-05-27 15:39:48.174214: train loss : 0.0919 +2022-05-27 15:40:13.072838: validation loss: 0.1280 +2022-05-27 15:40:13.073381: Average global foreground Dice: [0.6295, 0.8385] +2022-05-27 15:40:13.073514: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-27 15:40:13.505037: lr: 1.1e-05 +2022-05-27 15:40:13.505179: saving scheduled checkpoint file... +2022-05-27 15:40:13.565659: saving checkpoint... +2022-05-27 15:40:14.511063: done, saving took 1.01 seconds +2022-05-27 15:40:14.519791: done +2022-05-27 15:40:40.466559: This epoch took 499.097975 s + +2022-05-27 15:40:40.466837: +epoch: 111 +2022-05-27 15:48:07.168368: train loss : 0.0917 +2022-05-27 15:48:32.059286: validation loss: 0.1096 +2022-05-27 15:48:32.059820: Average global foreground Dice: [0.6401, 0.8737] +2022-05-27 15:48:32.059914: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-27 15:48:32.492384: lr: 9e-06 +2022-05-27 15:48:32.492533: saving scheduled checkpoint file... +2022-05-27 15:48:32.551532: saving checkpoint... +2022-05-27 15:48:33.358493: done, saving took 0.87 seconds +2022-05-27 15:48:33.366428: done +2022-05-27 15:48:51.182449: This epoch took 490.715497 s + +2022-05-27 15:48:51.182729: +epoch: 112 +2022-05-27 15:56:17.956728: train loss : 0.0903 +2022-05-27 15:56:42.918421: validation loss: 0.1119 +2022-05-27 15:56:42.918951: Average global foreground Dice: [0.6434, 0.8565] +2022-05-27 15:56:42.919049: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-27 15:56:43.347725: lr: 8e-06 +2022-05-27 15:56:43.347867: saving scheduled checkpoint file... +2022-05-27 15:56:43.407545: saving checkpoint... +2022-05-27 15:56:44.373216: done, saving took 1.03 seconds +2022-05-27 15:56:44.381617: done +2022-05-27 15:57:03.970626: This epoch took 492.787778 s + +2022-05-27 15:57:03.970927: +epoch: 113 +2022-05-27 16:04:30.841586: train loss : 0.0979 +2022-05-27 16:04:55.737650: validation loss: 0.0937 +2022-05-27 16:04:55.738170: Average global foreground Dice: [0.657, 0.8736] +2022-05-27 16:04:55.738266: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-27 16:04:56.171224: lr: 7e-06 +2022-05-27 16:04:56.171370: saving scheduled checkpoint file... +2022-05-27 16:04:56.231713: saving checkpoint... +2022-05-27 16:04:57.012997: done, saving took 0.84 seconds +2022-05-27 16:04:57.021365: done +2022-05-27 16:05:25.835487: This epoch took 501.864450 s + +2022-05-27 16:05:25.835761: +epoch: 114 +2022-05-27 16:12:52.703648: train loss : 0.0896 +2022-05-27 16:13:17.592824: validation loss: 0.1224 +2022-05-27 16:13:17.593349: Average global foreground Dice: [0.6334, 0.8565] +2022-05-27 16:13:17.593441: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-27 16:13:18.024523: lr: 6e-06 +2022-05-27 16:13:18.024674: saving scheduled checkpoint file... +2022-05-27 16:13:18.084560: saving checkpoint... +2022-05-27 16:13:19.021394: done, saving took 1.00 seconds +2022-05-27 16:13:19.030232: done +2022-05-27 16:13:45.562142: This epoch took 499.726276 s + +2022-05-27 16:13:45.562407: +epoch: 115 +2022-05-27 16:21:12.164934: train loss : 0.0956 +2022-05-27 16:21:37.102291: validation loss: 0.1189 +2022-05-27 16:21:37.102772: Average global foreground Dice: [0.6288, 0.8582] +2022-05-27 16:21:37.102869: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-27 16:21:37.541187: lr: 5e-06 +2022-05-27 16:21:37.541336: saving scheduled checkpoint file... +2022-05-27 16:21:37.600466: saving checkpoint... +2022-05-27 16:21:38.472806: done, saving took 0.93 seconds +2022-05-27 16:21:38.481578: done +2022-05-27 16:22:13.659984: This epoch took 508.097450 s + +2022-05-27 16:22:13.660254: +epoch: 116 +2022-05-27 16:29:40.276144: train loss : 0.1007 +2022-05-27 16:30:05.173776: validation loss: 0.1417 +2022-05-27 16:30:05.174305: Average global foreground Dice: [0.6301, 0.8305] +2022-05-27 16:30:05.174398: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-27 16:30:05.606249: lr: 4e-06 +2022-05-27 16:30:05.606395: saving scheduled checkpoint file... +2022-05-27 16:30:05.666514: saving checkpoint... +2022-05-27 16:30:06.625474: done, saving took 1.02 seconds +2022-05-27 16:30:06.634530: done +2022-05-27 16:30:30.101359: This epoch took 496.440919 s + +2022-05-27 16:30:30.101636: +epoch: 117 +2022-05-27 16:37:57.590614: train loss : 0.0959 +2022-05-27 16:38:22.525457: validation loss: 0.1067 +2022-05-27 16:38:22.525951: Average global foreground Dice: [0.6493, 0.8503] +2022-05-27 16:38:22.526044: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-27 16:38:22.957292: lr: 3e-06 +2022-05-27 16:38:22.957438: saving scheduled checkpoint file... +2022-05-27 16:38:23.018775: saving checkpoint... +2022-05-27 16:38:23.957217: done, saving took 1.00 seconds +2022-05-27 16:38:23.966562: done +2022-05-27 16:38:55.393576: This epoch took 505.291824 s + +2022-05-27 16:38:55.393852: +epoch: 118 +2022-05-27 16:46:22.528802: train loss : 0.0921 +2022-05-27 16:46:47.408857: validation loss: 0.1323 +2022-05-27 16:46:47.409426: Average global foreground Dice: [0.6483, 0.8195] +2022-05-27 16:46:47.409568: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-27 16:46:47.844984: lr: 2e-06 +2022-05-27 16:46:47.845133: saving scheduled checkpoint file... +2022-05-27 16:46:47.905187: saving checkpoint... +2022-05-27 16:46:48.888550: done, saving took 1.04 seconds +2022-05-27 16:46:48.896803: done +2022-05-27 16:47:04.766850: This epoch took 489.372885 s + +2022-05-27 16:47:04.767108: +epoch: 119 +2022-05-27 16:54:31.386301: train loss : 0.0948 +2022-05-27 16:54:56.287969: validation loss: 0.1138 +2022-05-27 16:54:56.288428: Average global foreground Dice: [0.6351, 0.8493] +2022-05-27 16:54:56.288521: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-27 16:54:56.721251: lr: 1e-06 +2022-05-27 16:54:56.721396: saving scheduled checkpoint file... +2022-05-27 16:54:56.780962: saving checkpoint... +2022-05-27 16:54:57.715408: done, saving took 0.99 seconds +2022-05-27 16:54:57.724087: done +2022-05-27 16:55:16.704445: This epoch took 491.937224 s + +2022-05-27 16:55:16.704731: +epoch: 120 +2022-05-27 17:02:43.409763: train loss : 0.0915 +2022-05-27 17:03:08.292662: validation loss: 0.1181 +2022-05-27 17:03:08.293314: Average global foreground Dice: [0.6352, 0.8234] +2022-05-27 17:03:08.293408: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-27 17:03:08.725948: lr: 1e-06 +2022-05-27 17:03:08.726123: saving scheduled checkpoint file... +2022-05-27 17:03:08.785564: saving checkpoint... +2022-05-27 17:03:09.732177: done, saving took 1.01 seconds +2022-05-27 17:03:09.740957: done +2022-05-27 17:03:27.522784: This epoch took 490.817944 s + +2022-05-27 17:03:27.523069: +epoch: 121 +2022-05-27 17:10:54.362101: train loss : 0.0937 +2022-05-27 17:11:19.299269: validation loss: 0.1193 +2022-05-27 17:11:19.299772: Average global foreground Dice: [0.6327, 0.8579] +2022-05-27 17:11:19.299869: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-27 17:11:19.733926: lr: 1e-06 +2022-05-27 17:11:19.734084: saving scheduled checkpoint file... +2022-05-27 17:11:19.793018: saving checkpoint... +2022-05-27 17:11:20.722040: done, saving took 0.99 seconds +2022-05-27 17:11:20.730273: done +2022-05-27 17:11:37.817078: This epoch took 490.293847 s + +2022-05-27 17:11:37.817355: +epoch: 122 +2022-05-27 17:19:05.053128: train loss : 0.0944 +2022-05-27 17:19:29.970866: validation loss: 0.1027 +2022-05-27 17:19:29.971473: Average global foreground Dice: [0.6488, 0.8721] +2022-05-27 17:19:29.971600: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-27 17:19:30.402567: lr: 0.0 +2022-05-27 17:19:30.402712: saving scheduled checkpoint file... +2022-05-27 17:19:30.461819: saving checkpoint... +2022-05-27 17:19:31.390275: done, saving took 0.99 seconds +2022-05-27 17:19:31.398940: done +2022-05-27 17:20:04.272370: This epoch took 506.454912 s + +2022-05-27 17:20:04.272653: +epoch: 123 +2022-05-27 17:27:30.965046: train loss : 0.1005 +2022-05-27 17:27:55.886603: validation loss: 0.0966 +2022-05-27 17:27:55.887138: Average global foreground Dice: [0.6397, 0.8297] +2022-05-27 17:27:55.887242: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-27 17:27:56.322790: lr: 0.0 +2022-05-27 17:27:56.322941: saving scheduled checkpoint file... +2022-05-27 17:27:56.382082: saving checkpoint... +2022-05-27 17:27:57.314850: done, saving took 0.99 seconds +2022-05-27 17:27:57.323086: done +2022-05-27 17:28:33.274993: This epoch took 509.002238 s + +2022-05-27 17:28:33.275266: +epoch: 124 +2022-05-27 17:35:59.980655: train loss : 0.0991 +2022-05-27 17:36:24.884819: validation loss: 0.1154 +2022-05-27 17:36:24.885308: Average global foreground Dice: [0.6377, 0.8568] +2022-05-27 17:36:24.885403: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-27 17:36:25.316325: lr: 0.0 +2022-05-27 17:36:25.316481: saving scheduled checkpoint file... +2022-05-27 17:36:25.376010: saving checkpoint... +2022-05-27 17:36:26.311341: done, saving took 0.99 seconds +2022-05-27 17:36:26.320225: done +2022-05-27 17:36:45.600882: This epoch took 492.325458 s + +2022-05-27 17:36:45.648484: saving checkpoint... +2022-05-27 17:36:46.277766: done, saving took 0.68 seconds diff --git a/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_3/debug.json b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_3/debug.json new file mode 100644 index 0000000000000000000000000000000000000000..6140760a6fc85b52fb6b9e14fa0728d0173e0a22 --- /dev/null +++ b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_3/debug.json @@ -0,0 +1,106 @@ +{ + "all_tr_losses": "[]", + "all_val_eval_metrics": "[]", + "all_val_losses": "[]", + "all_val_losses_tr_mode": "[]", + "also_val_in_tr_mode": "False", + "amp_grad_scaler": "None", + "args": "Namespace(batch_size=2, config='./configs/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32.yaml', continue_training=False, crop_size=[64, 192, 192], dbs=False, deep_supervision_scales=[], deterministic=False, disable_decoder=False, disable_ds=True, disable_postprocessing_on_folds=False, disable_saving=False, drop_path=0.0, find_lr=False, find_zero_weight_decay=False, fix_ds_net_numpool=False, fold='3', fp32=False, hdfs_base='GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32', initial_lr=0.0003, is_sigmoid=False, is_spatial_aug_only=False, is_ssl=False, layer_decay=1.0, local_rank=0, loss_name='', lrschedule='warmup_cosine', mask_ratio=0.75, max_loss_cal='v1', max_num_epochs=125, merge_femur=False, min_lr=0, model='Generic_TransUNet_max_ppbp', model_params={'is_masked_attn': True, 'max_dec_layers': 3, 'is_max_bottleneck_transformer': True, 'vit_depth': 1, 'max_msda': '', 'is_max_ms': True, 'max_ms_idxs': [-4, -3, -2], 'max_hidden_dim': 192, 'mw': 1.0, 'is_max_ds': True, 'is_masking': True, 'is_max_hungarian': True, 'num_queries': 20, 'is_max_cls': True, 'is_mhsa_float32': True}, n_class=17, network='3d_fullres', network_trainer='nnUNetTrainerV2_DDP', npz=False, opt_betas=None, opt_eps=1e-08, optim_name='adamw', p='nnUNetPlansv2.1', plan_update='', pretrained=False, pretrained_config=None, pretrained_weights=None, rank=0, reclip=None, resume='auto', skip_grad_nan=False, task='Task008_HepaticVessel', task_pretrained='Task801_WORD', total_batch_size=None, use_compressed_data=False, val_final=False, val_folder='validation_raw', valbest=False, validation_only=False, warmup_epochs=10, weight_decay=3e-05, world_size=8)", + "base_num_features": "32", + "basic_generator_patch_size": "[178 308 233]", + "batch_dice": "True", + "batch_size": "2", + "best_MA_tr_loss_for_patience": "None", + "best_epoch_based_on_MA_tr_loss": "None", + "best_val_eval_criterion_MA": "None", + "classes": "[1, 2]", + "conv_per_stage": "2", + "data_aug_params": "{'selected_data_channels': None, 'selected_seg_channels': [0], 'do_elastic': False, 'elastic_deform_alpha': (0.0, 900.0), 'elastic_deform_sigma': (9.0, 13.0), 'p_eldef': 0.2, 'do_scaling': True, 'scale_range': (0.7, 1.4), 'independent_scale_factor_for_each_axis': False, 'p_independent_scale_per_axis': 1, 'p_scale': 0.2, 'do_rotation': True, 'rotation_x': (-0.5235987755982988, 0.5235987755982988), 'rotation_y': (-0.5235987755982988, 0.5235987755982988), 'rotation_z': (-0.5235987755982988, 0.5235987755982988), 'rotation_p_per_axis': 1, 'p_rot': 0.2, 'random_crop': False, 'random_crop_dist_to_border': None, 'do_gamma': True, 'gamma_retain_stats': True, 'gamma_range': (0.7, 1.5), 'p_gamma': 0.3, 'do_mirror': True, 'mirror_axes': (0, 1, 2), 'dummy_2D': False, 'mask_was_used_for_normalization': OrderedDict([(0, False)]), 'border_mode_data': 'constant', 'all_segmentation_labels': None, 'move_last_seg_chanel_to_data': False, 'cascade_do_cascade_augmentations': False, 'cascade_random_binary_transform_p': 0.4, 'cascade_random_binary_transform_p_per_label': 1, 'cascade_random_binary_transform_size': (1, 8), 'cascade_remove_conn_comp_p': 0.2, 'cascade_remove_conn_comp_max_size_percent_threshold': 0.15, 'cascade_remove_conn_comp_fill_with_other_class_p': 0.0, 'do_additive_brightness': False, 'additive_brightness_p_per_sample': 0.15, 'additive_brightness_p_per_channel': 0.5, 'additive_brightness_mu': 0.0, 'additive_brightness_sigma': 0.1, 'num_threads': 12, 'num_cached_per_thread': 2, 'patch_size_for_spatialtransform': array([ 64, 192, 192])}", + "dataset_directory": "/opt/tiger/project/data/nnUNet_preprocessed/Task008_HepaticVessel", + "deep_supervision_scales": "[[1, 1, 1], [0.5, 0.5, 0.5], [0.25, 0.25, 0.25], [0.125, 0.125, 0.125], [0.0625, 0.0625, 0.0625]]", + "deterministic": "False", + "disable_ds": "True", + "distribute_batch_size": "False", + "dl_tr": "", + "dl_val": "", + "do_dummy_2D_aug": "False", + "ds_loss_weights": "[1. 0. 0. 0. 0.]", + "epoch": "0", + "experiment_name": "nnUNetTrainerV2_DDP", + "find_zero_weight_decay": "False", + "fold": "3", + "folder_with_preprocessed_data": "/opt/tiger/project/data/nnUNet_preprocessed/Task008_HepaticVessel/nnUNetData_plans_v2.1_stage1", + "fp16": "True", + "global_batch_size": "16", + "gt_niftis_folder": "/opt/tiger/project/data/nnUNet_preprocessed/Task008_HepaticVessel/gt_segmentations", + "inference_pad_border_mode": "constant", + "inference_pad_kwargs": "{'constant_values': 0}", + "init_args": "('/opt/tiger/project/data/nnUNet_preprocessed/Task008_HepaticVessel/nnUNetPlansv2.1_plans_3D.pkl', 3, 0, '/opt/tiger/project/data/nnUNet_trained_models/UNet_IN_NANFang/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32', '/opt/tiger/project/data/nnUNet_preprocessed/Task008_HepaticVessel', True, 1, True, False, False, True)", + "initial_lr": "0.0003", + "input_size": "[64, 192, 192]", + "is_spatial_aug_only": "False", + "is_ssl": "False", + "layer_decay": "1.0", + "local_rank": "0", + "log_file": "/opt/tiger/project/data/nnUNet_trained_models/UNet_IN_NANFang/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_3/training_log_2022_4_10_10_20_37.txt", + "loss": "None", + "lr_scheduler": "", + "lr_scheduler_eps": "0.001", + "lr_scheduler_name": "warmup_cosine", + "lr_scheduler_patience": "30", + "lr_threshold": "1e-06", + "max_num_epochs": "125", + "min_lr": "0", + "min_region_size_per_class": "None", + "min_size_per_class": "None", + "model": "Generic_TransUNet_max_ppbp", + "model_params": "{'is_masked_attn': True, 'max_dec_layers': 3, 'is_max_bottleneck_transformer': True, 'vit_depth': 1, 'max_msda': '', 'is_max_ms': True, 'max_ms_idxs': [-4, -3, -2], 'max_hidden_dim': 192, 'mw': 1.0, 'is_max_ds': True, 'is_masking': True, 'is_max_hungarian': True, 'num_queries': 20, 'is_max_cls': True, 'is_mhsa_float32': True}", + "net_conv_kernel_sizes": "[[3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3]]", + "net_num_pool_op_kernel_sizes": "[[2, 2, 2], [2, 2, 2], [2, 2, 2], [2, 2, 2], [1, 2, 2]]", + "net_pool_per_axis": "[4, 5, 5]", + "normalization_schemes": "OrderedDict([(0, 'CT')])", + "num_batches_per_epoch": "250", + "num_classes": "3", + "num_input_channels": "1", + "num_val_batches_per_epoch": "50", + "online_eval_fn": "[]", + "online_eval_foreground_dc": "[]", + "online_eval_fp": "[]", + "online_eval_tp": "[]", + "only_keep_largest_connected_component": "None", + "optim_name": "adamw", + "optimizer": "AdamW (\nParameter Group 0\n amsgrad: False\n betas: (0.9, 0.999)\n eps: 1e-08\n initial_lr: 0.0003\n lr: 0.0\n weight_decay: 3e-05\n)", + "output_folder": "/opt/tiger/project/data/nnUNet_trained_models/UNet_IN_NANFang/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_3", + "output_folder_base": "/opt/tiger/project/data/nnUNet_trained_models/UNet_IN_NANFang/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32", + "oversample_foreground_percent": "0.0", + "pad_all_sides": "None", + "patch_size": "[ 64 192 192]", + "patience": "50", + "pin_memory": "True", + "plans_file": "/opt/tiger/project/data/nnUNet_preprocessed/Task008_HepaticVessel/nnUNetPlansv2.1_plans_3D.pkl", + "reclip": "None", + "regions_class_order": "None", + "resume": "auto", + "save_best_checkpoint": "True", + "save_every": "1", + "save_final_checkpoint": "True", + "save_intermediate_checkpoints": "True", + "save_latest_only": "True", + "stage": "1", + "threeD": "True", + "tr_gen": "", + "train_loss_MA": "None", + "train_loss_MA_alpha": "0.93", + "train_loss_MA_eps": "0.0005", + "transpose_backward": "[0, 1, 2]", + "transpose_forward": "[0, 1, 2]", + "unpack_data": "True", + "use_mask_for_norm": "OrderedDict([(0, False)])", + "use_progress_bar": "False", + "val_eval_criterion_MA": "None", + "val_eval_criterion_alpha": "0.9", + "val_gen": "", + "warmup_epochs": "10", + "was_initialized": "True", + "weight_decay": "3e-05" +} \ No newline at end of file diff --git a/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_3/model_best.model b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_3/model_best.model new file mode 100644 index 0000000000000000000000000000000000000000..db10b801fc9afae405923c17c04ee8180e3edc19 --- /dev/null +++ b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_3/model_best.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e519551535b456c82bf4f72ec5616bc6adc12612e71cab3cfc05edd9ffee190 +size 496529529 diff --git a/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_3/model_best.model.pkl b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_3/model_best.model.pkl new file mode 100644 index 0000000000000000000000000000000000000000..4ec611528a491c429fd3624a637f65be150de68b --- /dev/null +++ b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_3/model_best.model.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ba3cdd78f78160c349ef0250bdf2abd5e4b8a5ea11911f42c5fdba0bfad77a6 +size 188314 diff --git a/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_3/model_final_checkpoint.model b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_3/model_final_checkpoint.model new file mode 100644 index 0000000000000000000000000000000000000000..5f00acb893e366bda07ee4419a50d47a22ef54c8 --- /dev/null +++ b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_3/model_final_checkpoint.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be1b43eaf3027ebfdf6dc5fa57ffcff2ae50d8c78a972165f3f36f67320d6618 +size 496529529 diff --git a/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_3/model_final_checkpoint.model.pkl b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_3/model_final_checkpoint.model.pkl new file mode 100644 index 0000000000000000000000000000000000000000..4ec611528a491c429fd3624a637f65be150de68b --- /dev/null +++ b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_3/model_final_checkpoint.model.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ba3cdd78f78160c349ef0250bdf2abd5e4b8a5ea11911f42c5fdba0bfad77a6 +size 188314 diff --git a/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_3/model_latest.model b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_3/model_latest.model new file mode 100644 index 0000000000000000000000000000000000000000..e8f28a882358d44b76f685a9c068d09e63dc5ab4 --- /dev/null +++ b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_3/model_latest.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1bc174f6585d0adbbdb6ae9fb91c8b11202dd6f22f20526debe0ac4490d16c01 +size 496529529 diff --git a/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_3/model_latest.model.pkl b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_3/model_latest.model.pkl new file mode 100644 index 0000000000000000000000000000000000000000..4ec611528a491c429fd3624a637f65be150de68b --- /dev/null +++ b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_3/model_latest.model.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ba3cdd78f78160c349ef0250bdf2abd5e4b8a5ea11911f42c5fdba0bfad77a6 +size 188314 diff --git a/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_3/progress.png b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_3/progress.png new file mode 100644 index 0000000000000000000000000000000000000000..2d099dcd8c5c2b3a1eadcfa66f0691802fdf680f --- /dev/null +++ b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_3/progress.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e095243af45f25d61594b85f88dcfe6e954eb0320e37551e40e5d0fa16b47b1f +size 276224 diff --git a/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_3/training_log_2022_4_10_10_20_37.txt b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_3/training_log_2022_4_10_10_20_37.txt new file mode 100644 index 0000000000000000000000000000000000000000..d31f7c5f38ca1ff34e971cd006a81b9482a49318 --- /dev/null +++ b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_3/training_log_2022_4_10_10_20_37.txt @@ -0,0 +1,1787 @@ +Starting... +2022-04-10 10:20:37.083468: Using splits from existing split file: /opt/tiger/project/data/nnUNet_preprocessed/Task008_HepaticVessel/splits_final.pkl +2022-04-10 10:20:37.084042: The split file contains 5 splits. +2022-04-10 10:20:37.084095: Desired fold for training: 3 +2022-04-10 10:20:37.084203: This split has 243 training and 60 validation cases. +2022-04-10 10:22:54.201807: TRAINING KEYS: + odict_keys(['hepaticvessel_002', 'hepaticvessel_004', 'hepaticvessel_007', 'hepaticvessel_008', 'hepaticvessel_016', 'hepaticvessel_018', 'hepaticvessel_019', 'hepaticvessel_020', 'hepaticvessel_022', 'hepaticvessel_023', 'hepaticvessel_025', 'hepaticvessel_026', 'hepaticvessel_027', 'hepaticvessel_028', 'hepaticvessel_029', 'hepaticvessel_030', 'hepaticvessel_032', 'hepaticvessel_033', 'hepaticvessel_039', 'hepaticvessel_040', 'hepaticvessel_042', 'hepaticvessel_044', 'hepaticvessel_046', 'hepaticvessel_049', 'hepaticvessel_050', 'hepaticvessel_051', 'hepaticvessel_053', 'hepaticvessel_057', 'hepaticvessel_058', 'hepaticvessel_059', 'hepaticvessel_062', 'hepaticvessel_065', 'hepaticvessel_066', 'hepaticvessel_067', 'hepaticvessel_068', 'hepaticvessel_070', 'hepaticvessel_071', 'hepaticvessel_075', 'hepaticvessel_077', 'hepaticvessel_078', 'hepaticvessel_079', 'hepaticvessel_080', 'hepaticvessel_081', 'hepaticvessel_082', 'hepaticvessel_083', 'hepaticvessel_084', 'hepaticvessel_085', 'hepaticvessel_086', 'hepaticvessel_087', 'hepaticvessel_088', 'hepaticvessel_089', 'hepaticvessel_091', 'hepaticvessel_092', 'hepaticvessel_094', 'hepaticvessel_097', 'hepaticvessel_098', 'hepaticvessel_100', 'hepaticvessel_103', 'hepaticvessel_104', 'hepaticvessel_108', 'hepaticvessel_110', 'hepaticvessel_111', 'hepaticvessel_112', 'hepaticvessel_113', 'hepaticvessel_115', 'hepaticvessel_116', 'hepaticvessel_117', 'hepaticvessel_119', 'hepaticvessel_121', 'hepaticvessel_123', 'hepaticvessel_124', 'hepaticvessel_125', 'hepaticvessel_127', 'hepaticvessel_128', 'hepaticvessel_129', 'hepaticvessel_131', 'hepaticvessel_132', 'hepaticvessel_136', 'hepaticvessel_137', 'hepaticvessel_138', 'hepaticvessel_139', 'hepaticvessel_140', 'hepaticvessel_141', 'hepaticvessel_142', 'hepaticvessel_144', 'hepaticvessel_145', 'hepaticvessel_146', 'hepaticvessel_147', 'hepaticvessel_150', 'hepaticvessel_151', 'hepaticvessel_154', 'hepaticvessel_157', 'hepaticvessel_158', 'hepaticvessel_159', 'hepaticvessel_160', 'hepaticvessel_161', 'hepaticvessel_164', 'hepaticvessel_165', 'hepaticvessel_167', 'hepaticvessel_172', 'hepaticvessel_173', 'hepaticvessel_174', 'hepaticvessel_175', 'hepaticvessel_178', 'hepaticvessel_181', 'hepaticvessel_184', 'hepaticvessel_185', 'hepaticvessel_186', 'hepaticvessel_189', 'hepaticvessel_192', 'hepaticvessel_193', 'hepaticvessel_194', 'hepaticvessel_195', 'hepaticvessel_196', 'hepaticvessel_197', 'hepaticvessel_200', 'hepaticvessel_201', 'hepaticvessel_203', 'hepaticvessel_206', 'hepaticvessel_207', 'hepaticvessel_208', 'hepaticvessel_209', 'hepaticvessel_210', 'hepaticvessel_211', 'hepaticvessel_213', 'hepaticvessel_214', 'hepaticvessel_217', 'hepaticvessel_218', 'hepaticvessel_219', 'hepaticvessel_223', 'hepaticvessel_224', 'hepaticvessel_225', 'hepaticvessel_229', 'hepaticvessel_230', 'hepaticvessel_231', 'hepaticvessel_233', 'hepaticvessel_234', 'hepaticvessel_235', 'hepaticvessel_236', 'hepaticvessel_237', 'hepaticvessel_238', 'hepaticvessel_240', 'hepaticvessel_242', 'hepaticvessel_244', 'hepaticvessel_248', 'hepaticvessel_255', 'hepaticvessel_256', 'hepaticvessel_258', 'hepaticvessel_262', 'hepaticvessel_263', 'hepaticvessel_265', 'hepaticvessel_266', 'hepaticvessel_268', 'hepaticvessel_269', 'hepaticvessel_270', 'hepaticvessel_271', 'hepaticvessel_272', 'hepaticvessel_274', 'hepaticvessel_275', 'hepaticvessel_278', 'hepaticvessel_280', 'hepaticvessel_282', 'hepaticvessel_285', 'hepaticvessel_286', 'hepaticvessel_287', 'hepaticvessel_289', 'hepaticvessel_293', 'hepaticvessel_297', 'hepaticvessel_299', 'hepaticvessel_303', 'hepaticvessel_305', 'hepaticvessel_307', 'hepaticvessel_308', 'hepaticvessel_309', 'hepaticvessel_318', 'hepaticvessel_321', 'hepaticvessel_322', 'hepaticvessel_324', 'hepaticvessel_329', 'hepaticvessel_330', 'hepaticvessel_332', 'hepaticvessel_333', 'hepaticvessel_335', 'hepaticvessel_338', 'hepaticvessel_340', 'hepaticvessel_341', 'hepaticvessel_344', 'hepaticvessel_345', 'hepaticvessel_348', 'hepaticvessel_349', 'hepaticvessel_350', 'hepaticvessel_353', 'hepaticvessel_356', 'hepaticvessel_358', 'hepaticvessel_359', 'hepaticvessel_361', 'hepaticvessel_362', 'hepaticvessel_363', 'hepaticvessel_367', 'hepaticvessel_368', 'hepaticvessel_372', 'hepaticvessel_373', 'hepaticvessel_375', 'hepaticvessel_378', 'hepaticvessel_379', 'hepaticvessel_382', 'hepaticvessel_383', 'hepaticvessel_384', 'hepaticvessel_385', 'hepaticvessel_386', 'hepaticvessel_388', 'hepaticvessel_389', 'hepaticvessel_391', 'hepaticvessel_393', 'hepaticvessel_396', 'hepaticvessel_397', 'hepaticvessel_398', 'hepaticvessel_402', 'hepaticvessel_404', 'hepaticvessel_406', 'hepaticvessel_407', 'hepaticvessel_408', 'hepaticvessel_409', 'hepaticvessel_419', 'hepaticvessel_420', 'hepaticvessel_422', 'hepaticvessel_423', 'hepaticvessel_424', 'hepaticvessel_425', 'hepaticvessel_431', 'hepaticvessel_432', 'hepaticvessel_433', 'hepaticvessel_437', 'hepaticvessel_440', 'hepaticvessel_441', 'hepaticvessel_442', 'hepaticvessel_443', 'hepaticvessel_444', 'hepaticvessel_445', 'hepaticvessel_454', 'hepaticvessel_455', 'hepaticvessel_456', 'hepaticvessel_458']) +2022-04-10 10:22:54.202080: VALIDATION KEYS: + odict_keys(['hepaticvessel_001', 'hepaticvessel_005', 'hepaticvessel_010', 'hepaticvessel_011', 'hepaticvessel_013', 'hepaticvessel_031', 'hepaticvessel_052', 'hepaticvessel_061', 'hepaticvessel_072', 'hepaticvessel_090', 'hepaticvessel_093', 'hepaticvessel_096', 'hepaticvessel_102', 'hepaticvessel_133', 'hepaticvessel_148', 'hepaticvessel_163', 'hepaticvessel_166', 'hepaticvessel_171', 'hepaticvessel_177', 'hepaticvessel_179', 'hepaticvessel_183', 'hepaticvessel_198', 'hepaticvessel_199', 'hepaticvessel_215', 'hepaticvessel_221', 'hepaticvessel_222', 'hepaticvessel_227', 'hepaticvessel_245', 'hepaticvessel_246', 'hepaticvessel_259', 'hepaticvessel_279', 'hepaticvessel_281', 'hepaticvessel_284', 'hepaticvessel_290', 'hepaticvessel_291', 'hepaticvessel_294', 'hepaticvessel_296', 'hepaticvessel_300', 'hepaticvessel_314', 'hepaticvessel_319', 'hepaticvessel_320', 'hepaticvessel_323', 'hepaticvessel_325', 'hepaticvessel_327', 'hepaticvessel_336', 'hepaticvessel_369', 'hepaticvessel_371', 'hepaticvessel_374', 'hepaticvessel_400', 'hepaticvessel_401', 'hepaticvessel_411', 'hepaticvessel_412', 'hepaticvessel_416', 'hepaticvessel_427', 'hepaticvessel_429', 'hepaticvessel_434', 'hepaticvessel_438', 'hepaticvessel_446', 'hepaticvessel_447', 'hepaticvessel_453']) +2022-04-10 10:22:57.898256: lr: 3.3e-05 +2022-04-10 10:23:11.866811: +epoch: 0 +2022-04-10 10:31:13.615781: train loss : 0.3499 +2022-04-10 10:31:40.062033: validation loss: 0.2614 +2022-04-10 10:31:40.062728: Average global foreground Dice: [0.5041, 0.0] +2022-04-10 10:31:40.062819: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 10:31:40.381839: lr: 3.3e-05 +2022-04-10 10:31:40.382035: saving scheduled checkpoint file... +2022-04-10 10:31:40.447974: saving checkpoint... +2022-04-10 10:31:41.099472: done, saving took 0.72 seconds +2022-04-10 10:31:41.109838: done +2022-04-10 10:31:51.594410: This epoch took 519.727127 s + +2022-04-10 10:31:51.594667: +epoch: 1 +2022-04-10 10:39:36.138235: train loss : 0.2302 +2022-04-10 10:40:02.573066: validation loss: 0.2136 +2022-04-10 10:40:02.573890: Average global foreground Dice: [0.5772, 0.3434] +2022-04-10 10:40:02.573979: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 10:40:02.963822: lr: 6.7e-05 +2022-04-10 10:40:02.964006: saving scheduled checkpoint file... +2022-04-10 10:40:03.019991: saving checkpoint... +2022-04-10 10:40:04.009094: done, saving took 1.04 seconds +2022-04-10 10:40:04.021043: done +2022-04-10 10:40:04.050357: saving checkpoint... +2022-04-10 10:40:04.687926: done, saving took 0.67 seconds +2022-04-10 10:40:27.874324: This epoch took 516.279555 s + +2022-04-10 10:40:27.874632: +epoch: 2 +2022-04-10 10:48:11.721261: train loss : 0.1984 +2022-04-10 10:48:38.134556: validation loss: 0.1919 +2022-04-10 10:48:38.135210: Average global foreground Dice: [0.6074, 0.3237] +2022-04-10 10:48:38.135298: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 10:48:38.620289: lr: 0.0001 +2022-04-10 10:48:38.620481: saving scheduled checkpoint file... +2022-04-10 10:48:38.677144: saving checkpoint... +2022-04-10 10:48:39.428774: done, saving took 0.81 seconds +2022-04-10 10:48:39.441379: done +2022-04-10 10:48:39.474663: saving checkpoint... +2022-04-10 10:48:40.445369: done, saving took 1.00 seconds +2022-04-10 10:48:59.806780: This epoch took 511.932049 s + +2022-04-10 10:48:59.807105: +epoch: 3 +2022-04-10 10:56:44.449374: train loss : 0.1867 +2022-04-10 10:57:10.882951: validation loss: 0.1974 +2022-04-10 10:57:10.883556: Average global foreground Dice: [0.5963, 0.4081] +2022-04-10 10:57:10.883644: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 10:57:11.283034: lr: 0.000133 +2022-04-10 10:57:11.283279: saving scheduled checkpoint file... +2022-04-10 10:57:11.340510: saving checkpoint... +2022-04-10 10:57:12.089612: done, saving took 0.81 seconds +2022-04-10 10:57:12.101259: done +2022-04-10 10:57:12.130748: saving checkpoint... +2022-04-10 10:57:13.082391: done, saving took 0.98 seconds +2022-04-10 10:57:31.599987: This epoch took 511.792763 s + +2022-04-10 10:57:31.600280: +epoch: 4 +2022-04-10 11:05:15.644006: train loss : 0.1836 +2022-04-10 11:05:42.069019: validation loss: 0.1585 +2022-04-10 11:05:42.069885: Average global foreground Dice: [0.621, 0.5706] +2022-04-10 11:05:42.069982: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 11:05:42.485110: lr: 0.000167 +2022-04-10 11:05:42.485316: saving scheduled checkpoint file... +2022-04-10 11:05:42.542745: saving checkpoint... +2022-04-10 11:05:43.298563: done, saving took 0.81 seconds +2022-04-10 11:05:43.311946: done +2022-04-10 11:05:43.345692: saving checkpoint... +2022-04-10 11:05:44.108574: done, saving took 0.80 seconds +2022-04-10 11:06:04.434891: This epoch took 512.834504 s + +2022-04-10 11:06:04.435206: +epoch: 5 +2022-04-10 11:13:49.219070: train loss : 0.1647 +2022-04-10 11:14:15.632681: validation loss: 0.1838 +2022-04-10 11:14:15.633498: Average global foreground Dice: [0.6195, 0.6222] +2022-04-10 11:14:15.633591: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 11:14:16.036102: lr: 0.0002 +2022-04-10 11:14:16.036308: saving scheduled checkpoint file... +2022-04-10 11:14:16.093601: saving checkpoint... +2022-04-10 11:14:16.851172: done, saving took 0.81 seconds +2022-04-10 11:14:16.863828: done +2022-04-10 11:14:16.894076: saving checkpoint... +2022-04-10 11:14:17.840830: done, saving took 0.98 seconds +2022-04-10 11:14:38.725897: This epoch took 514.290590 s + +2022-04-10 11:14:38.726236: +epoch: 6 +2022-04-10 11:22:23.072448: train loss : 0.1661 +2022-04-10 11:22:49.504532: validation loss: 0.1518 +2022-04-10 11:22:49.505234: Average global foreground Dice: [0.6306, 0.6597] +2022-04-10 11:22:49.505338: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 11:22:49.911081: lr: 0.000233 +2022-04-10 11:22:49.911281: saving scheduled checkpoint file... +2022-04-10 11:22:49.968486: saving checkpoint... +2022-04-10 11:22:50.876866: done, saving took 0.97 seconds +2022-04-10 11:22:50.889108: done +2022-04-10 11:22:50.918263: saving checkpoint... +2022-04-10 11:22:51.743119: done, saving took 0.85 seconds +2022-04-10 11:23:09.660321: This epoch took 510.933973 s + +2022-04-10 11:23:09.660594: +epoch: 7 +2022-04-10 11:30:53.846514: train loss : 0.1585 +2022-04-10 11:31:20.323537: validation loss: 0.1795 +2022-04-10 11:31:20.324148: Average global foreground Dice: [0.6117, 0.6074] +2022-04-10 11:31:20.324249: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 11:31:20.814788: lr: 0.000267 +2022-04-10 11:31:20.814977: saving scheduled checkpoint file... +2022-04-10 11:31:20.870976: saving checkpoint... +2022-04-10 11:31:21.903019: done, saving took 1.09 seconds +2022-04-10 11:31:21.915514: done +2022-04-10 11:31:21.948765: saving checkpoint... +2022-04-10 11:31:22.943116: done, saving took 1.03 seconds +2022-04-10 11:31:40.452112: This epoch took 510.791406 s + +2022-04-10 11:31:40.452425: +epoch: 8 +2022-04-10 11:39:24.678682: train loss : 0.1479 +2022-04-10 11:39:51.096808: validation loss: 0.1400 +2022-04-10 11:39:51.097403: Average global foreground Dice: [0.6304, 0.6577] +2022-04-10 11:39:51.097494: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 11:39:51.500313: lr: 0.0003 +2022-04-10 11:39:51.500487: saving scheduled checkpoint file... +2022-04-10 11:39:51.557127: saving checkpoint... +2022-04-10 11:39:52.527678: done, saving took 1.03 seconds +2022-04-10 11:39:52.540079: done +2022-04-10 11:39:52.572447: saving checkpoint... +2022-04-10 11:39:53.550024: done, saving took 1.01 seconds +2022-04-10 11:40:55.656973: This epoch took 555.204448 s + +2022-04-10 11:40:55.657299: +epoch: 9 +2022-04-10 11:48:39.339222: train loss : 0.1542 +2022-04-10 11:49:05.771391: validation loss: 0.1571 +2022-04-10 11:49:05.771981: Average global foreground Dice: [0.6411, 0.7239] +2022-04-10 11:49:05.772190: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 11:49:06.158440: lr: 0.0003 +2022-04-10 11:49:06.158624: saving scheduled checkpoint file... +2022-04-10 11:49:06.215422: saving checkpoint... +2022-04-10 11:49:06.981601: done, saving took 0.82 seconds +2022-04-10 11:49:06.995433: done +2022-04-10 11:49:07.029147: saving checkpoint... +2022-04-10 11:49:07.802680: done, saving took 0.81 seconds +2022-04-10 11:49:26.215940: This epoch took 510.558534 s + +2022-04-10 11:49:26.216227: +epoch: 10 +2022-04-10 11:57:11.079092: train loss : 0.1531 +2022-04-10 11:57:37.507640: validation loss: 0.1659 +2022-04-10 11:57:37.508284: Average global foreground Dice: [0.6197, 0.7407] +2022-04-10 11:57:37.508372: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 11:57:37.895661: lr: 0.0003 +2022-04-10 11:57:37.895844: saving scheduled checkpoint file... +2022-04-10 11:57:37.952314: saving checkpoint... +2022-04-10 11:57:38.733987: done, saving took 0.84 seconds +2022-04-10 11:57:38.746520: done +2022-04-10 11:57:38.778751: saving checkpoint... +2022-04-10 11:57:39.537382: done, saving took 0.79 seconds +2022-04-10 11:58:19.773912: This epoch took 533.557579 s + +2022-04-10 11:58:19.774240: +epoch: 11 +2022-04-10 12:06:04.175954: train loss : 0.1548 +2022-04-10 12:06:30.593664: validation loss: 0.1382 +2022-04-10 12:06:30.594279: Average global foreground Dice: [0.6397, 0.7383] +2022-04-10 12:06:30.594364: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 12:06:30.982803: lr: 0.0003 +2022-04-10 12:06:30.982992: saving scheduled checkpoint file... +2022-04-10 12:06:31.040812: saving checkpoint... +2022-04-10 12:06:31.837369: done, saving took 0.85 seconds +2022-04-10 12:06:31.851191: done +2022-04-10 12:06:31.885658: saving checkpoint... +2022-04-10 12:06:32.658396: done, saving took 0.81 seconds +2022-04-10 12:06:50.596610: This epoch took 510.822263 s + +2022-04-10 12:06:50.596882: +epoch: 12 +2022-04-10 12:14:35.233165: train loss : 0.1539 +2022-04-10 12:15:01.674145: validation loss: 0.1393 +2022-04-10 12:15:01.674678: Average global foreground Dice: [0.64, 0.7559] +2022-04-10 12:15:01.674765: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 12:15:02.080619: lr: 0.000299 +2022-04-10 12:15:02.080812: saving scheduled checkpoint file... +2022-04-10 12:15:02.136913: saving checkpoint... +2022-04-10 12:15:03.049953: done, saving took 0.97 seconds +2022-04-10 12:15:03.069047: done +2022-04-10 12:15:03.110219: saving checkpoint... +2022-04-10 12:15:03.924820: done, saving took 0.86 seconds +2022-04-10 12:15:23.449942: This epoch took 512.852957 s + +2022-04-10 12:15:23.450258: +epoch: 13 +2022-04-10 12:23:07.565952: train loss : 0.1443 +2022-04-10 12:23:33.993439: validation loss: 0.1450 +2022-04-10 12:23:33.994024: Average global foreground Dice: [0.6511, 0.7246] +2022-04-10 12:23:33.994111: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 12:23:34.394005: lr: 0.000299 +2022-04-10 12:23:34.394190: saving scheduled checkpoint file... +2022-04-10 12:23:34.449370: saving checkpoint... +2022-04-10 12:23:35.260366: done, saving took 0.87 seconds +2022-04-10 12:23:35.274855: done +2022-04-10 12:23:35.308610: saving checkpoint... +2022-04-10 12:23:36.161995: done, saving took 0.89 seconds +2022-04-10 12:24:00.189371: This epoch took 516.739017 s + +2022-04-10 12:24:00.189641: +epoch: 14 +2022-04-10 12:31:44.896982: train loss : 0.1400 +2022-04-10 12:32:11.312597: validation loss: 0.1379 +2022-04-10 12:32:11.313218: Average global foreground Dice: [0.6537, 0.7555] +2022-04-10 12:32:11.313303: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 12:32:11.705767: lr: 0.000299 +2022-04-10 12:32:11.705957: saving scheduled checkpoint file... +2022-04-10 12:32:11.762720: saving checkpoint... +2022-04-10 12:32:12.750886: done, saving took 1.04 seconds +2022-04-10 12:32:12.764289: done +2022-04-10 12:32:12.797271: saving checkpoint... +2022-04-10 12:32:13.818667: done, saving took 1.05 seconds +2022-04-10 12:32:30.088279: This epoch took 509.898535 s + +2022-04-10 12:32:30.088550: +epoch: 15 +2022-04-10 12:40:13.953102: train loss : 0.1418 +2022-04-10 12:40:40.372629: validation loss: 0.1334 +2022-04-10 12:40:40.373260: Average global foreground Dice: [0.6209, 0.7783] +2022-04-10 12:40:40.373347: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 12:40:40.763979: lr: 0.000298 +2022-04-10 12:40:40.764187: saving scheduled checkpoint file... +2022-04-10 12:40:40.819943: saving checkpoint... +2022-04-10 12:40:41.723094: done, saving took 0.96 seconds +2022-04-10 12:40:41.736442: done +2022-04-10 12:40:41.765638: saving checkpoint... +2022-04-10 12:40:42.644211: done, saving took 0.91 seconds +2022-04-10 12:41:06.292978: This epoch took 516.204326 s + +2022-04-10 12:41:06.293259: +epoch: 16 +2022-04-10 12:48:50.477393: train loss : 0.1384 +2022-04-10 12:49:16.901909: validation loss: 0.1506 +2022-04-10 12:49:16.902626: Average global foreground Dice: [0.6653, 0.7789] +2022-04-10 12:49:16.902716: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 12:49:17.309275: lr: 0.000297 +2022-04-10 12:49:17.309469: saving scheduled checkpoint file... +2022-04-10 12:49:17.366681: saving checkpoint... +2022-04-10 12:49:18.191713: done, saving took 0.88 seconds +2022-04-10 12:49:18.204602: done +2022-04-10 12:49:18.236095: saving checkpoint... +2022-04-10 12:49:19.085500: done, saving took 0.88 seconds +2022-04-10 12:49:46.800288: This epoch took 520.506924 s + +2022-04-10 12:49:46.800585: +epoch: 17 +2022-04-10 12:57:31.090198: train loss : 0.1419 +2022-04-10 12:57:57.504947: validation loss: 0.1332 +2022-04-10 12:57:57.505548: Average global foreground Dice: [0.6531, 0.7953] +2022-04-10 12:57:57.505652: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 12:57:57.899378: lr: 0.000296 +2022-04-10 12:57:57.899566: saving scheduled checkpoint file... +2022-04-10 12:57:57.955918: saving checkpoint... +2022-04-10 12:57:58.984966: done, saving took 1.09 seconds +2022-04-10 12:57:58.997235: done +2022-04-10 12:57:59.026696: saving checkpoint... +2022-04-10 12:58:00.024369: done, saving took 1.03 seconds +2022-04-10 12:58:19.962373: This epoch took 513.161687 s + +2022-04-10 12:58:19.962636: +epoch: 18 +2022-04-10 13:06:04.291584: train loss : 0.1330 +2022-04-10 13:06:30.691805: validation loss: 0.1220 +2022-04-10 13:06:30.692442: Average global foreground Dice: [0.6467, 0.7856] +2022-04-10 13:06:30.692527: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 13:06:31.100821: lr: 0.000295 +2022-04-10 13:06:31.101003: saving scheduled checkpoint file... +2022-04-10 13:06:31.156988: saving checkpoint... +2022-04-10 13:06:31.974964: done, saving took 0.87 seconds +2022-04-10 13:06:31.989408: done +2022-04-10 13:06:32.027232: saving checkpoint... +2022-04-10 13:06:32.907462: done, saving took 0.92 seconds +2022-04-10 13:06:52.191609: This epoch took 512.228871 s + +2022-04-10 13:06:52.191895: +epoch: 19 +2022-04-10 13:14:36.116318: train loss : 0.1297 +2022-04-10 13:15:02.543769: validation loss: 0.1172 +2022-04-10 13:15:02.544345: Average global foreground Dice: [0.6459, 0.7657] +2022-04-10 13:15:02.544434: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 13:15:02.940616: lr: 0.000294 +2022-04-10 13:15:02.940822: saving scheduled checkpoint file... +2022-04-10 13:15:02.997762: saving checkpoint... +2022-04-10 13:15:03.878990: done, saving took 0.94 seconds +2022-04-10 13:15:03.892158: done +2022-04-10 13:15:03.920777: saving checkpoint... +2022-04-10 13:15:04.813559: done, saving took 0.92 seconds +2022-04-10 13:15:22.673501: This epoch took 510.481505 s + +2022-04-10 13:15:22.673996: +epoch: 20 +2022-04-10 13:23:06.657405: train loss : 0.1352 +2022-04-10 13:23:33.083849: validation loss: 0.1349 +2022-04-10 13:23:33.084473: Average global foreground Dice: [0.6714, 0.808] +2022-04-10 13:23:33.084561: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 13:23:33.480651: lr: 0.000293 +2022-04-10 13:23:33.480853: saving scheduled checkpoint file... +2022-04-10 13:23:33.538591: saving checkpoint... +2022-04-10 13:23:34.356267: done, saving took 0.88 seconds +2022-04-10 13:23:34.369095: done +2022-04-10 13:23:34.399544: saving checkpoint... +2022-04-10 13:23:35.427487: done, saving took 1.06 seconds +2022-04-10 13:23:53.897202: This epoch took 511.223103 s + +2022-04-10 13:23:53.897502: +epoch: 21 +2022-04-10 13:31:38.240856: train loss : 0.1283 +2022-04-10 13:32:04.720810: validation loss: 0.1248 +2022-04-10 13:32:04.721455: Average global foreground Dice: [0.6588, 0.7597] +2022-04-10 13:32:04.721545: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 13:32:05.110909: lr: 0.000292 +2022-04-10 13:32:05.111096: saving scheduled checkpoint file... +2022-04-10 13:32:05.166841: saving checkpoint... +2022-04-10 13:32:06.163257: done, saving took 1.05 seconds +2022-04-10 13:32:06.176237: done +2022-04-10 13:32:06.206009: saving checkpoint... +2022-04-10 13:32:07.227566: done, saving took 1.05 seconds +2022-04-10 13:32:28.861050: This epoch took 514.963443 s + +2022-04-10 13:32:28.861373: +epoch: 22 +2022-04-10 13:40:13.694591: train loss : 0.1362 +2022-04-10 13:40:40.203345: validation loss: 0.1339 +2022-04-10 13:40:40.203902: Average global foreground Dice: [0.6557, 0.7788] +2022-04-10 13:40:40.203988: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 13:40:40.595093: lr: 0.000291 +2022-04-10 13:40:40.595306: saving scheduled checkpoint file... +2022-04-10 13:40:40.652963: saving checkpoint... +2022-04-10 13:40:41.539101: done, saving took 0.94 seconds +2022-04-10 13:40:41.552782: done +2022-04-10 13:40:41.581458: saving checkpoint... +2022-04-10 13:40:42.494650: done, saving took 0.94 seconds +2022-04-10 13:41:04.680154: This epoch took 515.818670 s + +2022-04-10 13:41:04.680424: +epoch: 23 +2022-04-10 13:48:49.546362: train loss : 0.1318 +2022-04-10 13:49:15.969859: validation loss: 0.1363 +2022-04-10 13:49:15.970432: Average global foreground Dice: [0.6691, 0.8296] +2022-04-10 13:49:15.970520: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 13:49:16.378339: lr: 0.000289 +2022-04-10 13:49:16.378529: saving scheduled checkpoint file... +2022-04-10 13:49:16.435903: saving checkpoint... +2022-04-10 13:49:17.250283: done, saving took 0.87 seconds +2022-04-10 13:49:17.263196: done +2022-04-10 13:49:17.292560: saving checkpoint... +2022-04-10 13:49:18.140007: done, saving took 0.88 seconds +2022-04-10 13:49:51.972338: This epoch took 527.291800 s + +2022-04-10 13:49:51.972694: +epoch: 24 +2022-04-10 13:57:35.897557: train loss : 0.1264 +2022-04-10 13:58:02.365453: validation loss: 0.1209 +2022-04-10 13:58:02.366063: Average global foreground Dice: [0.673, 0.8007] +2022-04-10 13:58:02.366160: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 13:58:02.757211: lr: 0.000288 +2022-04-10 13:58:02.757390: saving scheduled checkpoint file... +2022-04-10 13:58:02.814049: saving checkpoint... +2022-04-10 13:58:03.634842: done, saving took 0.88 seconds +2022-04-10 13:58:03.647964: done +2022-04-10 13:58:03.677928: saving checkpoint... +2022-04-10 13:58:04.668735: done, saving took 1.02 seconds +2022-04-10 13:58:25.510682: This epoch took 513.537880 s + +2022-04-10 13:58:25.510972: +epoch: 25 +2022-04-10 14:06:10.291184: train loss : 0.1344 +2022-04-10 14:06:36.814219: validation loss: 0.1457 +2022-04-10 14:06:36.814806: Average global foreground Dice: [0.6567, 0.7982] +2022-04-10 14:06:36.814891: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 14:06:37.206000: lr: 0.000286 +2022-04-10 14:06:37.206203: saving scheduled checkpoint file... +2022-04-10 14:06:37.263176: saving checkpoint... +2022-04-10 14:06:38.077608: done, saving took 0.87 seconds +2022-04-10 14:06:38.092010: done +2022-04-10 14:06:38.125357: saving checkpoint... +2022-04-10 14:06:38.978630: done, saving took 0.89 seconds +2022-04-10 14:07:20.580197: This epoch took 535.069123 s + +2022-04-10 14:07:20.580524: +epoch: 26 +2022-04-10 14:15:04.843510: train loss : 0.1285 +2022-04-10 14:15:31.288652: validation loss: 0.1136 +2022-04-10 14:15:31.289351: Average global foreground Dice: [0.6717, 0.8117] +2022-04-10 14:15:31.289440: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 14:15:31.681301: lr: 0.000284 +2022-04-10 14:15:31.681486: saving scheduled checkpoint file... +2022-04-10 14:15:31.737681: saving checkpoint... +2022-04-10 14:15:32.560060: done, saving took 0.88 seconds +2022-04-10 14:15:32.574474: done +2022-04-10 14:15:32.607931: saving checkpoint... +2022-04-10 14:15:33.546102: done, saving took 0.97 seconds +2022-04-10 14:15:51.125623: This epoch took 510.544989 s + +2022-04-10 14:15:51.125941: +epoch: 27 +2022-04-10 14:23:35.668289: train loss : 0.1283 +2022-04-10 14:24:02.100888: validation loss: 0.1354 +2022-04-10 14:24:02.101506: Average global foreground Dice: [0.6746, 0.8268] +2022-04-10 14:24:02.101593: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 14:24:02.501421: lr: 0.000282 +2022-04-10 14:24:02.501620: saving scheduled checkpoint file... +2022-04-10 14:24:02.558971: saving checkpoint... +2022-04-10 14:24:03.374263: done, saving took 0.87 seconds +2022-04-10 14:24:03.387265: done +2022-04-10 14:24:03.417263: saving checkpoint... +2022-04-10 14:24:04.248475: done, saving took 0.86 seconds +2022-04-10 14:24:22.496168: This epoch took 511.370062 s + +2022-04-10 14:24:22.496541: +epoch: 28 +2022-04-10 14:32:06.920557: train loss : 0.1259 +2022-04-10 14:32:33.388004: validation loss: 0.1439 +2022-04-10 14:32:33.388676: Average global foreground Dice: [0.6538, 0.7741] +2022-04-10 14:32:33.388770: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 14:32:33.793237: lr: 0.00028 +2022-04-10 14:32:33.793420: saving scheduled checkpoint file... +2022-04-10 14:32:33.849550: saving checkpoint... +2022-04-10 14:32:34.879028: done, saving took 1.09 seconds +2022-04-10 14:32:34.892491: done +2022-04-10 14:32:34.922424: saving checkpoint... +2022-04-10 14:32:35.899621: done, saving took 1.01 seconds +2022-04-10 14:33:03.751230: This epoch took 521.254506 s + +2022-04-10 14:33:03.751529: +epoch: 29 +2022-04-10 14:40:47.992644: train loss : 0.1316 +2022-04-10 14:41:14.459565: validation loss: 0.1396 +2022-04-10 14:41:14.460195: Average global foreground Dice: [0.6596, 0.813] +2022-04-10 14:41:14.460285: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 14:41:14.857873: lr: 0.000278 +2022-04-10 14:41:14.858066: saving scheduled checkpoint file... +2022-04-10 14:41:14.916051: saving checkpoint... +2022-04-10 14:41:15.740700: done, saving took 0.88 seconds +2022-04-10 14:41:15.753756: done +2022-04-10 14:41:15.784206: saving checkpoint... +2022-04-10 14:41:16.630979: done, saving took 0.88 seconds +2022-04-10 14:41:40.002280: This epoch took 516.250626 s + +2022-04-10 14:41:40.002810: +epoch: 30 +2022-04-10 14:49:25.384807: train loss : 0.1219 +2022-04-10 14:49:51.780580: validation loss: 0.1328 +2022-04-10 14:49:51.781229: Average global foreground Dice: [0.6734, 0.8252] +2022-04-10 14:49:51.781315: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 14:49:52.174561: lr: 0.000276 +2022-04-10 14:49:52.174739: saving scheduled checkpoint file... +2022-04-10 14:49:52.230343: saving checkpoint... +2022-04-10 14:49:53.044765: done, saving took 0.87 seconds +2022-04-10 14:49:53.057697: done +2022-04-10 14:49:53.087986: saving checkpoint... +2022-04-10 14:49:53.872538: done, saving took 0.81 seconds +2022-04-10 14:50:14.876592: This epoch took 514.873681 s + +2022-04-10 14:50:14.876891: +epoch: 31 +2022-04-10 14:57:59.018251: train loss : 0.1221 +2022-04-10 14:58:25.482395: validation loss: 0.1302 +2022-04-10 14:58:25.483011: Average global foreground Dice: [0.6671, 0.8113] +2022-04-10 14:58:25.483101: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 14:58:25.963676: lr: 0.000274 +2022-04-10 14:58:25.963878: saving scheduled checkpoint file... +2022-04-10 14:58:26.020991: saving checkpoint... +2022-04-10 14:58:26.843817: done, saving took 0.88 seconds +2022-04-10 14:58:26.857701: done +2022-04-10 14:58:26.887722: saving checkpoint... +2022-04-10 14:58:27.723910: done, saving took 0.87 seconds +2022-04-10 14:58:46.158249: This epoch took 511.281236 s + +2022-04-10 14:58:46.158544: +epoch: 32 +2022-04-10 15:06:30.691526: train loss : 0.1242 +2022-04-10 15:06:57.198744: validation loss: 0.1377 +2022-04-10 15:06:57.199334: Average global foreground Dice: [0.6668, 0.78] +2022-04-10 15:06:57.199420: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 15:06:57.592738: lr: 0.000271 +2022-04-10 15:06:57.592927: saving scheduled checkpoint file... +2022-04-10 15:06:57.649013: saving checkpoint... +2022-04-10 15:06:58.489680: done, saving took 0.90 seconds +2022-04-10 15:06:58.502633: done +2022-04-10 15:06:58.531625: saving checkpoint... +2022-04-10 15:06:59.443648: done, saving took 0.94 seconds +2022-04-10 15:07:32.971561: This epoch took 526.812918 s + +2022-04-10 15:07:32.971838: +epoch: 33 +2022-04-10 15:15:16.335568: train loss : 0.1276 +2022-04-10 15:15:42.779854: validation loss: 0.1262 +2022-04-10 15:15:42.780493: Average global foreground Dice: [0.6582, 0.7951] +2022-04-10 15:15:42.780579: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 15:15:43.174975: lr: 0.000269 +2022-04-10 15:15:43.175161: saving scheduled checkpoint file... +2022-04-10 15:15:43.233563: saving checkpoint... +2022-04-10 15:15:44.046446: done, saving took 0.87 seconds +2022-04-10 15:15:44.061023: done +2022-04-10 15:15:44.093986: saving checkpoint... +2022-04-10 15:15:44.937058: done, saving took 0.88 seconds +2022-04-10 15:16:09.125626: This epoch took 516.153557 s + +2022-04-10 15:16:09.125950: +epoch: 34 +2022-04-10 15:23:53.500041: train loss : 0.1234 +2022-04-10 15:24:19.916746: validation loss: 0.1222 +2022-04-10 15:24:19.917358: Average global foreground Dice: [0.6833, 0.8191] +2022-04-10 15:24:19.917466: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 15:24:20.319601: lr: 0.000266 +2022-04-10 15:24:20.319795: saving scheduled checkpoint file... +2022-04-10 15:24:20.377032: saving checkpoint... +2022-04-10 15:24:21.205632: done, saving took 0.89 seconds +2022-04-10 15:24:21.218979: done +2022-04-10 15:24:21.249791: saving checkpoint... +2022-04-10 15:24:22.103273: done, saving took 0.88 seconds +2022-04-10 15:25:04.843268: This epoch took 535.717218 s + +2022-04-10 15:25:04.843692: +epoch: 35 +2022-04-10 15:32:49.050870: train loss : 0.1262 +2022-04-10 15:33:15.471733: validation loss: 0.1308 +2022-04-10 15:33:15.472346: Average global foreground Dice: [0.6841, 0.8307] +2022-04-10 15:33:15.472435: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 15:33:15.868579: lr: 0.000264 +2022-04-10 15:33:15.868767: saving scheduled checkpoint file... +2022-04-10 15:33:15.926542: saving checkpoint... +2022-04-10 15:33:16.742320: done, saving took 0.87 seconds +2022-04-10 15:33:16.755079: done +2022-04-10 15:33:16.785022: saving checkpoint... +2022-04-10 15:33:17.623916: done, saving took 0.87 seconds +2022-04-10 15:34:44.608119: This epoch took 579.764162 s + +2022-04-10 15:34:44.608399: +epoch: 36 +2022-04-10 15:42:28.694982: train loss : 0.1238 +2022-04-10 15:42:55.139494: validation loss: 0.1320 +2022-04-10 15:42:55.140136: Average global foreground Dice: [0.6764, 0.8164] +2022-04-10 15:42:55.140226: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 15:42:55.536073: lr: 0.000261 +2022-04-10 15:42:55.536278: saving scheduled checkpoint file... +2022-04-10 15:42:55.593449: saving checkpoint... +2022-04-10 15:42:56.481532: done, saving took 0.95 seconds +2022-04-10 15:42:56.494972: done +2022-04-10 15:42:56.524474: saving checkpoint... +2022-04-10 15:42:57.390097: done, saving took 0.89 seconds +2022-04-10 15:43:15.633323: This epoch took 511.024823 s + +2022-04-10 15:43:15.633590: +epoch: 37 +2022-04-10 15:50:59.355224: train loss : 0.1200 +2022-04-10 15:51:25.819361: validation loss: 0.1254 +2022-04-10 15:51:25.819967: Average global foreground Dice: [0.6697, 0.8039] +2022-04-10 15:51:25.820054: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 15:51:26.215863: lr: 0.000258 +2022-04-10 15:51:26.216044: saving scheduled checkpoint file... +2022-04-10 15:51:26.272840: saving checkpoint... +2022-04-10 15:51:27.092546: done, saving took 0.88 seconds +2022-04-10 15:51:27.105568: done +2022-04-10 15:51:27.135628: saving checkpoint... +2022-04-10 15:51:27.958562: done, saving took 0.85 seconds +2022-04-10 15:51:48.602489: This epoch took 512.968800 s + +2022-04-10 15:51:48.602772: +epoch: 38 +2022-04-10 15:59:33.006900: train loss : 0.1190 +2022-04-10 15:59:59.450432: validation loss: 0.1380 +2022-04-10 15:59:59.451063: Average global foreground Dice: [0.6761, 0.8213] +2022-04-10 15:59:59.451165: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 15:59:59.850525: lr: 0.000255 +2022-04-10 15:59:59.850713: saving scheduled checkpoint file... +2022-04-10 15:59:59.907736: saving checkpoint... +2022-04-10 16:00:00.731169: done, saving took 0.88 seconds +2022-04-10 16:00:00.745807: done +2022-04-10 16:00:00.779578: saving checkpoint... +2022-04-10 16:00:01.617216: done, saving took 0.87 seconds +2022-04-10 16:00:33.311090: This epoch took 524.708216 s + +2022-04-10 16:00:33.311389: +epoch: 39 +2022-04-10 16:08:17.854805: train loss : 0.1224 +2022-04-10 16:08:44.288782: validation loss: 0.1213 +2022-04-10 16:08:44.289325: Average global foreground Dice: [0.6668, 0.8076] +2022-04-10 16:08:44.289408: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 16:08:44.684845: lr: 0.000252 +2022-04-10 16:08:44.685032: saving scheduled checkpoint file... +2022-04-10 16:08:44.741791: saving checkpoint... +2022-04-10 16:08:45.625904: done, saving took 0.94 seconds +2022-04-10 16:08:45.638534: done +2022-04-10 16:08:45.666790: saving checkpoint... +2022-04-10 16:08:46.570761: done, saving took 0.93 seconds +2022-04-10 16:09:11.486919: This epoch took 518.175428 s + +2022-04-10 16:09:11.487210: +epoch: 40 +2022-04-10 16:16:56.085935: train loss : 0.1210 +2022-04-10 16:17:22.497313: validation loss: 0.1290 +2022-04-10 16:17:22.498268: Average global foreground Dice: [0.6823, 0.8258] +2022-04-10 16:17:22.498356: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 16:17:22.898221: lr: 0.000249 +2022-04-10 16:17:22.898414: saving scheduled checkpoint file... +2022-04-10 16:17:22.955707: saving checkpoint... +2022-04-10 16:17:23.826893: done, saving took 0.93 seconds +2022-04-10 16:17:23.844111: done +2022-04-10 16:17:23.884347: saving checkpoint... +2022-04-10 16:17:24.741935: done, saving took 0.90 seconds +2022-04-10 16:17:41.128432: This epoch took 509.641095 s + +2022-04-10 16:17:41.128837: +epoch: 41 +2022-04-10 16:25:25.168788: train loss : 0.1161 +2022-04-10 16:25:51.590281: validation loss: 0.1345 +2022-04-10 16:25:51.590930: Average global foreground Dice: [0.6703, 0.8112] +2022-04-10 16:25:51.591014: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 16:25:51.982786: lr: 0.000246 +2022-04-10 16:25:51.982971: saving scheduled checkpoint file... +2022-04-10 16:25:52.041010: saving checkpoint... +2022-04-10 16:25:52.862645: done, saving took 0.88 seconds +2022-04-10 16:25:52.877097: done +2022-04-10 16:25:52.910192: saving checkpoint... +2022-04-10 16:25:53.717356: done, saving took 0.84 seconds +2022-04-10 16:26:10.727857: This epoch took 509.598911 s + +2022-04-10 16:26:10.728193: +epoch: 42 +2022-04-10 16:33:54.883646: train loss : 0.1175 +2022-04-10 16:34:21.286144: validation loss: 0.1158 +2022-04-10 16:34:21.286749: Average global foreground Dice: [0.6801, 0.8278] +2022-04-10 16:34:21.286836: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 16:34:21.682561: lr: 0.000243 +2022-04-10 16:34:21.682735: saving scheduled checkpoint file... +2022-04-10 16:34:21.738679: saving checkpoint... +2022-04-10 16:34:22.561829: done, saving took 0.88 seconds +2022-04-10 16:34:22.576387: done +2022-04-10 16:34:22.610392: saving checkpoint... +2022-04-10 16:34:23.640694: done, saving took 1.06 seconds +2022-04-10 16:34:40.223772: This epoch took 509.495473 s + +2022-04-10 16:34:40.224072: +epoch: 43 +2022-04-10 16:42:25.034397: train loss : 0.1192 +2022-04-10 16:42:51.475712: validation loss: 0.1188 +2022-04-10 16:42:51.476380: Average global foreground Dice: [0.6669, 0.8181] +2022-04-10 16:42:51.476470: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 16:42:51.873569: lr: 0.00024 +2022-04-10 16:42:51.873765: saving scheduled checkpoint file... +2022-04-10 16:42:51.932375: saving checkpoint... +2022-04-10 16:42:52.872739: done, saving took 1.00 seconds +2022-04-10 16:42:52.884951: done +2022-04-10 16:42:52.923041: saving checkpoint... +2022-04-10 16:42:53.797025: done, saving took 0.91 seconds +2022-04-10 16:43:11.915216: This epoch took 511.691028 s + +2022-04-10 16:43:11.915486: +epoch: 44 +2022-04-10 16:50:56.180829: train loss : 0.1151 +2022-04-10 16:51:22.595149: validation loss: 0.1148 +2022-04-10 16:51:22.595716: Average global foreground Dice: [0.6638, 0.8237] +2022-04-10 16:51:22.595804: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 16:51:22.994210: lr: 0.000237 +2022-04-10 16:51:22.994389: saving scheduled checkpoint file... +2022-04-10 16:51:23.051076: saving checkpoint... +2022-04-10 16:51:23.874643: done, saving took 0.88 seconds +2022-04-10 16:51:23.887641: done +2022-04-10 16:51:23.917653: saving checkpoint... +2022-04-10 16:51:24.744459: done, saving took 0.86 seconds +2022-04-10 16:51:48.896856: This epoch took 516.981267 s + +2022-04-10 16:51:48.897141: +epoch: 45 +2022-04-10 16:59:33.551281: train loss : 0.1183 +2022-04-10 16:59:59.957427: validation loss: 0.1165 +2022-04-10 16:59:59.957966: Average global foreground Dice: [0.6728, 0.8305] +2022-04-10 16:59:59.958050: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 17:00:00.346981: lr: 0.000233 +2022-04-10 17:00:00.347173: saving scheduled checkpoint file... +2022-04-10 17:00:00.403191: saving checkpoint... +2022-04-10 17:00:01.224226: done, saving took 0.88 seconds +2022-04-10 17:00:01.237031: done +2022-04-10 17:00:01.266955: saving checkpoint... +2022-04-10 17:00:02.090358: done, saving took 0.85 seconds +2022-04-10 17:00:21.914198: This epoch took 513.016935 s + +2022-04-10 17:00:21.914511: +epoch: 46 +2022-04-10 17:08:06.070720: train loss : 0.1206 +2022-04-10 17:08:32.516577: validation loss: 0.1104 +2022-04-10 17:08:32.517176: Average global foreground Dice: [0.6927, 0.833] +2022-04-10 17:08:32.517264: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 17:08:32.908116: lr: 0.00023 +2022-04-10 17:08:32.908305: saving scheduled checkpoint file... +2022-04-10 17:08:32.965157: saving checkpoint... +2022-04-10 17:08:33.775437: done, saving took 0.87 seconds +2022-04-10 17:08:33.789555: done +2022-04-10 17:08:33.823405: saving checkpoint... +2022-04-10 17:08:34.687897: done, saving took 0.90 seconds +2022-04-10 17:08:52.124776: This epoch took 510.210159 s + +2022-04-10 17:08:52.125062: +epoch: 47 +2022-04-10 17:16:36.440623: train loss : 0.1122 +2022-04-10 17:17:02.865947: validation loss: 0.1064 +2022-04-10 17:17:02.866593: Average global foreground Dice: [0.6858, 0.8226] +2022-04-10 17:17:02.866681: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 17:17:03.261701: lr: 0.000226 +2022-04-10 17:17:03.261898: saving scheduled checkpoint file... +2022-04-10 17:17:03.318717: saving checkpoint... +2022-04-10 17:17:04.221603: done, saving took 0.96 seconds +2022-04-10 17:17:04.235050: done +2022-04-10 17:17:04.264794: saving checkpoint... +2022-04-10 17:17:05.165858: done, saving took 0.93 seconds +2022-04-10 17:17:20.870166: This epoch took 508.744987 s + +2022-04-10 17:17:20.870448: +epoch: 48 +2022-04-10 17:25:05.172134: train loss : 0.1208 +2022-04-10 17:25:31.626356: validation loss: 0.1248 +2022-04-10 17:25:31.627004: Average global foreground Dice: [0.67, 0.8427] +2022-04-10 17:25:31.627095: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 17:25:32.025546: lr: 0.000223 +2022-04-10 17:25:32.025745: saving scheduled checkpoint file... +2022-04-10 17:25:32.081989: saving checkpoint... +2022-04-10 17:25:32.911008: done, saving took 0.89 seconds +2022-04-10 17:25:32.924431: done +2022-04-10 17:25:32.954215: saving checkpoint... +2022-04-10 17:25:33.777052: done, saving took 0.85 seconds +2022-04-10 17:25:50.387898: This epoch took 509.517355 s + +2022-04-10 17:25:50.388192: +epoch: 49 +2022-04-10 17:33:34.298388: train loss : 0.1160 +2022-04-10 17:34:00.759029: validation loss: 0.1186 +2022-04-10 17:34:00.759623: Average global foreground Dice: [0.6781, 0.8164] +2022-04-10 17:34:00.759709: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 17:34:01.154538: lr: 0.000219 +2022-04-10 17:34:01.154736: saving scheduled checkpoint file... +2022-04-10 17:34:01.212195: saving checkpoint... +2022-04-10 17:34:02.036551: done, saving took 0.88 seconds +2022-04-10 17:34:02.049695: done +2022-04-10 17:34:02.081493: saving checkpoint... +2022-04-10 17:34:03.100510: done, saving took 1.05 seconds +2022-04-10 17:34:20.323286: This epoch took 509.934981 s + +2022-04-10 17:34:20.323631: +epoch: 50 +2022-04-10 17:42:04.581979: train loss : 0.1102 +2022-04-10 17:42:31.013095: validation loss: 0.1094 +2022-04-10 17:42:31.013748: Average global foreground Dice: [0.6877, 0.8159] +2022-04-10 17:42:31.013835: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 17:42:31.411103: lr: 0.000215 +2022-04-10 17:42:31.411344: saving scheduled checkpoint file... +2022-04-10 17:42:31.468648: saving checkpoint... +2022-04-10 17:42:32.290036: done, saving took 0.88 seconds +2022-04-10 17:42:32.302916: done +2022-04-10 17:42:32.332727: saving checkpoint... +2022-04-10 17:42:33.158389: done, saving took 0.86 seconds +2022-04-10 17:42:50.719433: This epoch took 510.395678 s + +2022-04-10 17:42:50.719709: +epoch: 51 +2022-04-10 17:50:35.299026: train loss : 0.1140 +2022-04-10 17:51:01.694472: validation loss: 0.1296 +2022-04-10 17:51:01.695050: Average global foreground Dice: [0.6814, 0.8152] +2022-04-10 17:51:01.695205: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 17:51:02.091885: lr: 0.000212 +2022-04-10 17:51:02.092079: saving scheduled checkpoint file... +2022-04-10 17:51:02.148270: saving checkpoint... +2022-04-10 17:51:03.026335: done, saving took 0.93 seconds +2022-04-10 17:51:03.040011: done +2022-04-10 17:51:03.070823: saving checkpoint... +2022-04-10 17:51:03.920826: done, saving took 0.88 seconds +2022-04-10 17:51:20.021781: This epoch took 509.301962 s + +2022-04-10 17:51:20.022107: +epoch: 52 +2022-04-10 17:59:03.999905: train loss : 0.1107 +2022-04-10 17:59:30.398362: validation loss: 0.1222 +2022-04-10 17:59:30.398919: Average global foreground Dice: [0.6784, 0.8383] +2022-04-10 17:59:30.399002: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 17:59:30.794969: lr: 0.000208 +2022-04-10 17:59:30.795152: saving scheduled checkpoint file... +2022-04-10 17:59:30.852326: saving checkpoint... +2022-04-10 17:59:31.771925: done, saving took 0.98 seconds +2022-04-10 17:59:31.784384: done +2022-04-10 17:59:31.813557: saving checkpoint... +2022-04-10 17:59:32.740166: done, saving took 0.96 seconds +2022-04-10 17:59:59.598258: This epoch took 519.576030 s + +2022-04-10 17:59:59.598573: +epoch: 53 +2022-04-10 18:07:43.210997: train loss : 0.1139 +2022-04-10 18:08:09.612471: validation loss: 0.1190 +2022-04-10 18:08:09.613055: Average global foreground Dice: [0.676, 0.7978] +2022-04-10 18:08:09.613152: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 18:08:10.010361: lr: 0.000204 +2022-04-10 18:08:10.010548: saving scheduled checkpoint file... +2022-04-10 18:08:10.067865: saving checkpoint... +2022-04-10 18:08:11.062145: done, saving took 1.05 seconds +2022-04-10 18:08:11.075934: done +2022-04-10 18:08:28.214177: This epoch took 508.615498 s + +2022-04-10 18:08:28.214434: +epoch: 54 +2022-04-10 18:16:12.332705: train loss : 0.1131 +2022-04-10 18:16:38.747146: validation loss: 0.1268 +2022-04-10 18:16:38.747795: Average global foreground Dice: [0.6827, 0.8205] +2022-04-10 18:16:38.747883: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 18:16:39.152310: lr: 0.0002 +2022-04-10 18:16:39.152492: saving scheduled checkpoint file... +2022-04-10 18:16:39.209754: saving checkpoint... +2022-04-10 18:16:40.235250: done, saving took 1.08 seconds +2022-04-10 18:16:40.255110: done +2022-04-10 18:17:00.607598: This epoch took 512.393060 s + +2022-04-10 18:17:00.607884: +epoch: 55 +2022-04-10 18:24:44.721942: train loss : 0.1144 +2022-04-10 18:25:11.155861: validation loss: 0.1068 +2022-04-10 18:25:11.156523: Average global foreground Dice: [0.6799, 0.8303] +2022-04-10 18:25:11.156612: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 18:25:11.551606: lr: 0.000196 +2022-04-10 18:25:11.551789: saving scheduled checkpoint file... +2022-04-10 18:25:11.609357: saving checkpoint... +2022-04-10 18:25:12.632370: done, saving took 1.08 seconds +2022-04-10 18:25:12.645816: done +2022-04-10 18:25:12.679531: saving checkpoint... +2022-04-10 18:25:13.691989: done, saving took 1.05 seconds +2022-04-10 18:25:32.792464: This epoch took 512.184472 s + +2022-04-10 18:25:32.792747: +epoch: 56 +2022-04-10 18:33:17.584821: train loss : 0.1065 +2022-04-10 18:33:43.997276: validation loss: 0.1181 +2022-04-10 18:33:43.997907: Average global foreground Dice: [0.678, 0.8407] +2022-04-10 18:33:43.997996: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 18:33:44.394066: lr: 0.000192 +2022-04-10 18:33:44.394270: saving scheduled checkpoint file... +2022-04-10 18:33:44.450553: saving checkpoint... +2022-04-10 18:33:45.427743: done, saving took 1.03 seconds +2022-04-10 18:33:45.441588: done +2022-04-10 18:33:45.474707: saving checkpoint... +2022-04-10 18:33:46.468298: done, saving took 1.03 seconds +2022-04-10 18:34:08.129767: This epoch took 515.336923 s + +2022-04-10 18:34:08.130035: +epoch: 57 +2022-04-10 18:41:51.971055: train loss : 0.1110 +2022-04-10 18:42:18.381555: validation loss: 0.1133 +2022-04-10 18:42:18.382137: Average global foreground Dice: [0.6701, 0.8503] +2022-04-10 18:42:18.382223: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 18:42:18.778683: lr: 0.000188 +2022-04-10 18:42:18.778867: saving scheduled checkpoint file... +2022-04-10 18:42:18.835061: saving checkpoint... +2022-04-10 18:42:19.856058: done, saving took 1.08 seconds +2022-04-10 18:42:19.868814: done +2022-04-10 18:42:19.898291: saving checkpoint... +2022-04-10 18:42:20.924872: done, saving took 1.06 seconds +2022-04-10 18:42:38.315782: This epoch took 510.185647 s + +2022-04-10 18:42:38.316048: +epoch: 58 +2022-04-10 18:50:22.589246: train loss : 0.1072 +2022-04-10 18:50:49.023482: validation loss: 0.1275 +2022-04-10 18:50:49.024100: Average global foreground Dice: [0.6819, 0.8505] +2022-04-10 18:50:49.024200: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 18:50:49.425480: lr: 0.000185 +2022-04-10 18:50:49.425661: saving scheduled checkpoint file... +2022-04-10 18:50:49.483827: saving checkpoint... +2022-04-10 18:50:50.378861: done, saving took 0.95 seconds +2022-04-10 18:50:50.392459: done +2022-04-10 18:50:50.421762: saving checkpoint... +2022-04-10 18:50:51.184623: done, saving took 0.79 seconds +2022-04-10 18:51:08.298443: This epoch took 509.982294 s + +2022-04-10 18:51:08.298739: +epoch: 59 +2022-04-10 18:58:52.447703: train loss : 0.1149 +2022-04-10 18:59:18.876377: validation loss: 0.1082 +2022-04-10 18:59:18.876933: Average global foreground Dice: [0.6657, 0.8336] +2022-04-10 18:59:18.877015: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 18:59:19.275070: lr: 0.000181 +2022-04-10 18:59:19.275262: saving scheduled checkpoint file... +2022-04-10 18:59:19.332957: saving checkpoint... +2022-04-10 18:59:20.144989: done, saving took 0.87 seconds +2022-04-10 18:59:20.157806: done +2022-04-10 18:59:39.136008: This epoch took 510.837161 s + +2022-04-10 18:59:39.136494: +epoch: 60 +2022-04-10 19:07:23.413004: train loss : 0.1055 +2022-04-10 19:07:49.824942: validation loss: 0.1138 +2022-04-10 19:07:49.825573: Average global foreground Dice: [0.6866, 0.8298] +2022-04-10 19:07:49.825661: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 19:07:50.225639: lr: 0.000176 +2022-04-10 19:07:50.225836: saving scheduled checkpoint file... +2022-04-10 19:07:50.282266: saving checkpoint... +2022-04-10 19:07:51.097497: done, saving took 0.87 seconds +2022-04-10 19:07:51.110651: done +2022-04-10 19:07:51.141016: saving checkpoint... +2022-04-10 19:07:51.968139: done, saving took 0.86 seconds +2022-04-10 19:08:14.329498: This epoch took 515.192860 s + +2022-04-10 19:08:14.329780: +epoch: 61 +2022-04-10 19:15:59.130463: train loss : 0.1149 +2022-04-10 19:16:25.522785: validation loss: 0.1265 +2022-04-10 19:16:25.523375: Average global foreground Dice: [0.6823, 0.8411] +2022-04-10 19:16:25.523461: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 19:16:25.922982: lr: 0.000172 +2022-04-10 19:16:25.923174: saving scheduled checkpoint file... +2022-04-10 19:16:25.979764: saving checkpoint... +2022-04-10 19:16:26.806254: done, saving took 0.88 seconds +2022-04-10 19:16:26.827599: done +2022-04-10 19:16:26.875393: saving checkpoint... +2022-04-10 19:16:27.700795: done, saving took 0.87 seconds +2022-04-10 19:16:46.722876: This epoch took 512.392988 s + +2022-04-10 19:16:46.723271: +epoch: 62 +2022-04-10 19:24:30.682276: train loss : 0.1133 +2022-04-10 19:24:57.073393: validation loss: 0.1242 +2022-04-10 19:24:57.074032: Average global foreground Dice: [0.6717, 0.819] +2022-04-10 19:24:57.074132: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 19:24:57.473785: lr: 0.000168 +2022-04-10 19:24:57.473975: saving scheduled checkpoint file... +2022-04-10 19:24:57.529838: saving checkpoint... +2022-04-10 19:24:58.425819: done, saving took 0.95 seconds +2022-04-10 19:24:58.439954: done +2022-04-10 19:25:16.047675: This epoch took 509.324290 s + +2022-04-10 19:25:16.047951: +epoch: 63 +2022-04-10 19:33:00.216703: train loss : 0.1116 +2022-04-10 19:33:26.640915: validation loss: 0.1079 +2022-04-10 19:33:26.641525: Average global foreground Dice: [0.6792, 0.8322] +2022-04-10 19:33:26.641613: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 19:33:27.041514: lr: 0.000164 +2022-04-10 19:33:27.041698: saving scheduled checkpoint file... +2022-04-10 19:33:27.100014: saving checkpoint... +2022-04-10 19:33:27.916322: done, saving took 0.87 seconds +2022-04-10 19:33:27.930566: done +2022-04-10 19:34:00.179866: This epoch took 524.131808 s + +2022-04-10 19:34:00.180168: +epoch: 64 +2022-04-10 19:41:44.503989: train loss : 0.1053 +2022-04-10 19:42:10.946545: validation loss: 0.1388 +2022-04-10 19:42:10.947120: Average global foreground Dice: [0.6806, 0.8359] +2022-04-10 19:42:10.947219: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 19:42:11.349726: lr: 0.00016 +2022-04-10 19:42:11.349920: saving scheduled checkpoint file... +2022-04-10 19:42:11.406990: saving checkpoint... +2022-04-10 19:42:12.227030: done, saving took 0.88 seconds +2022-04-10 19:42:12.241160: done +2022-04-10 19:42:12.273988: saving checkpoint... +2022-04-10 19:42:13.116432: done, saving took 0.88 seconds +2022-04-10 19:42:33.935011: This epoch took 513.754740 s + +2022-04-10 19:42:33.935300: +epoch: 65 +2022-04-10 19:50:18.183059: train loss : 0.1043 +2022-04-10 19:50:44.627819: validation loss: 0.1199 +2022-04-10 19:50:44.628426: Average global foreground Dice: [0.6882, 0.8332] +2022-04-10 19:50:44.628519: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 19:50:45.028141: lr: 0.000156 +2022-04-10 19:50:45.028336: saving scheduled checkpoint file... +2022-04-10 19:50:45.084832: saving checkpoint... +2022-04-10 19:50:45.992005: done, saving took 0.96 seconds +2022-04-10 19:50:46.004952: done +2022-04-10 19:50:46.033453: saving checkpoint... +2022-04-10 19:50:46.886314: done, saving took 0.88 seconds +2022-04-10 19:51:07.609160: This epoch took 513.673750 s + +2022-04-10 19:51:07.609432: +epoch: 66 +2022-04-10 19:58:52.244639: train loss : 0.1063 +2022-04-10 19:59:18.726210: validation loss: 0.1061 +2022-04-10 19:59:18.726827: Average global foreground Dice: [0.6785, 0.8475] +2022-04-10 19:59:18.726912: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 19:59:19.126150: lr: 0.000152 +2022-04-10 19:59:19.126345: saving scheduled checkpoint file... +2022-04-10 19:59:19.183133: saving checkpoint... +2022-04-10 19:59:20.021074: done, saving took 0.89 seconds +2022-04-10 19:59:20.039160: done +2022-04-10 19:59:20.081260: saving checkpoint... +2022-04-10 19:59:21.088437: done, saving took 1.05 seconds +2022-04-10 19:59:46.658662: This epoch took 519.049109 s + +2022-04-10 19:59:46.659066: +epoch: 67 +2022-04-10 20:07:32.121053: train loss : 0.1109 +2022-04-10 20:07:58.613213: validation loss: 0.1249 +2022-04-10 20:07:58.613809: Average global foreground Dice: [0.6774, 0.8245] +2022-04-10 20:07:58.613895: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 20:07:59.015267: lr: 0.000148 +2022-04-10 20:07:59.015451: saving scheduled checkpoint file... +2022-04-10 20:07:59.071609: saving checkpoint... +2022-04-10 20:08:00.061162: done, saving took 1.05 seconds +2022-04-10 20:08:00.074353: done +2022-04-10 20:08:17.876787: This epoch took 511.217606 s + +2022-04-10 20:08:17.877050: +epoch: 68 +2022-04-10 20:16:02.475337: train loss : 0.1051 +2022-04-10 20:16:28.888537: validation loss: 0.1147 +2022-04-10 20:16:28.889154: Average global foreground Dice: [0.6785, 0.8365] +2022-04-10 20:16:28.889242: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 20:16:29.290329: lr: 0.000144 +2022-04-10 20:16:29.290522: saving scheduled checkpoint file... +2022-04-10 20:16:29.348368: saving checkpoint... +2022-04-10 20:16:30.164791: done, saving took 0.87 seconds +2022-04-10 20:16:30.178703: done +2022-04-10 20:16:30.212388: saving checkpoint... +2022-04-10 20:16:31.050560: done, saving took 0.87 seconds +2022-04-10 20:16:48.825418: This epoch took 510.948266 s + +2022-04-10 20:16:48.825706: +epoch: 69 +2022-04-10 20:24:33.085171: train loss : 0.1041 +2022-04-10 20:24:59.530747: validation loss: 0.1181 +2022-04-10 20:24:59.531362: Average global foreground Dice: [0.6885, 0.8294] +2022-04-10 20:24:59.531446: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 20:24:59.932734: lr: 0.00014 +2022-04-10 20:24:59.932929: saving scheduled checkpoint file... +2022-04-10 20:24:59.989264: saving checkpoint... +2022-04-10 20:25:00.845014: done, saving took 0.91 seconds +2022-04-10 20:25:00.858279: done +2022-04-10 20:25:00.888326: saving checkpoint... +2022-04-10 20:25:01.755199: done, saving took 0.90 seconds +2022-04-10 20:25:32.849222: This epoch took 524.023408 s + +2022-04-10 20:25:32.849483: +epoch: 70 +2022-04-10 20:33:17.221514: train loss : 0.1111 +2022-04-10 20:33:43.645937: validation loss: 0.1109 +2022-04-10 20:33:43.646557: Average global foreground Dice: [0.6816, 0.8459] +2022-04-10 20:33:43.646643: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 20:33:44.048525: lr: 0.000136 +2022-04-10 20:33:44.048718: saving scheduled checkpoint file... +2022-04-10 20:33:44.105591: saving checkpoint... +2022-04-10 20:33:44.918302: done, saving took 0.87 seconds +2022-04-10 20:33:44.932221: done +2022-04-10 20:33:44.965675: saving checkpoint... +2022-04-10 20:33:45.807418: done, saving took 0.88 seconds +2022-04-10 20:34:01.503301: This epoch took 508.653713 s + +2022-04-10 20:34:01.503607: +epoch: 71 +2022-04-10 20:41:45.190751: train loss : 0.1071 +2022-04-10 20:42:11.630767: validation loss: 0.1136 +2022-04-10 20:42:11.631518: Average global foreground Dice: [0.6814, 0.8316] +2022-04-10 20:42:11.631627: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 20:42:12.045070: lr: 0.000132 +2022-04-10 20:42:12.045274: saving scheduled checkpoint file... +2022-04-10 20:42:12.102142: saving checkpoint... +2022-04-10 20:42:12.921462: done, saving took 0.88 seconds +2022-04-10 20:42:12.934599: done +2022-04-10 20:42:12.966705: saving checkpoint... +2022-04-10 20:42:13.805837: done, saving took 0.87 seconds +2022-04-10 20:42:30.006332: This epoch took 508.502621 s + +2022-04-10 20:42:30.006595: +epoch: 72 +2022-04-10 20:50:14.361686: train loss : 0.1033 +2022-04-10 20:50:40.742469: validation loss: 0.1165 +2022-04-10 20:50:40.743101: Average global foreground Dice: [0.6795, 0.8391] +2022-04-10 20:50:40.743203: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 20:50:41.144576: lr: 0.000128 +2022-04-10 20:50:41.144771: saving scheduled checkpoint file... +2022-04-10 20:50:41.200487: saving checkpoint... +2022-04-10 20:50:42.080346: done, saving took 0.94 seconds +2022-04-10 20:50:42.098068: done +2022-04-10 20:50:42.138490: saving checkpoint... +2022-04-10 20:50:43.029720: done, saving took 0.93 seconds +2022-04-10 20:51:06.517043: This epoch took 516.510338 s + +2022-04-10 20:51:06.517421: +epoch: 73 +2022-04-10 20:58:50.825136: train loss : 0.1090 +2022-04-10 20:59:17.230968: validation loss: 0.1088 +2022-04-10 20:59:17.231548: Average global foreground Dice: [0.6845, 0.8406] +2022-04-10 20:59:17.231637: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 20:59:17.644915: lr: 0.000124 +2022-04-10 20:59:17.645104: saving scheduled checkpoint file... +2022-04-10 20:59:17.700877: saving checkpoint... +2022-04-10 20:59:18.517768: done, saving took 0.87 seconds +2022-04-10 20:59:18.530975: done +2022-04-10 20:59:18.561591: saving checkpoint... +2022-04-10 20:59:19.398534: done, saving took 0.87 seconds +2022-04-10 20:59:44.137997: This epoch took 517.620464 s + +2022-04-10 20:59:44.138342: +epoch: 74 +2022-04-10 21:07:28.487542: train loss : 0.1029 +2022-04-10 21:07:54.958955: validation loss: 0.1067 +2022-04-10 21:07:54.959582: Average global foreground Dice: [0.6802, 0.8535] +2022-04-10 21:07:54.959670: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 21:07:55.392074: lr: 0.000119 +2022-04-10 21:07:55.392340: saving scheduled checkpoint file... +2022-04-10 21:07:55.449190: saving checkpoint... +2022-04-10 21:07:56.360561: done, saving took 0.97 seconds +2022-04-10 21:07:56.373199: done +2022-04-10 21:07:56.401571: saving checkpoint... +2022-04-10 21:07:57.380114: done, saving took 1.01 seconds +2022-04-10 21:08:13.644437: This epoch took 509.505987 s + +2022-04-10 21:08:13.644716: +epoch: 75 +2022-04-10 21:15:58.329515: train loss : 0.1083 +2022-04-10 21:16:24.757826: validation loss: 0.1105 +2022-04-10 21:16:24.758451: Average global foreground Dice: [0.6787, 0.8204] +2022-04-10 21:16:24.758538: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 21:16:25.167104: lr: 0.000115 +2022-04-10 21:16:25.167304: saving scheduled checkpoint file... +2022-04-10 21:16:25.222942: saving checkpoint... +2022-04-10 21:16:26.209396: done, saving took 1.04 seconds +2022-04-10 21:16:26.222431: done +2022-04-10 21:16:43.684899: This epoch took 510.040067 s + +2022-04-10 21:16:43.685183: +epoch: 76 +2022-04-10 21:24:27.664006: train loss : 0.1067 +2022-04-10 21:24:54.058547: validation loss: 0.1184 +2022-04-10 21:24:54.059151: Average global foreground Dice: [0.6864, 0.826] +2022-04-10 21:24:54.059239: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 21:24:54.463619: lr: 0.000112 +2022-04-10 21:24:54.463814: saving scheduled checkpoint file... +2022-04-10 21:24:54.521375: saving checkpoint... +2022-04-10 21:24:55.419768: done, saving took 0.96 seconds +2022-04-10 21:24:55.433089: done +2022-04-10 21:25:43.506177: This epoch took 539.820878 s + +2022-04-10 21:25:43.506477: +epoch: 77 +2022-04-10 21:33:27.970421: train loss : 0.1038 +2022-04-10 21:33:54.375484: validation loss: 0.1003 +2022-04-10 21:33:54.376197: Average global foreground Dice: [0.6903, 0.8447] +2022-04-10 21:33:54.376287: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 21:33:54.780878: lr: 0.000108 +2022-04-10 21:33:54.781075: saving scheduled checkpoint file... +2022-04-10 21:33:54.838813: saving checkpoint... +2022-04-10 21:33:55.687676: done, saving took 0.91 seconds +2022-04-10 21:33:55.708688: done +2022-04-10 21:33:55.754733: saving checkpoint... +2022-04-10 21:33:56.715688: done, saving took 1.01 seconds +2022-04-10 21:34:13.951702: This epoch took 510.445126 s + +2022-04-10 21:34:13.951967: +epoch: 78 +2022-04-10 21:41:58.402602: train loss : 0.1055 +2022-04-10 21:42:24.836214: validation loss: 0.1130 +2022-04-10 21:42:24.836861: Average global foreground Dice: [0.6769, 0.8276] +2022-04-10 21:42:24.836946: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 21:42:25.251900: lr: 0.000104 +2022-04-10 21:42:25.252105: saving scheduled checkpoint file... +2022-04-10 21:42:25.307953: saving checkpoint... +2022-04-10 21:42:26.306599: done, saving took 1.05 seconds +2022-04-10 21:42:26.320059: done +2022-04-10 21:42:48.013165: This epoch took 514.061085 s + +2022-04-10 21:42:48.013431: +epoch: 79 +2022-04-10 21:50:31.912853: train loss : 0.1071 +2022-04-10 21:50:58.372591: validation loss: 0.1196 +2022-04-10 21:50:58.373209: Average global foreground Dice: [0.6843, 0.8407] +2022-04-10 21:50:58.373297: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 21:50:58.779419: lr: 0.0001 +2022-04-10 21:50:58.779611: saving scheduled checkpoint file... +2022-04-10 21:50:58.835851: saving checkpoint... +2022-04-10 21:50:59.710677: done, saving took 0.93 seconds +2022-04-10 21:50:59.723847: done +2022-04-10 21:51:16.350377: This epoch took 508.336843 s + +2022-04-10 21:51:16.350659: +epoch: 80 +2022-04-10 21:59:00.787403: train loss : 0.1006 +2022-04-10 21:59:27.253777: validation loss: 0.1196 +2022-04-10 21:59:27.254355: Average global foreground Dice: [0.6714, 0.8375] +2022-04-10 21:59:27.254447: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 21:59:27.657805: lr: 9.6e-05 +2022-04-10 21:59:27.657999: saving scheduled checkpoint file... +2022-04-10 21:59:27.714311: saving checkpoint... +2022-04-10 21:59:28.716470: done, saving took 1.06 seconds +2022-04-10 21:59:28.729259: done +2022-04-10 21:59:46.833516: This epoch took 510.482752 s + +2022-04-10 21:59:46.833987: +epoch: 81 +2022-04-10 22:07:31.159551: train loss : 0.1011 +2022-04-10 22:07:57.566154: validation loss: 0.1180 +2022-04-10 22:07:57.566702: Average global foreground Dice: [0.668, 0.8251] +2022-04-10 22:07:57.566787: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 22:07:57.968626: lr: 9.2e-05 +2022-04-10 22:07:57.968816: saving scheduled checkpoint file... +2022-04-10 22:07:58.024693: saving checkpoint... +2022-04-10 22:07:58.840169: done, saving took 0.87 seconds +2022-04-10 22:07:58.853130: done +2022-04-10 22:08:35.418130: This epoch took 528.584032 s + +2022-04-10 22:08:35.418416: +epoch: 82 +2022-04-10 22:16:20.329367: train loss : 0.1013 +2022-04-10 22:16:46.729283: validation loss: 0.1138 +2022-04-10 22:16:46.729886: Average global foreground Dice: [0.673, 0.8545] +2022-04-10 22:16:46.729971: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 22:16:47.125348: lr: 8.8e-05 +2022-04-10 22:16:47.125546: saving scheduled checkpoint file... +2022-04-10 22:16:47.183120: saving checkpoint... +2022-04-10 22:16:48.087715: done, saving took 0.96 seconds +2022-04-10 22:16:48.100435: done +2022-04-10 22:17:05.743816: This epoch took 510.325293 s + +2022-04-10 22:17:05.744102: +epoch: 83 +2022-04-10 22:24:50.166980: train loss : 0.0978 +2022-04-10 22:25:16.521012: validation loss: 0.1167 +2022-04-10 22:25:16.521608: Average global foreground Dice: [0.6725, 0.8296] +2022-04-10 22:25:16.521699: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 22:25:16.928774: lr: 8.5e-05 +2022-04-10 22:25:16.928965: saving scheduled checkpoint file... +2022-04-10 22:25:16.986414: saving checkpoint... +2022-04-10 22:25:17.804583: done, saving took 0.88 seconds +2022-04-10 22:25:17.818675: done +2022-04-10 22:25:35.578867: This epoch took 509.834507 s + +2022-04-10 22:25:35.579188: +epoch: 84 +2022-04-10 22:33:19.554222: train loss : 0.1029 +2022-04-10 22:33:45.962773: validation loss: 0.1183 +2022-04-10 22:33:45.963323: Average global foreground Dice: [0.6749, 0.8491] +2022-04-10 22:33:45.963412: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 22:33:46.359160: lr: 8.1e-05 +2022-04-10 22:33:46.359357: saving scheduled checkpoint file... +2022-04-10 22:33:46.415623: saving checkpoint... +2022-04-10 22:33:47.250269: done, saving took 0.89 seconds +2022-04-10 22:33:47.269721: done +2022-04-10 22:34:29.004492: This epoch took 533.425202 s + +2022-04-10 22:34:29.004789: +epoch: 85 +2022-04-10 22:42:13.138645: train loss : 0.1068 +2022-04-10 22:42:39.558180: validation loss: 0.1133 +2022-04-10 22:42:39.558820: Average global foreground Dice: [0.6622, 0.8185] +2022-04-10 22:42:39.558923: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 22:42:39.955683: lr: 7.7e-05 +2022-04-10 22:42:39.955875: saving scheduled checkpoint file... +2022-04-10 22:42:40.013230: saving checkpoint... +2022-04-10 22:42:40.833405: done, saving took 0.88 seconds +2022-04-10 22:42:40.846349: done +2022-04-10 22:42:57.502624: This epoch took 508.497737 s + +2022-04-10 22:42:57.502898: +epoch: 86 +2022-04-10 22:50:41.896936: train loss : 0.0952 +2022-04-10 22:51:08.347039: validation loss: 0.1119 +2022-04-10 22:51:08.347669: Average global foreground Dice: [0.678, 0.8513] +2022-04-10 22:51:08.347837: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 22:51:08.743234: lr: 7.4e-05 +2022-04-10 22:51:08.743423: saving scheduled checkpoint file... +2022-04-10 22:51:08.799773: saving checkpoint... +2022-04-10 22:51:09.625768: done, saving took 0.88 seconds +2022-04-10 22:51:09.640283: done +2022-04-10 22:51:27.745762: This epoch took 510.242764 s + +2022-04-10 22:51:27.746026: +epoch: 87 +2022-04-10 22:59:12.076954: train loss : 0.1028 +2022-04-10 22:59:38.494577: validation loss: 0.1176 +2022-04-10 22:59:38.495210: Average global foreground Dice: [0.6747, 0.8458] +2022-04-10 22:59:38.495412: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 22:59:38.890906: lr: 7e-05 +2022-04-10 22:59:38.891107: saving scheduled checkpoint file... +2022-04-10 22:59:38.947201: saving checkpoint... +2022-04-10 22:59:39.993511: done, saving took 1.10 seconds +2022-04-10 22:59:40.007102: done +2022-04-10 22:59:57.703154: This epoch took 509.957018 s + +2022-04-10 22:59:57.703449: +epoch: 88 +2022-04-10 23:07:41.957302: train loss : 0.0994 +2022-04-10 23:08:08.367649: validation loss: 0.1117 +2022-04-10 23:08:08.368330: Average global foreground Dice: [0.6688, 0.8499] +2022-04-10 23:08:08.368418: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 23:08:08.766754: lr: 6.7e-05 +2022-04-10 23:08:08.766953: saving scheduled checkpoint file... +2022-04-10 23:08:08.824034: saving checkpoint... +2022-04-10 23:08:09.799193: done, saving took 1.03 seconds +2022-04-10 23:08:09.813503: done +2022-04-10 23:08:26.260740: This epoch took 508.557184 s + +2022-04-10 23:08:26.261011: +epoch: 89 +2022-04-10 23:16:10.343827: train loss : 0.1032 +2022-04-10 23:16:36.766505: validation loss: 0.1197 +2022-04-10 23:16:36.767093: Average global foreground Dice: [0.6688, 0.8224] +2022-04-10 23:16:36.767190: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 23:16:37.164858: lr: 6.3e-05 +2022-04-10 23:16:37.165061: saving scheduled checkpoint file... +2022-04-10 23:16:37.222269: saving checkpoint... +2022-04-10 23:16:38.031365: done, saving took 0.87 seconds +2022-04-10 23:16:38.045878: done +2022-04-10 23:16:54.738627: This epoch took 508.477517 s + +2022-04-10 23:16:54.738901: +epoch: 90 +2022-04-10 23:24:38.980242: train loss : 0.1039 +2022-04-10 23:25:05.393491: validation loss: 0.1023 +2022-04-10 23:25:05.394150: Average global foreground Dice: [0.6921, 0.8579] +2022-04-10 23:25:05.394243: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 23:25:05.808189: lr: 6e-05 +2022-04-10 23:25:05.808378: saving scheduled checkpoint file... +2022-04-10 23:25:05.866400: saving checkpoint... +2022-04-10 23:25:06.703244: done, saving took 0.89 seconds +2022-04-10 23:25:06.718752: done +2022-04-10 23:25:24.890051: This epoch took 510.151049 s + +2022-04-10 23:25:24.890347: +epoch: 91 +2022-04-10 23:33:08.572079: train loss : 0.1002 +2022-04-10 23:33:35.014786: validation loss: 0.1068 +2022-04-10 23:33:35.015433: Average global foreground Dice: [0.6819, 0.8442] +2022-04-10 23:33:35.015542: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 23:33:35.416020: lr: 5.7e-05 +2022-04-10 23:33:35.416218: saving scheduled checkpoint file... +2022-04-10 23:33:35.473092: saving checkpoint... +2022-04-10 23:33:36.318601: done, saving took 0.90 seconds +2022-04-10 23:33:36.339875: done +2022-04-10 23:33:36.388475: saving checkpoint... +2022-04-10 23:33:37.408233: done, saving took 1.07 seconds +2022-04-10 23:33:57.774623: This epoch took 512.884171 s + +2022-04-10 23:33:57.774908: +epoch: 92 +2022-04-10 23:41:42.513762: train loss : 0.0971 +2022-04-10 23:42:08.950256: validation loss: 0.1189 +2022-04-10 23:42:08.950849: Average global foreground Dice: [0.6669, 0.8311] +2022-04-10 23:42:08.950937: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 23:42:09.361015: lr: 5.4e-05 +2022-04-10 23:42:09.361218: saving scheduled checkpoint file... +2022-04-10 23:42:09.416616: saving checkpoint... +2022-04-10 23:42:10.455077: done, saving took 1.09 seconds +2022-04-10 23:42:10.468337: done +2022-04-10 23:42:29.419454: This epoch took 511.644444 s + +2022-04-10 23:42:29.419739: +epoch: 93 +2022-04-10 23:50:13.616708: train loss : 0.0994 +2022-04-10 23:50:40.072582: validation loss: 0.1222 +2022-04-10 23:50:40.073347: Average global foreground Dice: [0.6704, 0.8395] +2022-04-10 23:50:40.073437: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 23:50:40.474370: lr: 5.1e-05 +2022-04-10 23:50:40.474565: saving scheduled checkpoint file... +2022-04-10 23:50:40.531165: saving checkpoint... +2022-04-10 23:50:41.417615: done, saving took 0.94 seconds +2022-04-10 23:50:41.430809: done +2022-04-10 23:51:01.087107: This epoch took 511.667259 s + +2022-04-10 23:51:01.087400: +epoch: 94 +2022-04-10 23:58:45.297549: train loss : 0.0983 +2022-04-10 23:59:11.734450: validation loss: 0.1203 +2022-04-10 23:59:11.735062: Average global foreground Dice: [0.6508, 0.849] +2022-04-10 23:59:11.735160: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-10 23:59:12.134044: lr: 4.8e-05 +2022-04-10 23:59:12.134236: saving scheduled checkpoint file... +2022-04-10 23:59:12.191916: saving checkpoint... +2022-04-10 23:59:13.012859: done, saving took 0.88 seconds +2022-04-10 23:59:13.025613: done +2022-04-10 23:59:44.263065: This epoch took 523.175564 s + +2022-04-10 23:59:44.263381: +epoch: 95 +2022-04-11 00:07:28.551691: train loss : 0.0973 +2022-04-11 00:07:54.956961: validation loss: 0.1107 +2022-04-11 00:07:54.957588: Average global foreground Dice: [0.6677, 0.8522] +2022-04-11 00:07:54.957672: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 00:07:55.359202: lr: 4.5e-05 +2022-04-11 00:07:55.359390: saving scheduled checkpoint file... +2022-04-11 00:07:55.417162: saving checkpoint... +2022-04-11 00:07:56.294970: done, saving took 0.94 seconds +2022-04-11 00:07:56.310632: done +2022-04-11 00:08:14.391358: This epoch took 510.127879 s + +2022-04-11 00:08:14.391646: +epoch: 96 +2022-04-11 00:15:58.379297: train loss : 0.1001 +2022-04-11 00:16:24.829058: validation loss: 0.1118 +2022-04-11 00:16:24.829665: Average global foreground Dice: [0.6776, 0.804] +2022-04-11 00:16:24.829749: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 00:16:25.232229: lr: 4.2e-05 +2022-04-11 00:16:25.232417: saving scheduled checkpoint file... +2022-04-11 00:16:25.287850: saving checkpoint... +2022-04-11 00:16:26.116492: done, saving took 0.88 seconds +2022-04-11 00:16:26.129269: done +2022-04-11 00:16:47.828458: This epoch took 513.436691 s + +2022-04-11 00:16:47.828829: +epoch: 97 +2022-04-11 00:24:31.950984: train loss : 0.0986 +2022-04-11 00:24:58.329758: validation loss: 0.1188 +2022-04-11 00:24:58.330380: Average global foreground Dice: [0.6682, 0.8525] +2022-04-11 00:24:58.330466: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 00:24:58.732141: lr: 3.9e-05 +2022-04-11 00:24:58.732334: saving scheduled checkpoint file... +2022-04-11 00:24:58.790140: saving checkpoint... +2022-04-11 00:24:59.646938: done, saving took 0.91 seconds +2022-04-11 00:24:59.660223: done +2022-04-11 00:25:24.683101: This epoch took 516.854163 s + +2022-04-11 00:25:24.683408: +epoch: 98 +2022-04-11 00:33:08.420400: train loss : 0.0902 +2022-04-11 00:33:34.909183: validation loss: 0.1224 +2022-04-11 00:33:34.909782: Average global foreground Dice: [0.6841, 0.8114] +2022-04-11 00:33:34.909868: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 00:33:35.310576: lr: 3.6e-05 +2022-04-11 00:33:35.310761: saving scheduled checkpoint file... +2022-04-11 00:33:35.367307: saving checkpoint... +2022-04-11 00:33:36.389843: done, saving took 1.08 seconds +2022-04-11 00:33:36.402301: done +2022-04-11 00:33:53.173640: This epoch took 508.490132 s + +2022-04-11 00:33:53.173917: +epoch: 99 +2022-04-11 00:41:37.226401: train loss : 0.1021 +2022-04-11 00:42:03.608253: validation loss: 0.1000 +2022-04-11 00:42:03.608792: Average global foreground Dice: [0.676, 0.8473] +2022-04-11 00:42:03.608874: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 00:42:04.008083: lr: 3.4e-05 +2022-04-11 00:42:04.008273: saving scheduled checkpoint file... +2022-04-11 00:42:04.065178: saving checkpoint... +2022-04-11 00:42:05.096664: done, saving took 1.09 seconds +2022-04-11 00:42:05.108881: done +2022-04-11 00:42:24.771225: This epoch took 511.597147 s + +2022-04-11 00:42:24.771495: +epoch: 100 +2022-04-11 00:50:08.618661: train loss : 0.1002 +2022-04-11 00:50:35.018148: validation loss: 0.1056 +2022-04-11 00:50:35.018898: Average global foreground Dice: [0.666, 0.8368] +2022-04-11 00:50:35.019013: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 00:50:35.432341: lr: 3.1e-05 +2022-04-11 00:50:35.432531: saving scheduled checkpoint file... +2022-04-11 00:50:35.489046: saving checkpoint... +2022-04-11 00:50:36.532067: done, saving took 1.10 seconds +2022-04-11 00:50:36.544300: done +2022-04-11 00:50:54.964104: This epoch took 510.192511 s + +2022-04-11 00:50:54.964399: +epoch: 101 +2022-04-11 00:58:38.495480: train loss : 0.0997 +2022-04-11 00:59:04.898658: validation loss: 0.1131 +2022-04-11 00:59:04.899313: Average global foreground Dice: [0.6778, 0.8556] +2022-04-11 00:59:04.899400: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 00:59:05.301531: lr: 2.9e-05 +2022-04-11 00:59:05.301713: saving scheduled checkpoint file... +2022-04-11 00:59:05.358215: saving checkpoint... +2022-04-11 00:59:06.325876: done, saving took 1.02 seconds +2022-04-11 00:59:06.339094: done +2022-04-11 00:59:26.638718: This epoch took 511.674201 s + +2022-04-11 00:59:26.639078: +epoch: 102 +2022-04-11 01:07:10.752940: train loss : 0.0990 +2022-04-11 01:07:37.122280: validation loss: 0.1250 +2022-04-11 01:07:37.122834: Average global foreground Dice: [0.6707, 0.8374] +2022-04-11 01:07:37.122919: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 01:07:37.525373: lr: 2.6e-05 +2022-04-11 01:07:37.525567: saving scheduled checkpoint file... +2022-04-11 01:07:37.581885: saving checkpoint... +2022-04-11 01:07:38.411311: done, saving took 0.89 seconds +2022-04-11 01:07:38.424269: done +2022-04-11 01:07:56.737182: This epoch took 510.097934 s + +2022-04-11 01:07:56.737455: +epoch: 103 +2022-04-11 01:15:40.390130: train loss : 0.0967 +2022-04-11 01:16:06.785503: validation loss: 0.1075 +2022-04-11 01:16:06.786162: Average global foreground Dice: [0.6584, 0.8374] +2022-04-11 01:16:06.786257: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 01:16:07.186196: lr: 2.4e-05 +2022-04-11 01:16:07.186376: saving scheduled checkpoint file... +2022-04-11 01:16:07.245436: saving checkpoint... +2022-04-11 01:16:08.223803: done, saving took 1.04 seconds +2022-04-11 01:16:08.237586: done +2022-04-11 01:16:26.410596: This epoch took 509.673037 s + +2022-04-11 01:16:26.410898: +epoch: 104 +2022-04-11 01:24:10.693968: train loss : 0.0962 +2022-04-11 01:24:37.085784: validation loss: 0.1174 +2022-04-11 01:24:37.086400: Average global foreground Dice: [0.6659, 0.8253] +2022-04-11 01:24:37.086489: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 01:24:37.489119: lr: 2.2e-05 +2022-04-11 01:24:37.489316: saving scheduled checkpoint file... +2022-04-11 01:24:37.546186: saving checkpoint... +2022-04-11 01:24:38.549479: done, saving took 1.06 seconds +2022-04-11 01:24:38.562896: done +2022-04-11 01:24:58.902035: This epoch took 512.491037 s + +2022-04-11 01:24:58.902358: +epoch: 105 +2022-04-11 01:32:43.345103: train loss : 0.0967 +2022-04-11 01:33:09.745736: validation loss: 0.1190 +2022-04-11 01:33:09.746476: Average global foreground Dice: [0.6618, 0.8518] +2022-04-11 01:33:09.746567: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 01:33:10.150225: lr: 2e-05 +2022-04-11 01:33:10.150417: saving scheduled checkpoint file... +2022-04-11 01:33:10.207119: saving checkpoint... +2022-04-11 01:33:11.022336: done, saving took 0.87 seconds +2022-04-11 01:33:11.035040: done +2022-04-11 01:33:28.587493: This epoch took 509.685036 s + +2022-04-11 01:33:28.587766: +epoch: 106 +2022-04-11 01:41:12.666342: train loss : 0.0958 +2022-04-11 01:41:39.122420: validation loss: 0.1110 +2022-04-11 01:41:39.123175: Average global foreground Dice: [0.672, 0.8456] +2022-04-11 01:41:39.123265: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 01:41:39.549054: lr: 1.8e-05 +2022-04-11 01:41:39.549254: saving scheduled checkpoint file... +2022-04-11 01:41:39.605859: saving checkpoint... +2022-04-11 01:41:40.427374: done, saving took 0.88 seconds +2022-04-11 01:41:40.440282: done +2022-04-11 01:42:17.150831: This epoch took 528.562963 s + +2022-04-11 01:42:17.151115: +epoch: 107 +2022-04-11 01:50:01.437896: train loss : 0.1020 +2022-04-11 01:50:27.837887: validation loss: 0.1066 +2022-04-11 01:50:27.838478: Average global foreground Dice: [0.6706, 0.8587] +2022-04-11 01:50:27.838567: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 01:50:28.240228: lr: 1.6e-05 +2022-04-11 01:50:28.240415: saving scheduled checkpoint file... +2022-04-11 01:50:28.297682: saving checkpoint... +2022-04-11 01:50:29.306407: done, saving took 1.07 seconds +2022-04-11 01:50:29.319225: done +2022-04-11 01:50:52.725280: This epoch took 515.574038 s + +2022-04-11 01:50:52.725604: +epoch: 108 +2022-04-11 01:58:37.514179: train loss : 0.1048 +2022-04-11 01:59:03.925513: validation loss: 0.0996 +2022-04-11 01:59:03.926161: Average global foreground Dice: [0.6717, 0.8588] +2022-04-11 01:59:03.926348: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 01:59:04.330530: lr: 1.4e-05 +2022-04-11 01:59:04.330709: saving scheduled checkpoint file... +2022-04-11 01:59:04.386258: saving checkpoint... +2022-04-11 01:59:05.280504: done, saving took 0.95 seconds +2022-04-11 01:59:05.294529: done +2022-04-11 01:59:23.206556: This epoch took 510.480844 s + +2022-04-11 01:59:23.206860: +epoch: 109 +2022-04-11 02:07:07.691472: train loss : 0.1003 +2022-04-11 02:07:34.116198: validation loss: 0.1191 +2022-04-11 02:07:34.116776: Average global foreground Dice: [0.6779, 0.8419] +2022-04-11 02:07:34.116864: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 02:07:34.520701: lr: 1.2e-05 +2022-04-11 02:07:34.520870: saving scheduled checkpoint file... +2022-04-11 02:07:34.577405: saving checkpoint... +2022-04-11 02:07:35.395641: done, saving took 0.87 seconds +2022-04-11 02:07:35.408465: done +2022-04-11 02:07:56.198654: This epoch took 512.991689 s + +2022-04-11 02:07:56.198972: +epoch: 110 +2022-04-11 02:15:40.677786: train loss : 0.0900 +2022-04-11 02:16:07.066796: validation loss: 0.1121 +2022-04-11 02:16:07.067431: Average global foreground Dice: [0.6768, 0.8574] +2022-04-11 02:16:07.067522: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 02:16:07.470771: lr: 1.1e-05 +2022-04-11 02:16:07.470952: saving scheduled checkpoint file... +2022-04-11 02:16:07.528092: saving checkpoint... +2022-04-11 02:16:08.348845: done, saving took 0.88 seconds +2022-04-11 02:16:08.362813: done +2022-04-11 02:16:27.957670: This epoch took 511.758583 s + +2022-04-11 02:16:27.957937: +epoch: 111 +2022-04-11 02:24:12.534302: train loss : 0.0937 +2022-04-11 02:24:38.922717: validation loss: 0.1108 +2022-04-11 02:24:38.923304: Average global foreground Dice: [0.6711, 0.8324] +2022-04-11 02:24:38.923392: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 02:24:39.341762: lr: 9e-06 +2022-04-11 02:24:39.341936: saving scheduled checkpoint file... +2022-04-11 02:24:39.401751: saving checkpoint... +2022-04-11 02:24:40.233243: done, saving took 0.89 seconds +2022-04-11 02:24:40.246188: done +2022-04-11 02:24:59.187464: This epoch took 511.229429 s + +2022-04-11 02:24:59.187735: +epoch: 112 +2022-04-11 02:32:42.822770: train loss : 0.0990 +2022-04-11 02:33:09.284059: validation loss: 0.1129 +2022-04-11 02:33:09.284718: Average global foreground Dice: [0.6703, 0.8522] +2022-04-11 02:33:09.284807: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 02:33:09.692517: lr: 8e-06 +2022-04-11 02:33:09.692719: saving scheduled checkpoint file... +2022-04-11 02:33:09.752110: saving checkpoint... +2022-04-11 02:33:10.768755: done, saving took 1.08 seconds +2022-04-11 02:33:10.782397: done +2022-04-11 02:33:29.761414: This epoch took 510.573584 s + +2022-04-11 02:33:29.761697: +epoch: 113 +2022-04-11 02:41:13.891632: train loss : 0.0943 +2022-04-11 02:41:40.315355: validation loss: 0.1016 +2022-04-11 02:41:40.316029: Average global foreground Dice: [0.6762, 0.8627] +2022-04-11 02:41:40.316132: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 02:41:40.719229: lr: 7e-06 +2022-04-11 02:41:40.719421: saving scheduled checkpoint file... +2022-04-11 02:41:40.776216: saving checkpoint... +2022-04-11 02:41:41.793904: done, saving took 1.07 seconds +2022-04-11 02:41:41.807141: done +2022-04-11 02:41:41.836925: saving checkpoint... +2022-04-11 02:41:42.808490: done, saving took 1.00 seconds +2022-04-11 02:42:09.560593: This epoch took 519.798784 s + +2022-04-11 02:42:09.560908: +epoch: 114 +2022-04-11 02:49:53.667938: train loss : 0.0916 +2022-04-11 02:50:20.091251: validation loss: 0.1308 +2022-04-11 02:50:20.091951: Average global foreground Dice: [0.6721, 0.8363] +2022-04-11 02:50:20.092038: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 02:50:20.497740: lr: 6e-06 +2022-04-11 02:50:20.497926: saving scheduled checkpoint file... +2022-04-11 02:50:20.553823: saving checkpoint... +2022-04-11 02:50:21.590157: done, saving took 1.09 seconds +2022-04-11 02:50:21.603029: done +2022-04-11 02:50:44.950327: This epoch took 515.389312 s + +2022-04-11 02:50:44.950629: +epoch: 115 +2022-04-11 02:58:29.238728: train loss : 0.0989 +2022-04-11 02:58:55.640378: validation loss: 0.0862 +2022-04-11 02:58:55.640920: Average global foreground Dice: [0.6794, 0.8411] +2022-04-11 02:58:55.641003: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 02:58:56.046362: lr: 5e-06 +2022-04-11 02:58:56.046554: saving scheduled checkpoint file... +2022-04-11 02:58:56.103735: saving checkpoint... +2022-04-11 02:58:57.014611: done, saving took 0.97 seconds +2022-04-11 02:58:57.027855: done +2022-04-11 02:59:18.129505: This epoch took 513.178772 s + +2022-04-11 02:59:18.129809: +epoch: 116 +2022-04-11 03:07:02.338817: train loss : 0.0955 +2022-04-11 03:07:28.774480: validation loss: 0.0984 +2022-04-11 03:07:28.775194: Average global foreground Dice: [0.6729, 0.8408] +2022-04-11 03:07:28.775284: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 03:07:29.179642: lr: 4e-06 +2022-04-11 03:07:29.179827: saving scheduled checkpoint file... +2022-04-11 03:07:29.236301: saving checkpoint... +2022-04-11 03:07:30.282170: done, saving took 1.10 seconds +2022-04-11 03:07:30.307718: done +2022-04-11 03:07:53.854745: This epoch took 515.724835 s + +2022-04-11 03:07:53.855031: +epoch: 117 +2022-04-11 03:15:38.147197: train loss : 0.1001 +2022-04-11 03:16:04.535587: validation loss: 0.1082 +2022-04-11 03:16:04.536175: Average global foreground Dice: [0.6766, 0.8522] +2022-04-11 03:16:04.536261: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 03:16:04.940399: lr: 3e-06 +2022-04-11 03:16:04.940575: saving scheduled checkpoint file... +2022-04-11 03:16:04.998541: saving checkpoint... +2022-04-11 03:16:06.022472: done, saving took 1.08 seconds +2022-04-11 03:16:06.034996: done +2022-04-11 03:16:06.064238: saving checkpoint... +2022-04-11 03:16:07.092155: done, saving took 1.06 seconds +2022-04-11 03:16:32.860877: This epoch took 519.005736 s + +2022-04-11 03:16:32.861167: +epoch: 118 +2022-04-11 03:24:17.241937: train loss : 0.0953 +2022-04-11 03:24:43.653858: validation loss: 0.1013 +2022-04-11 03:24:43.654464: Average global foreground Dice: [0.6799, 0.8529] +2022-04-11 03:24:43.654552: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 03:24:44.059050: lr: 2e-06 +2022-04-11 03:24:44.059263: saving scheduled checkpoint file... +2022-04-11 03:24:44.116023: saving checkpoint... +2022-04-11 03:24:45.028354: done, saving took 0.97 seconds +2022-04-11 03:24:45.040962: done +2022-04-11 03:24:45.070351: saving checkpoint... +2022-04-11 03:24:45.943426: done, saving took 0.90 seconds +2022-04-11 03:25:06.090891: This epoch took 513.229614 s + +2022-04-11 03:25:06.091197: +epoch: 119 +2022-04-11 03:32:49.935872: train loss : 0.0902 +2022-04-11 03:33:16.314792: validation loss: 0.1010 +2022-04-11 03:33:16.315364: Average global foreground Dice: [0.677, 0.8553] +2022-04-11 03:33:16.315454: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 03:33:16.720098: lr: 1e-06 +2022-04-11 03:33:16.720294: saving scheduled checkpoint file... +2022-04-11 03:33:16.777173: saving checkpoint... +2022-04-11 03:33:17.601157: done, saving took 0.88 seconds +2022-04-11 03:33:17.615427: done +2022-04-11 03:33:17.648793: saving checkpoint... +2022-04-11 03:33:18.499983: done, saving took 0.88 seconds +2022-04-11 03:33:43.146490: This epoch took 517.055192 s + +2022-04-11 03:33:43.146762: +epoch: 120 +2022-04-11 03:41:27.493778: train loss : 0.0995 +2022-04-11 03:41:53.864388: validation loss: 0.1036 +2022-04-11 03:41:53.864993: Average global foreground Dice: [0.672, 0.8379] +2022-04-11 03:41:53.865082: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 03:41:54.271022: lr: 1e-06 +2022-04-11 03:41:54.271216: saving scheduled checkpoint file... +2022-04-11 03:41:54.329515: saving checkpoint... +2022-04-11 03:41:55.278565: done, saving took 1.01 seconds +2022-04-11 03:41:55.291958: done +2022-04-11 03:42:16.711583: This epoch took 513.564715 s + +2022-04-11 03:42:16.711873: +epoch: 121 +2022-04-11 03:50:00.528970: train loss : 0.0930 +2022-04-11 03:50:26.907942: validation loss: 0.1102 +2022-04-11 03:50:26.908743: Average global foreground Dice: [0.6683, 0.839] +2022-04-11 03:50:26.908829: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 03:50:27.325293: lr: 1e-06 +2022-04-11 03:50:27.325490: saving scheduled checkpoint file... +2022-04-11 03:50:27.383005: saving checkpoint... +2022-04-11 03:50:28.426841: done, saving took 1.10 seconds +2022-04-11 03:50:28.443830: done +2022-04-11 03:50:50.745321: This epoch took 514.033344 s + +2022-04-11 03:50:50.745633: +epoch: 122 +2022-04-11 03:58:34.703974: train loss : 0.0927 +2022-04-11 03:59:01.127075: validation loss: 0.1155 +2022-04-11 03:59:01.127673: Average global foreground Dice: [0.6688, 0.8489] +2022-04-11 03:59:01.127759: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 03:59:01.538339: lr: 0.0 +2022-04-11 03:59:01.538524: saving scheduled checkpoint file... +2022-04-11 03:59:01.597607: saving checkpoint... +2022-04-11 03:59:02.523829: done, saving took 0.99 seconds +2022-04-11 03:59:02.537603: done +2022-04-11 03:59:31.505065: This epoch took 520.759331 s + +2022-04-11 03:59:31.505372: +epoch: 123 +2022-04-11 04:07:16.063560: train loss : 0.0929 +2022-04-11 04:07:42.469208: validation loss: 0.1100 +2022-04-11 04:07:42.469787: Average global foreground Dice: [0.6818, 0.8564] +2022-04-11 04:07:42.469872: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 04:07:42.875737: lr: 0.0 +2022-04-11 04:07:42.875914: saving scheduled checkpoint file... +2022-04-11 04:07:42.932545: saving checkpoint... +2022-04-11 04:07:43.988114: done, saving took 1.11 seconds +2022-04-11 04:07:44.006090: done +2022-04-11 04:08:06.132996: This epoch took 514.627523 s + +2022-04-11 04:08:06.133329: +epoch: 124 +2022-04-11 04:15:50.707030: train loss : 0.0948 +2022-04-11 04:16:17.195768: validation loss: 0.1099 +2022-04-11 04:16:17.196349: Average global foreground Dice: [0.6777, 0.8503] +2022-04-11 04:16:17.196436: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 04:16:17.600261: lr: 0.0 +2022-04-11 04:16:17.600437: saving scheduled checkpoint file... +2022-04-11 04:16:17.657469: saving checkpoint... +2022-04-11 04:16:18.493284: done, saving took 0.89 seconds +2022-04-11 04:16:18.508385: done +2022-04-11 04:16:18.542899: saving checkpoint... +2022-04-11 04:16:19.395175: done, saving took 0.89 seconds +2022-04-11 04:16:41.287956: This epoch took 515.154497 s + +2022-04-11 04:16:41.358451: saving checkpoint... +2022-04-11 04:16:42.239291: done, saving took 0.95 seconds diff --git a/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_4/debug.json b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_4/debug.json new file mode 100644 index 0000000000000000000000000000000000000000..3656bf6f74d017ab803535fa9e70f98d5c958c1d --- /dev/null +++ b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_4/debug.json @@ -0,0 +1,106 @@ +{ + "all_tr_losses": "[0.33880347, 0.23667154, 0.20787217, 0.17866741, 0.1688485, 0.15650244, 0.17122895, 0.16600037, 0.1662374, 0.14666808, 0.14834467, 0.14682831, 0.137326, 0.14589843, 0.14086902, 0.13520584, 0.13806397, 0.13688643, 0.14157186, 0.13476749, 0.13293879, 0.13792813, 0.12690659, 0.13224676, 0.12589811, 0.13101529, 0.121173665, 0.12743156, 0.12973718, 0.13155001, 0.12538257, 0.12880784, 0.11983098, 0.12570767, 0.12043454, 0.11550015, 0.12463293, 0.12751071, 0.12391978, 0.12015278, 0.122464985, 0.122478046, 0.11480512, 0.11710864, 0.11619793, 0.10997589, 0.121831164, 0.11592714, 0.11416377, 0.11117799, 0.12309044, 0.11643512, 0.111628205, 0.11246636, 0.11181403, 0.11948247, 0.11015611, 0.11349469, 0.10995185, 0.10290526, 0.11850418, 0.10616533, 0.10980081, 0.10753825, 0.10982819, 0.10064859, 0.10937555]", + "all_val_eval_metrics": "[0.24441803128538458, 0.3646200873534005, 0.4073836012941684, 0.5280979869389617, 0.5576282711651834, 0.5587112244661616, 0.616372627127902, 0.6399068696652392, 0.6740998923677397, 0.6733766897687383, 0.685950202676759, 0.6730891690792249, 0.6578517604180589, 0.6903512099612064, 0.6788547685772561, 0.6683383469176356, 0.6963632808493894, 0.706660171040598, 0.708526498652341, 0.7028626785285068, 0.6997068508123063, 0.671519035061525, 0.7146784462843951, 0.6687458102034116, 0.7247048220907977, 0.7174996675681652, 0.7223397463625885, 0.7132866945673728, 0.735415754668584, 0.7273902097652161, 0.7007612073468041, 0.7301585193118795, 0.7303057009425757, 0.7373153163204972, 0.719404512274642, 0.7335313201714158, 0.7210674690019635, 0.7344483733135108, 0.7313691909763442, 0.7268537501917084, 0.7311546278074248, 0.7148105526320396, 0.7228389278014391, 0.7419200118250955, 0.7401590515296674, 0.7414424569003581, 0.721725488437178, 0.736920930365019, 0.7281322335384874, 0.7534104846465608, 0.7341914504593126, 0.7376134606875642, 0.7306174240033567, 0.7329268753119342, 0.7365485371439937, 0.72699150625508, 0.7234505903870534, 0.7399885495692289, 0.7414867155021148, 0.7468206649655602, 0.744086049173363, 0.7281900657719618, 0.7319929288277218, 0.7345449775248943, 0.7430525868014256, 0.7610991095584116, 0.7367976474920181]", + "all_val_losses": "[0.26946953, 0.21931683, 0.20502658, 0.18677522, 0.17296845, 0.16148563, 0.15531741, 0.17155254, 0.1604264, 0.16581497, 0.15573883, 0.16156188, 0.15351474, 0.12892939, 0.15402336, 0.13484189, 0.14251389, 0.148972, 0.1342064, 0.13652013, 0.1399847, 0.12008429, 0.1167369, 0.1384284, 0.13724485, 0.12959942, 0.12515497, 0.14603548, 0.1266138, 0.13267235, 0.13720961, 0.1475334, 0.12584028, 0.1582998, 0.13734983, 0.15010451, 0.12458315, 0.1293641, 0.13138947, 0.15694696, 0.11656036, 0.12966883, 0.13392729, 0.12910281, 0.13069594, 0.12563862, 0.13533488, 0.1157934, 0.12913467, 0.13287137, 0.1307886, 0.12317187, 0.13162261, 0.134247, 0.114636295, 0.13997744, 0.11866906, 0.10774542, 0.118344985, 0.122042455, 0.15040416, 0.13680579, 0.124536514, 0.12542078, 0.11461933, 0.12392085, 0.12589943]", + "all_val_losses_tr_mode": "[]", + "also_val_in_tr_mode": "False", + "amp_grad_scaler": "", + "args": "Namespace(batch_size=2, config='./configs/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32.yaml', continue_training=False, crop_size=[64, 192, 192], dbs=False, deep_supervision_scales=[], deterministic=False, disable_decoder=False, disable_ds=True, disable_postprocessing_on_folds=False, disable_saving=False, drop_path=0.0, find_lr=False, find_zero_weight_decay=False, fix_ds_net_numpool=False, fold='4', fp32=False, hdfs_base='GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32', initial_lr=0.0003, is_sigmoid=False, is_spatial_aug_only=False, is_ssl=False, layer_decay=1.0, local_rank=0, loss_name='', lrschedule='warmup_cosine', mask_ratio=0.75, max_loss_cal='v1', max_num_epochs=125, merge_femur=False, min_lr=0, model='Generic_TransUNet_max_ppbp', model_params={'is_masked_attn': True, 'max_dec_layers': 3, 'is_max_bottleneck_transformer': True, 'vit_depth': 1, 'max_msda': '', 'is_max_ms': True, 'max_ms_idxs': [-4, -3, -2], 'max_hidden_dim': 192, 'mw': 1.0, 'is_max_ds': True, 'is_masking': True, 'is_max_hungarian': True, 'num_queries': 20, 'is_max_cls': True, 'is_mhsa_float32': True}, n_class=17, network='3d_fullres', network_trainer='nnUNetTrainerV2_DDP', npz=False, opt_betas=None, opt_eps=1e-08, optim_name='adamw', p='nnUNetPlansv2.1', plan_update='', pretrained=False, pretrained_config=None, pretrained_weights=None, rank=0, reclip=None, resume='auto', skip_grad_nan=False, task='Task008_HepaticVessel', task_pretrained='Task801_WORD', total_batch_size=None, use_compressed_data=False, val_final=False, val_folder='validation_raw', valbest=False, validation_only=False, vallatest=False, warmup_epochs=10, weight_decay=3e-05, world_size=8)", + "base_num_features": "32", + "basic_generator_patch_size": "[178 308 233]", + "batch_dice": "True", + "batch_size": "2", + "best_MA_tr_loss_for_patience": "None", + "best_epoch_based_on_MA_tr_loss": "None", + "best_val_eval_criterion_MA": "None", + "classes": "[1, 2]", + "conv_per_stage": "2", + "data_aug_params": "{'selected_data_channels': None, 'selected_seg_channels': [0], 'do_elastic': False, 'elastic_deform_alpha': (0.0, 900.0), 'elastic_deform_sigma': (9.0, 13.0), 'p_eldef': 0.2, 'do_scaling': True, 'scale_range': (0.7, 1.4), 'independent_scale_factor_for_each_axis': False, 'p_independent_scale_per_axis': 1, 'p_scale': 0.2, 'do_rotation': True, 'rotation_x': (-0.5235987755982988, 0.5235987755982988), 'rotation_y': (-0.5235987755982988, 0.5235987755982988), 'rotation_z': (-0.5235987755982988, 0.5235987755982988), 'rotation_p_per_axis': 1, 'p_rot': 0.2, 'random_crop': False, 'random_crop_dist_to_border': None, 'do_gamma': True, 'gamma_retain_stats': True, 'gamma_range': (0.7, 1.5), 'p_gamma': 0.3, 'do_mirror': True, 'mirror_axes': (0, 1, 2), 'dummy_2D': False, 'mask_was_used_for_normalization': OrderedDict([(0, False)]), 'border_mode_data': 'constant', 'all_segmentation_labels': None, 'move_last_seg_chanel_to_data': False, 'cascade_do_cascade_augmentations': False, 'cascade_random_binary_transform_p': 0.4, 'cascade_random_binary_transform_p_per_label': 1, 'cascade_random_binary_transform_size': (1, 8), 'cascade_remove_conn_comp_p': 0.2, 'cascade_remove_conn_comp_max_size_percent_threshold': 0.15, 'cascade_remove_conn_comp_fill_with_other_class_p': 0.0, 'do_additive_brightness': False, 'additive_brightness_p_per_sample': 0.15, 'additive_brightness_p_per_channel': 0.5, 'additive_brightness_mu': 0.0, 'additive_brightness_sigma': 0.1, 'num_threads': 12, 'num_cached_per_thread': 2, 'patch_size_for_spatialtransform': array([ 64, 192, 192])}", + "dataset_directory": "/opt/tiger/project/data/nnUNet_preprocessed/Task008_HepaticVessel", + "deep_supervision_scales": "[[1, 1, 1], [0.5, 0.5, 0.5], [0.25, 0.25, 0.25], [0.125, 0.125, 0.125], [0.0625, 0.0625, 0.0625]]", + "deterministic": "False", + "disable_ds": "True", + "distribute_batch_size": "False", + "dl_tr": "", + "dl_val": "", + "do_dummy_2D_aug": "False", + "ds_loss_weights": "[1. 0. 0. 0. 0.]", + "epoch": "67", + "experiment_name": "nnUNetTrainerV2_DDP", + "find_zero_weight_decay": "False", + "fold": "4", + "folder_with_preprocessed_data": "/opt/tiger/project/data/nnUNet_preprocessed/Task008_HepaticVessel/nnUNetData_plans_v2.1_stage1", + "fp16": "True", + "global_batch_size": "16", + "gt_niftis_folder": "/opt/tiger/project/data/nnUNet_preprocessed/Task008_HepaticVessel/gt_segmentations", + "inference_pad_border_mode": "constant", + "inference_pad_kwargs": "{'constant_values': 0}", + "init_args": "('/opt/tiger/project/data/nnUNet_preprocessed/Task008_HepaticVessel/nnUNetPlansv2.1_plans_3D.pkl', 4, 0, '/opt/tiger/project/data/nnUNet_trained_models/UNet_IN_NANFang/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32', '/opt/tiger/project/data/nnUNet_preprocessed/Task008_HepaticVessel', True, 1, True, False, False, True)", + "initial_lr": "0.0003", + "input_size": "[64, 192, 192]", + "is_spatial_aug_only": "False", + "is_ssl": "False", + "layer_decay": "1.0", + "local_rank": "0", + "log_file": "/opt/tiger/project/data/nnUNet_trained_models/UNet_IN_NANFang/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_4/training_log_2022_5_19_21_33_38.txt", + "loss": "None", + "lr_scheduler": "", + "lr_scheduler_eps": "0.001", + "lr_scheduler_name": "warmup_cosine", + "lr_scheduler_patience": "30", + "lr_threshold": "1e-06", + "max_num_epochs": "125", + "min_lr": "0", + "min_region_size_per_class": "None", + "min_size_per_class": "None", + "model": "Generic_TransUNet_max_ppbp", + "model_params": "{'is_masked_attn': True, 'max_dec_layers': 3, 'is_max_bottleneck_transformer': True, 'vit_depth': 1, 'max_msda': '', 'is_max_ms': True, 'max_ms_idxs': [-4, -3, -2], 'max_hidden_dim': 192, 'mw': 1.0, 'is_max_ds': True, 'is_masking': True, 'is_max_hungarian': True, 'num_queries': 20, 'is_max_cls': True, 'is_mhsa_float32': True}", + "net_conv_kernel_sizes": "[[3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3]]", + "net_num_pool_op_kernel_sizes": "[[2, 2, 2], [2, 2, 2], [2, 2, 2], [2, 2, 2], [1, 2, 2]]", + "net_pool_per_axis": "[4, 5, 5]", + "normalization_schemes": "OrderedDict([(0, 'CT')])", + "num_batches_per_epoch": "250", + "num_classes": "3", + "num_input_channels": "1", + "num_val_batches_per_epoch": "50", + "online_eval_fn": "[]", + "online_eval_foreground_dc": "[]", + "online_eval_fp": "[]", + "online_eval_tp": "[]", + "only_keep_largest_connected_component": "None", + "optim_name": "adamw", + "optimizer": "AdamW (\nParameter Group 0\n amsgrad: False\n betas: (0.9, 0.999)\n eps: 1e-08\n initial_lr: 0.0003\n lr: 0.00015204880106489267\n weight_decay: 3e-05\n)", + "output_folder": "/opt/tiger/project/data/nnUNet_trained_models/UNet_IN_NANFang/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_4", + "output_folder_base": "/opt/tiger/project/data/nnUNet_trained_models/UNet_IN_NANFang/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32", + "oversample_foreground_percent": "0.0", + "pad_all_sides": "None", + "patch_size": "[ 64 192 192]", + "patience": "50", + "pin_memory": "True", + "plans_file": "/opt/tiger/project/data/nnUNet_preprocessed/Task008_HepaticVessel/nnUNetPlansv2.1_plans_3D.pkl", + "reclip": "None", + "regions_class_order": "None", + "resume": "auto", + "save_best_checkpoint": "True", + "save_every": "1", + "save_final_checkpoint": "True", + "save_intermediate_checkpoints": "True", + "save_latest_only": "True", + "stage": "1", + "threeD": "True", + "tr_gen": "", + "train_loss_MA": "None", + "train_loss_MA_alpha": "0.93", + "train_loss_MA_eps": "0.0005", + "transpose_backward": "[0, 1, 2]", + "transpose_forward": "[0, 1, 2]", + "unpack_data": "True", + "use_mask_for_norm": "OrderedDict([(0, False)])", + "use_progress_bar": "False", + "val_eval_criterion_MA": "None", + "val_eval_criterion_alpha": "0.9", + "val_gen": "", + "warmup_epochs": "10", + "was_initialized": "True", + "weight_decay": "3e-05" +} \ No newline at end of file diff --git a/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_4/model_best.model b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_4/model_best.model new file mode 100644 index 0000000000000000000000000000000000000000..28e681831bcf5c95b78439b8005ac2249c6721b1 --- /dev/null +++ b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_4/model_best.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7aa266ff7a185c653bb63793d85a41205e156d57c092be397f71f569136a17bc +size 496519226 diff --git a/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_4/model_best.model.pkl b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_4/model_best.model.pkl new file mode 100644 index 0000000000000000000000000000000000000000..30cfff6325da1fa721dd28270552e28d5ac004e3 --- /dev/null +++ b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_4/model_best.model.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8206610de77e6d58ec32958396e4238a0b7a427ba091b1ac2b3cadfa42a66571 +size 188932 diff --git a/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_4/model_final_checkpoint.model b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_4/model_final_checkpoint.model new file mode 100644 index 0000000000000000000000000000000000000000..57e6790f703cbb7f50a081232124a79d4848fe85 --- /dev/null +++ b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_4/model_final_checkpoint.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed3d8700250477037c72f32e3f6746ed4c4853e59fc1166e0d4a78197beea8fc +size 496529530 diff --git a/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_4/model_final_checkpoint.model.pkl b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_4/model_final_checkpoint.model.pkl new file mode 100644 index 0000000000000000000000000000000000000000..30cfff6325da1fa721dd28270552e28d5ac004e3 --- /dev/null +++ b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_4/model_final_checkpoint.model.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8206610de77e6d58ec32958396e4238a0b7a427ba091b1ac2b3cadfa42a66571 +size 188932 diff --git a/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_4/model_latest.model b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_4/model_latest.model new file mode 100644 index 0000000000000000000000000000000000000000..a554a7c25295929c61ec8a2d4fb4ae2776fbcd90 --- /dev/null +++ b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_4/model_latest.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:96c5f9246437d9eb2ff06ce5c36bfff6397271fb895676c382d37d473de70fa8 +size 496529530 diff --git a/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_4/model_latest.model.pkl b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_4/model_latest.model.pkl new file mode 100644 index 0000000000000000000000000000000000000000..30cfff6325da1fa721dd28270552e28d5ac004e3 --- /dev/null +++ b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_4/model_latest.model.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8206610de77e6d58ec32958396e4238a0b7a427ba091b1ac2b3cadfa42a66571 +size 188932 diff --git a/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_4/progress.png b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_4/progress.png new file mode 100644 index 0000000000000000000000000000000000000000..b12bbc3fbbcee68773a7578541e967a0266bda2d --- /dev/null +++ b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_4/progress.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6a2b55775c509a7c62a72c24a94b11d5794593674590365f9436ead90ab63c78 +size 284072 diff --git a/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_4/training_log_2022_5_19_21_33_38.txt b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_4/training_log_2022_5_19_21_33_38.txt new file mode 100644 index 0000000000000000000000000000000000000000..21bbbad144f0edbcc8ebcc55f414faad22326bb9 --- /dev/null +++ b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_4/training_log_2022_5_19_21_33_38.txt @@ -0,0 +1,769 @@ +Starting... +2022-05-19 21:33:38.214491: Using splits from existing split file: /opt/tiger/project/data/nnUNet_preprocessed/Task008_HepaticVessel/splits_final.pkl +2022-05-19 21:33:38.215025: The split file contains 5 splits. +2022-05-19 21:33:38.215077: Desired fold for training: 4 +2022-05-19 21:33:38.215127: This split has 243 training and 60 validation cases. +2022-05-19 21:35:17.162176: TRAINING KEYS: + odict_keys(['hepaticvessel_001', 'hepaticvessel_002', 'hepaticvessel_004', 'hepaticvessel_005', 'hepaticvessel_007', 'hepaticvessel_010', 'hepaticvessel_011', 'hepaticvessel_013', 'hepaticvessel_016', 'hepaticvessel_018', 'hepaticvessel_019', 'hepaticvessel_022', 'hepaticvessel_023', 'hepaticvessel_025', 'hepaticvessel_026', 'hepaticvessel_027', 'hepaticvessel_028', 'hepaticvessel_029', 'hepaticvessel_030', 'hepaticvessel_031', 'hepaticvessel_039', 'hepaticvessel_040', 'hepaticvessel_042', 'hepaticvessel_044', 'hepaticvessel_046', 'hepaticvessel_050', 'hepaticvessel_051', 'hepaticvessel_052', 'hepaticvessel_053', 'hepaticvessel_058', 'hepaticvessel_061', 'hepaticvessel_062', 'hepaticvessel_065', 'hepaticvessel_066', 'hepaticvessel_068', 'hepaticvessel_071', 'hepaticvessel_072', 'hepaticvessel_077', 'hepaticvessel_078', 'hepaticvessel_079', 'hepaticvessel_080', 'hepaticvessel_081', 'hepaticvessel_082', 'hepaticvessel_083', 'hepaticvessel_084', 'hepaticvessel_085', 'hepaticvessel_086', 'hepaticvessel_087', 'hepaticvessel_090', 'hepaticvessel_091', 'hepaticvessel_093', 'hepaticvessel_096', 'hepaticvessel_097', 'hepaticvessel_098', 'hepaticvessel_100', 'hepaticvessel_102', 'hepaticvessel_103', 'hepaticvessel_104', 'hepaticvessel_110', 'hepaticvessel_112', 'hepaticvessel_116', 'hepaticvessel_117', 'hepaticvessel_125', 'hepaticvessel_127', 'hepaticvessel_128', 'hepaticvessel_129', 'hepaticvessel_131', 'hepaticvessel_132', 'hepaticvessel_133', 'hepaticvessel_137', 'hepaticvessel_138', 'hepaticvessel_139', 'hepaticvessel_140', 'hepaticvessel_141', 'hepaticvessel_142', 'hepaticvessel_144', 'hepaticvessel_145', 'hepaticvessel_146', 'hepaticvessel_147', 'hepaticvessel_148', 'hepaticvessel_150', 'hepaticvessel_151', 'hepaticvessel_157', 'hepaticvessel_160', 'hepaticvessel_161', 'hepaticvessel_163', 'hepaticvessel_164', 'hepaticvessel_165', 'hepaticvessel_166', 'hepaticvessel_167', 'hepaticvessel_171', 'hepaticvessel_172', 'hepaticvessel_175', 'hepaticvessel_177', 'hepaticvessel_178', 'hepaticvessel_179', 'hepaticvessel_183', 'hepaticvessel_184', 'hepaticvessel_185', 'hepaticvessel_186', 'hepaticvessel_192', 'hepaticvessel_193', 'hepaticvessel_194', 'hepaticvessel_196', 'hepaticvessel_197', 'hepaticvessel_198', 'hepaticvessel_199', 'hepaticvessel_201', 'hepaticvessel_203', 'hepaticvessel_206', 'hepaticvessel_209', 'hepaticvessel_210', 'hepaticvessel_213', 'hepaticvessel_214', 'hepaticvessel_215', 'hepaticvessel_217', 'hepaticvessel_218', 'hepaticvessel_219', 'hepaticvessel_221', 'hepaticvessel_222', 'hepaticvessel_224', 'hepaticvessel_225', 'hepaticvessel_227', 'hepaticvessel_229', 'hepaticvessel_230', 'hepaticvessel_231', 'hepaticvessel_233', 'hepaticvessel_234', 'hepaticvessel_235', 'hepaticvessel_236', 'hepaticvessel_238', 'hepaticvessel_240', 'hepaticvessel_242', 'hepaticvessel_244', 'hepaticvessel_245', 'hepaticvessel_246', 'hepaticvessel_248', 'hepaticvessel_255', 'hepaticvessel_256', 'hepaticvessel_258', 'hepaticvessel_259', 'hepaticvessel_263', 'hepaticvessel_265', 'hepaticvessel_266', 'hepaticvessel_268', 'hepaticvessel_269', 'hepaticvessel_270', 'hepaticvessel_272', 'hepaticvessel_274', 'hepaticvessel_275', 'hepaticvessel_279', 'hepaticvessel_280', 'hepaticvessel_281', 'hepaticvessel_282', 'hepaticvessel_284', 'hepaticvessel_285', 'hepaticvessel_286', 'hepaticvessel_287', 'hepaticvessel_289', 'hepaticvessel_290', 'hepaticvessel_291', 'hepaticvessel_294', 'hepaticvessel_296', 'hepaticvessel_297', 'hepaticvessel_299', 'hepaticvessel_300', 'hepaticvessel_303', 'hepaticvessel_307', 'hepaticvessel_309', 'hepaticvessel_314', 'hepaticvessel_318', 'hepaticvessel_319', 'hepaticvessel_320', 'hepaticvessel_321', 'hepaticvessel_322', 'hepaticvessel_323', 'hepaticvessel_324', 'hepaticvessel_325', 'hepaticvessel_327', 'hepaticvessel_329', 'hepaticvessel_330', 'hepaticvessel_332', 'hepaticvessel_333', 'hepaticvessel_336', 'hepaticvessel_338', 'hepaticvessel_340', 'hepaticvessel_341', 'hepaticvessel_345', 'hepaticvessel_348', 'hepaticvessel_349', 'hepaticvessel_353', 'hepaticvessel_356', 'hepaticvessel_358', 'hepaticvessel_359', 'hepaticvessel_361', 'hepaticvessel_362', 'hepaticvessel_363', 'hepaticvessel_367', 'hepaticvessel_368', 'hepaticvessel_369', 'hepaticvessel_371', 'hepaticvessel_374', 'hepaticvessel_375', 'hepaticvessel_382', 'hepaticvessel_383', 'hepaticvessel_384', 'hepaticvessel_385', 'hepaticvessel_388', 'hepaticvessel_389', 'hepaticvessel_391', 'hepaticvessel_398', 'hepaticvessel_400', 'hepaticvessel_401', 'hepaticvessel_406', 'hepaticvessel_407', 'hepaticvessel_408', 'hepaticvessel_409', 'hepaticvessel_411', 'hepaticvessel_412', 'hepaticvessel_416', 'hepaticvessel_419', 'hepaticvessel_420', 'hepaticvessel_422', 'hepaticvessel_423', 'hepaticvessel_424', 'hepaticvessel_427', 'hepaticvessel_429', 'hepaticvessel_431', 'hepaticvessel_433', 'hepaticvessel_434', 'hepaticvessel_437', 'hepaticvessel_438', 'hepaticvessel_440', 'hepaticvessel_441', 'hepaticvessel_442', 'hepaticvessel_443', 'hepaticvessel_444', 'hepaticvessel_445', 'hepaticvessel_446', 'hepaticvessel_447', 'hepaticvessel_453', 'hepaticvessel_456', 'hepaticvessel_458']) +2022-05-19 21:35:17.162389: VALIDATION KEYS: + odict_keys(['hepaticvessel_008', 'hepaticvessel_020', 'hepaticvessel_032', 'hepaticvessel_033', 'hepaticvessel_049', 'hepaticvessel_057', 'hepaticvessel_059', 'hepaticvessel_067', 'hepaticvessel_070', 'hepaticvessel_075', 'hepaticvessel_088', 'hepaticvessel_089', 'hepaticvessel_092', 'hepaticvessel_094', 'hepaticvessel_108', 'hepaticvessel_111', 'hepaticvessel_113', 'hepaticvessel_115', 'hepaticvessel_119', 'hepaticvessel_121', 'hepaticvessel_123', 'hepaticvessel_124', 'hepaticvessel_136', 'hepaticvessel_154', 'hepaticvessel_158', 'hepaticvessel_159', 'hepaticvessel_173', 'hepaticvessel_174', 'hepaticvessel_181', 'hepaticvessel_189', 'hepaticvessel_195', 'hepaticvessel_200', 'hepaticvessel_207', 'hepaticvessel_208', 'hepaticvessel_211', 'hepaticvessel_223', 'hepaticvessel_237', 'hepaticvessel_262', 'hepaticvessel_271', 'hepaticvessel_278', 'hepaticvessel_293', 'hepaticvessel_305', 'hepaticvessel_308', 'hepaticvessel_335', 'hepaticvessel_344', 'hepaticvessel_350', 'hepaticvessel_372', 'hepaticvessel_373', 'hepaticvessel_378', 'hepaticvessel_379', 'hepaticvessel_386', 'hepaticvessel_393', 'hepaticvessel_396', 'hepaticvessel_397', 'hepaticvessel_402', 'hepaticvessel_404', 'hepaticvessel_425', 'hepaticvessel_432', 'hepaticvessel_454', 'hepaticvessel_455']) +2022-05-19 21:35:25.556375: loading checkpoint /opt/tiger/project/data/nnUNet_trained_models/UNet_IN_NANFang/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_4/model_latest.model train= True +2022-05-19 21:35:26.553340: lr: 0.000148 +2022-05-19 21:35:37.490287: +epoch: 67 +2022-05-19 21:43:46.253406: train loss : 0.1096 +2022-05-19 21:44:12.563544: validation loss: 0.1260 +2022-05-19 21:44:12.564116: Average global foreground Dice: [0.6343, 0.8639] +2022-05-19 21:44:12.564205: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-19 21:44:12.977126: lr: 0.000148 +2022-05-19 21:44:12.977350: saving scheduled checkpoint file... +2022-05-19 21:44:13.040467: saving checkpoint... +2022-05-19 21:44:13.845106: done, saving took 0.87 seconds +2022-05-19 21:44:13.853450: done +2022-05-19 21:44:25.647957: This epoch took 528.157012 s + +2022-05-19 21:44:25.648269: +epoch: 68 +2022-05-19 21:52:17.157872: train loss : 0.1094 +2022-05-19 21:52:43.458801: validation loss: 0.1116 +2022-05-19 21:52:43.459407: Average global foreground Dice: [0.6421, 0.8485] +2022-05-19 21:52:43.459496: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-19 21:52:43.873522: lr: 0.000144 +2022-05-19 21:52:43.873729: saving scheduled checkpoint file... +2022-05-19 21:52:43.926191: saving checkpoint... +2022-05-19 21:52:44.797381: done, saving took 0.92 seconds +2022-05-19 21:52:44.806211: done +2022-05-19 21:52:54.506831: This epoch took 508.858466 s + +2022-05-19 21:52:54.507107: +epoch: 69 +2022-05-19 22:00:45.682789: train loss : 0.0999 +2022-05-19 22:01:11.989265: validation loss: 0.1199 +2022-05-19 22:01:11.989834: Average global foreground Dice: [0.6452, 0.8647] +2022-05-19 22:01:11.989922: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-19 22:01:12.394476: lr: 0.00014 +2022-05-19 22:01:12.394692: saving scheduled checkpoint file... +2022-05-19 22:01:12.447033: saving checkpoint... +2022-05-19 22:01:13.409617: done, saving took 1.01 seconds +2022-05-19 22:01:13.418336: done +2022-05-19 22:01:13.449659: saving checkpoint... +2022-05-19 22:01:14.039936: done, saving took 0.62 seconds +2022-05-19 22:01:32.580511: This epoch took 518.073294 s + +2022-05-19 22:01:32.580812: +epoch: 70 +2022-05-19 22:09:23.738479: train loss : 0.1008 +2022-05-19 22:09:50.031369: validation loss: 0.1189 +2022-05-19 22:09:50.031960: Average global foreground Dice: [0.6239, 0.8363] +2022-05-19 22:09:50.032046: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-19 22:09:50.440741: lr: 0.000136 +2022-05-19 22:09:50.440928: saving scheduled checkpoint file... +2022-05-19 22:09:50.493215: saving checkpoint... +2022-05-19 22:09:51.395359: done, saving took 0.95 seconds +2022-05-19 22:09:51.404801: done +2022-05-19 22:10:08.475046: This epoch took 515.894123 s + +2022-05-19 22:10:08.475374: +epoch: 71 +2022-05-19 22:17:59.751940: train loss : 0.1026 +2022-05-19 22:18:26.083147: validation loss: 0.1145 +2022-05-19 22:18:26.083803: Average global foreground Dice: [0.6274, 0.8535] +2022-05-19 22:18:26.083889: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-19 22:18:26.487436: lr: 0.000132 +2022-05-19 22:18:26.487646: saving scheduled checkpoint file... +2022-05-19 22:18:26.540078: saving checkpoint... +2022-05-19 22:18:27.534005: done, saving took 1.05 seconds +2022-05-19 22:18:27.543228: done +2022-05-19 22:18:50.600508: This epoch took 522.125028 s + +2022-05-19 22:18:50.600816: +epoch: 72 +2022-05-19 22:26:42.433744: train loss : 0.0976 +2022-05-19 22:27:08.774569: validation loss: 0.1175 +2022-05-19 22:27:08.775301: Average global foreground Dice: [0.6343, 0.8426] +2022-05-19 22:27:08.775386: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-19 22:27:09.178573: lr: 0.000128 +2022-05-19 22:27:09.178778: saving scheduled checkpoint file... +2022-05-19 22:27:09.231058: saving checkpoint... +2022-05-19 22:27:10.137891: done, saving took 0.96 seconds +2022-05-19 22:27:10.147586: done +2022-05-19 22:28:05.570841: This epoch took 554.969912 s + +2022-05-19 22:28:05.571191: +epoch: 73 +2022-05-19 22:35:57.063693: train loss : 0.1077 +2022-05-19 22:36:23.386150: validation loss: 0.1141 +2022-05-19 22:36:23.386826: Average global foreground Dice: [0.6311, 0.8464] +2022-05-19 22:36:23.386918: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-19 22:36:23.802702: lr: 0.000124 +2022-05-19 22:36:23.802902: saving scheduled checkpoint file... +2022-05-19 22:36:23.854852: saving checkpoint... +2022-05-19 22:36:24.772301: done, saving took 0.97 seconds +2022-05-19 22:36:24.781648: done +2022-05-19 22:36:53.371611: This epoch took 527.800304 s + +2022-05-19 22:36:53.371925: +epoch: 74 +2022-05-19 22:44:45.138723: train loss : 0.1085 +2022-05-19 22:45:11.473521: validation loss: 0.1311 +2022-05-19 22:45:11.474173: Average global foreground Dice: [0.6314, 0.8545] +2022-05-19 22:45:11.474261: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-19 22:45:11.882670: lr: 0.000119 +2022-05-19 22:45:11.882869: saving scheduled checkpoint file... +2022-05-19 22:45:11.935653: saving checkpoint... +2022-05-19 22:45:12.894191: done, saving took 1.01 seconds +2022-05-19 22:45:12.903475: done +2022-05-19 22:45:39.162692: This epoch took 525.790651 s + +2022-05-19 22:45:39.163004: +epoch: 75 +2022-05-19 22:53:30.929458: train loss : 0.1164 +2022-05-19 22:53:57.243286: validation loss: 0.1266 +2022-05-19 22:53:57.243943: Average global foreground Dice: [0.6344, 0.8554] +2022-05-19 22:53:57.244030: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-19 22:53:57.649608: lr: 0.000115 +2022-05-19 22:53:57.649810: saving scheduled checkpoint file... +2022-05-19 22:53:57.701248: saving checkpoint... +2022-05-19 22:53:58.606626: done, saving took 0.96 seconds +2022-05-19 22:53:58.616650: done +2022-05-19 22:54:19.218716: This epoch took 520.055598 s + +2022-05-19 22:54:19.219012: +epoch: 76 +2022-05-19 23:02:10.648831: train loss : 0.1008 +2022-05-19 23:02:36.993830: validation loss: 0.1352 +2022-05-19 23:02:36.994434: Average global foreground Dice: [0.6334, 0.8521] +2022-05-19 23:02:36.994521: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-19 23:02:37.404704: lr: 0.000112 +2022-05-19 23:02:37.404987: saving scheduled checkpoint file... +2022-05-19 23:02:37.457659: saving checkpoint... +2022-05-19 23:02:38.377484: done, saving took 0.97 seconds +2022-05-19 23:02:38.386996: done +2022-05-19 23:05:15.718623: This epoch took 656.499499 s + +2022-05-19 23:05:15.718937: +epoch: 77 +2022-05-19 23:13:07.240921: train loss : 0.1029 +2022-05-19 23:13:33.559342: validation loss: 0.1186 +2022-05-19 23:13:33.560020: Average global foreground Dice: [0.6441, 0.8557] +2022-05-19 23:13:33.560115: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-19 23:13:33.966158: lr: 0.000108 +2022-05-19 23:13:33.966356: saving scheduled checkpoint file... +2022-05-19 23:13:34.018284: saving checkpoint... +2022-05-19 23:13:34.829733: done, saving took 0.86 seconds +2022-05-19 23:13:34.838025: done +2022-05-19 23:13:57.556308: This epoch took 521.837245 s + +2022-05-19 23:13:57.556623: +epoch: 78 +2022-05-19 23:21:48.694082: train loss : 0.1020 +2022-05-19 23:22:15.095253: validation loss: 0.1360 +2022-05-19 23:22:15.095980: Average global foreground Dice: [0.6268, 0.8481] +2022-05-19 23:22:15.096067: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-19 23:22:15.508937: lr: 0.000104 +2022-05-19 23:22:15.509162: saving scheduled checkpoint file... +2022-05-19 23:22:15.562340: saving checkpoint... +2022-05-19 23:22:16.491808: done, saving took 0.98 seconds +2022-05-19 23:22:16.501440: done +2022-05-19 23:23:46.252426: This epoch took 588.695693 s + +2022-05-19 23:23:46.252746: +epoch: 79 +2022-05-19 23:31:37.754753: train loss : 0.1019 +2022-05-19 23:32:04.079164: validation loss: 0.1164 +2022-05-19 23:32:04.079794: Average global foreground Dice: [0.6178, 0.858] +2022-05-19 23:32:04.079901: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-19 23:32:04.494575: lr: 0.0001 +2022-05-19 23:32:04.494790: saving scheduled checkpoint file... +2022-05-19 23:32:04.547216: saving checkpoint... +2022-05-19 23:32:05.491689: done, saving took 1.00 seconds +2022-05-19 23:32:05.501075: done +2022-05-19 23:32:26.758811: This epoch took 520.505951 s + +2022-05-19 23:32:26.759151: +epoch: 80 +2022-05-19 23:40:18.601043: train loss : 0.1096 +2022-05-19 23:40:44.935995: validation loss: 0.1081 +2022-05-19 23:40:44.936596: Average global foreground Dice: [0.6288, 0.8453] +2022-05-19 23:40:44.936678: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-19 23:40:45.349712: lr: 9.6e-05 +2022-05-19 23:40:45.349933: saving scheduled checkpoint file... +2022-05-19 23:40:45.401574: saving checkpoint... +2022-05-19 23:40:46.374263: done, saving took 1.02 seconds +2022-05-19 23:40:46.384009: done +2022-05-19 23:41:04.100768: This epoch took 517.341507 s + +2022-05-19 23:41:04.101078: +epoch: 81 +2022-05-19 23:48:55.173277: train loss : 0.1031 +2022-05-19 23:49:21.487772: validation loss: 0.1328 +2022-05-19 23:49:21.488401: Average global foreground Dice: [0.6338, 0.8411] +2022-05-19 23:49:21.488488: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-19 23:49:21.896226: lr: 9.2e-05 +2022-05-19 23:49:21.896455: saving scheduled checkpoint file... +2022-05-19 23:49:21.948079: saving checkpoint... +2022-05-19 23:49:22.787215: done, saving took 0.89 seconds +2022-05-19 23:49:22.795243: done +2022-05-19 23:49:53.547502: This epoch took 529.446245 s + +2022-05-19 23:49:53.547791: +epoch: 82 +2022-05-19 23:57:45.126702: train loss : 0.1039 +2022-05-19 23:58:11.455153: validation loss: 0.1031 +2022-05-19 23:58:11.455822: Average global foreground Dice: [0.6542, 0.8079] +2022-05-19 23:58:11.455908: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-19 23:58:11.869607: lr: 8.8e-05 +2022-05-19 23:58:11.869792: saving scheduled checkpoint file... +2022-05-19 23:58:11.920667: saving checkpoint... +2022-05-19 23:58:12.876374: done, saving took 1.01 seconds +2022-05-19 23:58:12.885792: done +2022-05-19 23:58:36.540402: This epoch took 522.992498 s + +2022-05-19 23:58:36.540703: +epoch: 83 +2022-05-20 00:06:28.234009: train loss : 0.1017 +2022-05-20 00:06:54.558066: validation loss: 0.1203 +2022-05-20 00:06:54.558773: Average global foreground Dice: [0.6339, 0.8592] +2022-05-20 00:06:54.558858: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-20 00:06:54.966769: lr: 8.5e-05 +2022-05-20 00:06:54.966974: saving scheduled checkpoint file... +2022-05-20 00:06:55.019888: saving checkpoint... +2022-05-20 00:06:55.936837: done, saving took 0.97 seconds +2022-05-20 00:06:55.946531: done +2022-05-20 00:07:15.500197: This epoch took 518.959382 s + +2022-05-20 00:07:15.500503: +epoch: 84 +2022-05-20 00:15:07.352095: train loss : 0.1042 +2022-05-20 00:15:33.680643: validation loss: 0.1308 +2022-05-20 00:15:33.681300: Average global foreground Dice: [0.6427, 0.8537] +2022-05-20 00:15:33.681387: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-20 00:15:34.084127: lr: 8.1e-05 +2022-05-20 00:15:34.084335: saving scheduled checkpoint file... +2022-05-20 00:15:34.136832: saving checkpoint... +2022-05-20 00:15:35.066612: done, saving took 0.98 seconds +2022-05-20 00:15:35.074948: done +2022-05-20 00:17:03.181084: This epoch took 587.680459 s + +2022-05-20 00:17:03.181515: +epoch: 85 +2022-05-20 00:24:54.542538: train loss : 0.1049 +2022-05-20 00:25:20.865211: validation loss: 0.1240 +2022-05-20 00:25:20.865885: Average global foreground Dice: [0.6303, 0.8546] +2022-05-20 00:25:20.866011: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-20 00:25:21.265488: lr: 7.7e-05 +2022-05-20 00:25:21.265695: saving scheduled checkpoint file... +2022-05-20 00:25:21.317370: saving checkpoint... +2022-05-20 00:25:22.143121: done, saving took 0.88 seconds +2022-05-20 00:25:22.151938: done +2022-05-20 00:28:43.128512: This epoch took 699.946877 s + +2022-05-20 00:28:43.128819: +epoch: 86 +2022-05-20 00:36:34.436306: train loss : 0.1015 +2022-05-20 00:37:00.770818: validation loss: 0.1247 +2022-05-20 00:37:00.771546: Average global foreground Dice: [0.613, 0.8285] +2022-05-20 00:37:00.771630: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-20 00:37:01.170964: lr: 7.4e-05 +2022-05-20 00:37:01.171159: saving scheduled checkpoint file... +2022-05-20 00:37:01.223203: saving checkpoint... +2022-05-20 00:37:02.199708: done, saving took 1.03 seconds +2022-05-20 00:37:02.209254: done +2022-05-20 00:37:26.599241: This epoch took 523.470312 s + +2022-05-20 00:37:26.599560: +epoch: 87 +2022-05-20 00:45:17.708436: train loss : 0.0980 +2022-05-20 00:45:44.049387: validation loss: 0.1210 +2022-05-20 00:45:44.049983: Average global foreground Dice: [0.6293, 0.8585] +2022-05-20 00:45:44.050079: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-20 00:45:44.462501: lr: 7e-05 +2022-05-20 00:45:44.462708: saving scheduled checkpoint file... +2022-05-20 00:45:44.515059: saving checkpoint... +2022-05-20 00:45:45.328977: done, saving took 0.87 seconds +2022-05-20 00:45:45.337806: done +2022-05-20 00:46:03.925183: This epoch took 517.325508 s + +2022-05-20 00:46:03.925475: +epoch: 88 +2022-05-20 00:53:55.753352: train loss : 0.1008 +2022-05-20 00:54:22.117807: validation loss: 0.1083 +2022-05-20 00:54:22.118433: Average global foreground Dice: [0.619, 0.8445] +2022-05-20 00:54:22.118520: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-20 00:54:22.522735: lr: 6.7e-05 +2022-05-20 00:54:22.522942: saving scheduled checkpoint file... +2022-05-20 00:54:22.575411: saving checkpoint... +2022-05-20 00:54:23.507722: done, saving took 0.98 seconds +2022-05-20 00:54:23.517888: done +2022-05-20 00:54:41.384289: This epoch took 517.458702 s + +2022-05-20 00:54:41.384619: +epoch: 89 +2022-05-20 01:02:33.513410: train loss : 0.0984 +2022-05-20 01:02:59.893457: validation loss: 0.1084 +2022-05-20 01:02:59.894064: Average global foreground Dice: [0.6251, 0.8562] +2022-05-20 01:02:59.894159: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-20 01:03:00.295628: lr: 6.3e-05 +2022-05-20 01:03:00.295824: saving scheduled checkpoint file... +2022-05-20 01:03:00.347775: saving checkpoint... +2022-05-20 01:03:01.273407: done, saving took 0.98 seconds +2022-05-20 01:03:01.283113: done +2022-05-20 01:03:18.312431: This epoch took 516.927703 s + +2022-05-20 01:03:18.312745: +epoch: 90 +2022-05-20 01:11:10.211580: train loss : 0.0982 +2022-05-20 01:11:36.570696: validation loss: 0.1058 +2022-05-20 01:11:36.571369: Average global foreground Dice: [0.6349, 0.8587] +2022-05-20 01:11:36.571456: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-20 01:11:36.977980: lr: 6e-05 +2022-05-20 01:11:36.978197: saving scheduled checkpoint file... +2022-05-20 01:11:37.031275: saving checkpoint... +2022-05-20 01:11:37.827788: done, saving took 0.85 seconds +2022-05-20 01:11:37.835813: done +2022-05-20 01:13:26.951646: This epoch took 608.638784 s + +2022-05-20 01:13:26.951973: +epoch: 91 +2022-05-20 01:21:19.469252: train loss : 0.0958 +2022-05-20 01:21:45.811795: validation loss: 0.1163 +2022-05-20 01:21:45.812330: Average global foreground Dice: [0.6339, 0.856] +2022-05-20 01:21:45.812414: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-20 01:21:46.213187: lr: 5.7e-05 +2022-05-20 01:21:46.213390: saving scheduled checkpoint file... +2022-05-20 01:21:46.265514: saving checkpoint... +2022-05-20 01:21:47.122302: done, saving took 0.91 seconds +2022-05-20 01:21:47.130528: done +2022-05-20 01:22:12.649498: This epoch took 525.697405 s + +2022-05-20 01:22:12.649810: +epoch: 92 +2022-05-20 01:30:04.032043: train loss : 0.1002 +2022-05-20 01:30:30.354064: validation loss: 0.1185 +2022-05-20 01:30:30.354711: Average global foreground Dice: [0.6387, 0.8538] +2022-05-20 01:30:30.354801: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-20 01:30:30.756380: lr: 5.4e-05 +2022-05-20 01:30:30.756582: saving scheduled checkpoint file... +2022-05-20 01:30:30.808923: saving checkpoint... +2022-05-20 01:30:31.793130: done, saving took 1.04 seconds +2022-05-20 01:30:31.802863: done +2022-05-20 01:31:01.605214: This epoch took 528.955278 s + +2022-05-20 01:31:01.605654: +epoch: 93 +2022-05-20 01:38:53.267040: train loss : 0.0967 +2022-05-20 01:39:19.595411: validation loss: 0.1152 +2022-05-20 01:39:19.596012: Average global foreground Dice: [0.6315, 0.8593] +2022-05-20 01:39:19.596096: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-20 01:39:19.997767: lr: 5.1e-05 +2022-05-20 01:39:19.997963: saving scheduled checkpoint file... +2022-05-20 01:39:20.050242: saving checkpoint... +2022-05-20 01:39:20.886524: done, saving took 0.89 seconds +2022-05-20 01:39:20.894508: done +2022-05-20 01:42:28.062499: This epoch took 686.456728 s + +2022-05-20 01:42:28.062840: +epoch: 94 +2022-05-20 01:50:20.017456: train loss : 0.1003 +2022-05-20 01:50:46.362335: validation loss: 0.1336 +2022-05-20 01:50:46.362880: Average global foreground Dice: [0.6105, 0.8429] +2022-05-20 01:50:46.363023: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-20 01:50:46.767658: lr: 4.8e-05 +2022-05-20 01:50:46.767870: saving scheduled checkpoint file... +2022-05-20 01:50:46.820096: saving checkpoint... +2022-05-20 01:50:47.736527: done, saving took 0.97 seconds +2022-05-20 01:50:47.746044: done +2022-05-20 01:51:08.281750: This epoch took 520.218798 s + +2022-05-20 01:51:08.282052: +epoch: 95 +2022-05-20 01:58:59.877185: train loss : 0.1001 +2022-05-20 01:59:26.236571: validation loss: 0.1172 +2022-05-20 01:59:26.237278: Average global foreground Dice: [0.6491, 0.8619] +2022-05-20 01:59:26.237365: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-20 01:59:26.636260: lr: 4.5e-05 +2022-05-20 01:59:26.636457: saving scheduled checkpoint file... +2022-05-20 01:59:26.687738: saving checkpoint... +2022-05-20 01:59:27.466919: done, saving took 0.83 seconds +2022-05-20 01:59:27.475549: done +2022-05-20 01:59:53.023152: This epoch took 524.740975 s + +2022-05-20 01:59:53.023460: +epoch: 96 +2022-05-20 02:07:44.268248: train loss : 0.0998 +2022-05-20 02:08:10.591882: validation loss: 0.1221 +2022-05-20 02:08:10.592546: Average global foreground Dice: [0.631, 0.8314] +2022-05-20 02:08:10.592634: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-20 02:08:10.994893: lr: 4.2e-05 +2022-05-20 02:08:10.995090: saving scheduled checkpoint file... +2022-05-20 02:08:11.047369: saving checkpoint... +2022-05-20 02:08:11.965285: done, saving took 0.97 seconds +2022-05-20 02:08:11.974049: done +2022-05-20 02:08:43.907636: This epoch took 530.884061 s + +2022-05-20 02:08:43.907938: +epoch: 97 +2022-05-20 02:16:35.550293: train loss : 0.0944 +2022-05-20 02:17:01.880734: validation loss: 0.1243 +2022-05-20 02:17:01.881325: Average global foreground Dice: [0.6381, 0.8535] +2022-05-20 02:17:01.881420: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-20 02:17:02.286358: lr: 3.9e-05 +2022-05-20 02:17:02.286548: saving scheduled checkpoint file... +2022-05-20 02:17:02.338381: saving checkpoint... +2022-05-20 02:17:03.328708: done, saving took 1.04 seconds +2022-05-20 02:17:03.337584: done +2022-05-20 02:17:29.167714: This epoch took 525.259665 s + +2022-05-20 02:17:29.168004: +epoch: 98 +2022-05-20 02:25:21.013715: train loss : 0.0988 +2022-05-20 02:25:47.364299: validation loss: 0.1448 +2022-05-20 02:25:47.365052: Average global foreground Dice: [0.6265, 0.8502] +2022-05-20 02:25:47.365150: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-20 02:25:47.767186: lr: 3.6e-05 +2022-05-20 02:25:47.767388: saving scheduled checkpoint file... +2022-05-20 02:25:47.819592: saving checkpoint... +2022-05-20 02:25:48.755710: done, saving took 0.99 seconds +2022-05-20 02:25:48.765253: done +2022-05-20 02:28:07.864649: This epoch took 638.696529 s + +2022-05-20 02:28:07.864943: +epoch: 99 +2022-05-20 02:35:59.198253: train loss : 0.0939 +2022-05-20 02:36:25.520954: validation loss: 0.1165 +2022-05-20 02:36:25.521577: Average global foreground Dice: [0.6436, 0.8525] +2022-05-20 02:36:25.521663: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-20 02:36:25.927535: lr: 3.4e-05 +2022-05-20 02:36:25.927735: saving scheduled checkpoint file... +2022-05-20 02:36:25.980280: saving checkpoint... +2022-05-20 02:36:26.921046: done, saving took 0.99 seconds +2022-05-20 02:36:26.930815: done +2022-05-20 02:36:54.334535: This epoch took 526.469486 s + +2022-05-20 02:36:54.334835: +epoch: 100 +2022-05-20 02:44:45.652627: train loss : 0.0979 +2022-05-20 02:45:11.994721: validation loss: 0.1243 +2022-05-20 02:45:11.995292: Average global foreground Dice: [0.6365, 0.8596] +2022-05-20 02:45:11.995375: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-20 02:45:12.401132: lr: 3.1e-05 +2022-05-20 02:45:12.401325: saving scheduled checkpoint file... +2022-05-20 02:45:12.453279: saving checkpoint... +2022-05-20 02:45:13.373629: done, saving took 0.97 seconds +2022-05-20 02:45:13.382883: done +2022-05-20 02:45:39.618601: This epoch took 525.283652 s + +2022-05-20 02:45:39.618912: +epoch: 101 +2022-05-20 02:53:31.211427: train loss : 0.0937 +2022-05-20 02:53:57.576347: validation loss: 0.1245 +2022-05-20 02:53:57.576890: Average global foreground Dice: [0.6152, 0.8626] +2022-05-20 02:53:57.576975: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-20 02:53:57.984055: lr: 2.9e-05 +2022-05-20 02:53:57.984264: saving scheduled checkpoint file... +2022-05-20 02:53:58.036330: saving checkpoint... +2022-05-20 02:53:58.962407: done, saving took 0.98 seconds +2022-05-20 02:53:58.972052: done +2022-05-20 02:54:20.832186: This epoch took 521.213156 s + +2022-05-20 02:54:20.832523: +epoch: 102 +2022-05-20 03:02:12.280318: train loss : 0.0934 +2022-05-20 03:02:38.666991: validation loss: 0.1375 +2022-05-20 03:02:38.667647: Average global foreground Dice: [0.6343, 0.8601] +2022-05-20 03:02:38.667732: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-20 03:02:39.070033: lr: 2.6e-05 +2022-05-20 03:02:39.070246: saving scheduled checkpoint file... +2022-05-20 03:02:39.122334: saving checkpoint... +2022-05-20 03:02:40.129189: done, saving took 1.06 seconds +2022-05-20 03:02:40.137928: done +2022-05-20 03:05:48.942263: This epoch took 688.109630 s + +2022-05-20 03:05:48.942561: +epoch: 103 +2022-05-20 03:13:41.032162: train loss : 0.1003 +2022-05-20 03:14:07.360496: validation loss: 0.1131 +2022-05-20 03:14:07.361150: Average global foreground Dice: [0.6326, 0.8471] +2022-05-20 03:14:07.361236: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-20 03:14:07.767490: lr: 2.4e-05 +2022-05-20 03:14:07.767702: saving scheduled checkpoint file... +2022-05-20 03:14:07.820111: saving checkpoint... +2022-05-20 03:14:08.638200: done, saving took 0.87 seconds +2022-05-20 03:14:08.646991: done +2022-05-20 03:14:40.265126: This epoch took 531.322451 s + +2022-05-20 03:14:40.265419: +epoch: 104 +2022-05-20 03:22:32.252051: train loss : 0.1000 +2022-05-20 03:22:58.600253: validation loss: 0.1231 +2022-05-20 03:22:58.600855: Average global foreground Dice: [0.6363, 0.8609] +2022-05-20 03:22:58.600941: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-20 03:22:59.004965: lr: 2.2e-05 +2022-05-20 03:22:59.005167: saving scheduled checkpoint file... +2022-05-20 03:22:59.057498: saving checkpoint... +2022-05-20 03:22:59.996806: done, saving took 0.99 seconds +2022-05-20 03:23:00.006322: done +2022-05-20 03:26:17.375167: This epoch took 697.109617 s + +2022-05-20 03:26:17.375500: +epoch: 105 +2022-05-20 03:34:09.052386: train loss : 0.0949 +2022-05-20 03:34:35.405204: validation loss: 0.1240 +2022-05-20 03:34:35.405837: Average global foreground Dice: [0.6289, 0.8445] +2022-05-20 03:34:35.405925: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-20 03:34:35.808467: lr: 2e-05 +2022-05-20 03:34:35.808662: saving scheduled checkpoint file... +2022-05-20 03:34:35.860185: saving checkpoint... +2022-05-20 03:34:36.782282: done, saving took 0.97 seconds +2022-05-20 03:34:36.791335: done +2022-05-20 03:37:41.530821: This epoch took 684.155207 s + +2022-05-20 03:37:41.531134: +epoch: 106 +2022-05-20 03:45:33.236562: train loss : 0.0954 +2022-05-20 03:45:59.574929: validation loss: 0.1532 +2022-05-20 03:45:59.575588: Average global foreground Dice: [0.6228, 0.8469] +2022-05-20 03:45:59.575673: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-20 03:45:59.995384: lr: 1.8e-05 +2022-05-20 03:45:59.995584: saving scheduled checkpoint file... +2022-05-20 03:46:00.047075: saving checkpoint... +2022-05-20 03:46:00.978893: done, saving took 0.98 seconds +2022-05-20 03:46:00.988956: done +2022-05-20 03:46:23.181058: This epoch took 521.649814 s + +2022-05-20 03:46:23.181443: +epoch: 107 +2022-05-20 03:54:14.554516: train loss : 0.0993 +2022-05-20 03:54:40.881835: validation loss: 0.1141 +2022-05-20 03:54:40.882494: Average global foreground Dice: [0.6254, 0.8507] +2022-05-20 03:54:40.882581: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-20 03:54:41.286762: lr: 1.6e-05 +2022-05-20 03:54:41.286955: saving scheduled checkpoint file... +2022-05-20 03:54:41.338754: saving checkpoint... +2022-05-20 03:54:42.326008: done, saving took 1.04 seconds +2022-05-20 03:54:42.335795: done +2022-05-20 03:56:40.975352: This epoch took 617.793770 s + +2022-05-20 03:56:40.975672: +epoch: 108 +2022-05-20 04:04:32.644532: train loss : 0.1000 +2022-05-20 04:04:58.969759: validation loss: 0.1226 +2022-05-20 04:04:58.970464: Average global foreground Dice: [0.6307, 0.851] +2022-05-20 04:04:58.970550: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-20 04:04:59.377042: lr: 1.4e-05 +2022-05-20 04:04:59.377254: saving scheduled checkpoint file... +2022-05-20 04:04:59.429801: saving checkpoint... +2022-05-20 04:05:00.355347: done, saving took 0.98 seconds +2022-05-20 04:05:00.365554: done +2022-05-20 04:05:22.335599: This epoch took 521.359818 s + +2022-05-20 04:05:22.335909: +epoch: 109 +2022-05-20 04:13:13.812703: train loss : 0.0915 +2022-05-20 04:13:40.113782: validation loss: 0.1293 +2022-05-20 04:13:40.114422: Average global foreground Dice: [0.6202, 0.8523] +2022-05-20 04:13:40.114508: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-20 04:13:40.530390: lr: 1.2e-05 +2022-05-20 04:13:40.530597: saving scheduled checkpoint file... +2022-05-20 04:13:40.582570: saving checkpoint... +2022-05-20 04:13:41.503951: done, saving took 0.97 seconds +2022-05-20 04:13:41.512445: done +2022-05-20 04:14:03.764734: This epoch took 521.428715 s + +2022-05-20 04:14:03.765044: +epoch: 110 +2022-05-20 04:21:55.725989: train loss : 0.0942 +2022-05-20 04:22:22.036795: validation loss: 0.1194 +2022-05-20 04:22:22.037431: Average global foreground Dice: [0.6314, 0.8651] +2022-05-20 04:22:22.037520: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-20 04:22:22.452833: lr: 1.1e-05 +2022-05-20 04:22:22.453032: saving scheduled checkpoint file... +2022-05-20 04:22:22.505664: saving checkpoint... +2022-05-20 04:22:23.434414: done, saving took 0.98 seconds +2022-05-20 04:22:23.444342: done +2022-05-20 04:22:44.076867: This epoch took 520.311683 s + +2022-05-20 04:22:44.077249: +epoch: 111 +2022-05-20 04:30:35.312382: train loss : 0.0924 +2022-05-20 04:31:01.651766: validation loss: 0.1230 +2022-05-20 04:31:01.652409: Average global foreground Dice: [0.6343, 0.8449] +2022-05-20 04:31:01.652494: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-20 04:31:02.057842: lr: 9e-06 +2022-05-20 04:31:02.058036: saving scheduled checkpoint file... +2022-05-20 04:31:02.110332: saving checkpoint... +2022-05-20 04:31:03.050282: done, saving took 0.99 seconds +2022-05-20 04:31:03.060034: done +2022-05-20 04:32:52.310267: This epoch took 608.232898 s + +2022-05-20 04:32:52.310611: +epoch: 112 +2022-05-20 04:40:43.789970: train loss : 0.0894 +2022-05-20 04:41:10.128337: validation loss: 0.1268 +2022-05-20 04:41:10.129044: Average global foreground Dice: [0.6437, 0.8632] +2022-05-20 04:41:10.129141: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-20 04:41:10.530688: lr: 8e-06 +2022-05-20 04:41:10.530893: saving scheduled checkpoint file... +2022-05-20 04:41:10.583108: saving checkpoint... +2022-05-20 04:41:11.519285: done, saving took 0.99 seconds +2022-05-20 04:41:11.529615: done +2022-05-20 04:41:29.566789: This epoch took 517.256067 s + +2022-05-20 04:41:29.567090: +epoch: 113 +2022-05-20 04:49:20.947436: train loss : 0.0966 +2022-05-20 04:49:47.284910: validation loss: 0.1468 +2022-05-20 04:49:47.285512: Average global foreground Dice: [0.618, 0.8502] +2022-05-20 04:49:47.285598: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-20 04:49:47.691832: lr: 7e-06 +2022-05-20 04:49:47.692035: saving scheduled checkpoint file... +2022-05-20 04:49:47.744670: saving checkpoint... +2022-05-20 04:49:48.684906: done, saving took 0.99 seconds +2022-05-20 04:49:48.694838: done +2022-05-20 04:50:09.197347: This epoch took 519.630142 s + +2022-05-20 04:50:09.197633: +epoch: 114 +2022-05-20 04:58:01.186598: train loss : 0.0925 +2022-05-20 04:58:27.537900: validation loss: 0.1089 +2022-05-20 04:58:27.538580: Average global foreground Dice: [0.6332, 0.8375] +2022-05-20 04:58:27.538666: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-20 04:58:27.943815: lr: 6e-06 +2022-05-20 04:58:27.944010: saving scheduled checkpoint file... +2022-05-20 04:58:27.995519: saving checkpoint... +2022-05-20 04:58:28.926910: done, saving took 0.98 seconds +2022-05-20 04:58:28.936626: done +2022-05-20 05:01:42.696367: This epoch took 693.498621 s + +2022-05-20 05:01:42.696679: +epoch: 115 +2022-05-20 05:09:34.346075: train loss : 0.0918 +2022-05-20 05:10:00.666482: validation loss: 0.1287 +2022-05-20 05:10:00.667158: Average global foreground Dice: [0.6204, 0.8482] +2022-05-20 05:10:00.667245: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-20 05:10:01.072694: lr: 5e-06 +2022-05-20 05:10:01.072886: saving scheduled checkpoint file... +2022-05-20 05:10:01.124359: saving checkpoint... +2022-05-20 05:10:02.036735: done, saving took 0.96 seconds +2022-05-20 05:10:02.046803: done +2022-05-20 05:10:26.429659: This epoch took 523.732872 s + +2022-05-20 05:10:26.429962: +epoch: 116 +2022-05-20 05:18:17.464725: train loss : 0.0892 +2022-05-20 05:18:43.790609: validation loss: 0.1289 +2022-05-20 05:18:43.791240: Average global foreground Dice: [0.6553, 0.869] +2022-05-20 05:18:43.791324: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-20 05:18:44.193709: lr: 4e-06 +2022-05-20 05:18:44.193909: saving scheduled checkpoint file... +2022-05-20 05:18:44.245325: saving checkpoint... +2022-05-20 05:18:45.182998: done, saving took 0.99 seconds +2022-05-20 05:18:45.192671: done +2022-05-20 05:19:03.726619: This epoch took 517.296544 s + +2022-05-20 05:19:03.726914: +epoch: 117 +2022-05-20 05:26:55.115848: train loss : 0.1028 +2022-05-20 05:27:21.442067: validation loss: 0.1189 +2022-05-20 05:27:21.442729: Average global foreground Dice: [0.6368, 0.8535] +2022-05-20 05:27:21.442820: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-20 05:27:21.853065: lr: 3e-06 +2022-05-20 05:27:21.853264: saving scheduled checkpoint file... +2022-05-20 05:27:21.905330: saving checkpoint... +2022-05-20 05:27:22.731925: done, saving took 0.88 seconds +2022-05-20 05:27:22.740766: done +2022-05-20 05:27:43.240691: This epoch took 519.513662 s + +2022-05-20 05:27:43.241021: +epoch: 118 +2022-05-20 05:35:34.916461: train loss : 0.0958 +2022-05-20 05:36:01.299965: validation loss: 0.1195 +2022-05-20 05:36:01.300652: Average global foreground Dice: [0.6323, 0.8559] +2022-05-20 05:36:01.300737: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-20 05:36:01.706110: lr: 2e-06 +2022-05-20 05:36:01.706300: saving scheduled checkpoint file... +2022-05-20 05:36:01.757981: saving checkpoint... +2022-05-20 05:36:02.699279: done, saving took 0.99 seconds +2022-05-20 05:36:02.708307: done +2022-05-20 05:36:22.913836: This epoch took 519.672703 s + +2022-05-20 05:36:22.914159: +epoch: 119 +2022-05-20 05:44:14.476660: train loss : 0.0903 +2022-05-20 05:44:40.818745: validation loss: 0.1210 +2022-05-20 05:44:40.819325: Average global foreground Dice: [0.6312, 0.843] +2022-05-20 05:44:40.819409: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-20 05:44:41.226760: lr: 1e-06 +2022-05-20 05:44:41.226958: saving scheduled checkpoint file... +2022-05-20 05:44:41.279076: saving checkpoint... +2022-05-20 05:44:42.207369: done, saving took 0.98 seconds +2022-05-20 05:44:42.216247: done +2022-05-20 05:46:34.731322: This epoch took 611.817053 s + +2022-05-20 05:46:34.731674: +epoch: 120 +2022-05-20 05:54:26.176627: train loss : 0.0946 +2022-05-20 05:54:52.549079: validation loss: 0.1347 +2022-05-20 05:54:52.549742: Average global foreground Dice: [0.6324, 0.8419] +2022-05-20 05:54:52.549827: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-20 05:54:52.957762: lr: 1e-06 +2022-05-20 05:54:52.957964: saving scheduled checkpoint file... +2022-05-20 05:54:53.009961: saving checkpoint... +2022-05-20 05:54:53.928981: done, saving took 0.97 seconds +2022-05-20 05:54:53.938824: done +2022-05-20 05:55:16.334688: This epoch took 521.602904 s + +2022-05-20 05:55:16.335009: +epoch: 121 +2022-05-20 06:03:07.710399: train loss : 0.0942 +2022-05-20 06:03:34.044385: validation loss: 0.1102 +2022-05-20 06:03:34.044974: Average global foreground Dice: [0.6333, 0.846] +2022-05-20 06:03:34.045056: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-20 06:03:34.450281: lr: 1e-06 +2022-05-20 06:03:34.450484: saving scheduled checkpoint file... +2022-05-20 06:03:34.502645: saving checkpoint... +2022-05-20 06:03:35.344863: done, saving took 0.89 seconds +2022-05-20 06:03:35.353413: done +2022-05-20 06:04:27.428057: This epoch took 551.092916 s + +2022-05-20 06:04:27.428370: +epoch: 122 +2022-05-20 06:12:18.864177: train loss : 0.0996 +2022-05-20 06:12:45.195073: validation loss: 0.1357 +2022-05-20 06:12:45.195686: Average global foreground Dice: [0.6336, 0.8554] +2022-05-20 06:12:45.195814: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-20 06:12:45.604357: lr: 0.0 +2022-05-20 06:12:45.604557: saving scheduled checkpoint file... +2022-05-20 06:12:45.657175: saving checkpoint... +2022-05-20 06:12:46.580700: done, saving took 0.98 seconds +2022-05-20 06:12:46.590537: done +2022-05-20 06:15:12.863706: This epoch took 645.435229 s + +2022-05-20 06:15:12.864010: +epoch: 123 +2022-05-20 06:23:04.519852: train loss : 0.0890 +2022-05-20 06:23:30.856823: validation loss: 0.1017 +2022-05-20 06:23:30.857710: Average global foreground Dice: [0.633, 0.8658] +2022-05-20 06:23:30.857796: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-20 06:23:31.264113: lr: 0.0 +2022-05-20 06:23:31.264309: saving scheduled checkpoint file... +2022-05-20 06:23:31.316732: saving checkpoint... +2022-05-20 06:23:32.185140: done, saving took 0.92 seconds +2022-05-20 06:23:32.207525: done +2022-05-20 06:23:49.408279: This epoch took 516.544146 s + +2022-05-20 06:23:49.408664: +epoch: 124 +2022-05-20 06:31:40.682584: train loss : 0.0959 +2022-05-20 06:32:07.023501: validation loss: 0.1012 +2022-05-20 06:32:07.024133: Average global foreground Dice: [0.6263, 0.8546] +2022-05-20 06:32:07.024221: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-20 06:32:07.431305: lr: 0.0 +2022-05-20 06:32:07.431503: saving scheduled checkpoint file... +2022-05-20 06:32:07.483509: saving checkpoint... +2022-05-20 06:32:08.413666: done, saving took 0.98 seconds +2022-05-20 06:32:08.423411: done +2022-05-20 06:34:36.347142: This epoch took 646.938243 s + +2022-05-20 06:34:36.392874: saving checkpoint... +2022-05-20 06:34:37.017723: done, saving took 0.67 seconds diff --git a/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/plans.pkl b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/plans.pkl new file mode 100644 index 0000000000000000000000000000000000000000..711088b30bfadc3f35366c1863cd5768dc3c702f --- /dev/null +++ b/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/plans.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f86e3298caef295e5634424ec1e239a4b7b0f32613863d610242265817922ee8 +size 187584 diff --git a/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_0/debug.json b/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_0/debug.json new file mode 100644 index 0000000000000000000000000000000000000000..9d173daf952413efd93803bfe0e90077dcfc437e --- /dev/null +++ b/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_0/debug.json @@ -0,0 +1,107 @@ +{ + "all_tr_losses": "[]", + "all_val_eval_metrics": "[]", + "all_val_losses": "[]", + "all_val_losses_tr_mode": "[]", + "also_val_in_tr_mode": "False", + "amp_grad_scaler": "None", + "args": "Namespace(batch_size=2, config='./configs/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250.yaml', continue_training=False, crop_size=[128, 128, 128], dbs=False, deep_supervision_scales=[], deterministic=False, disable_decoder=False, disable_ds=True, disable_postprocessing_on_folds=False, disable_saving=False, drop_path=0.0, find_lr=False, find_zero_weight_decay=False, fix_ds_net_numpool=False, fold='0', fp32=False, hdfs_base='GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250', initial_lr=0.0003, is_sigmoid=False, is_spatial_aug_only=False, is_ssl=False, layer_decay=1.0, local_rank=0, loss_name='', lrschedule='warmup_cosine', mask_ratio=0.75, max_loss_cal='v1', max_num_epochs=250, merge_femur=False, min_lr=0, model='Generic_TransUNet_max_ppbp', model_params={'is_masked_attn': True, 'max_dec_layers': 3, 'is_max_bottleneck_transformer': False, 'max_msda': '', 'is_max_ms': True, 'max_ms_idxs': [-4, -3, -2], 'max_hidden_dim': 192, 'mw': 1.0, 'is_max_ds': True, 'is_masking': True, 'is_max_hungarian': True, 'num_queries': 20, 'is_max_cls': True, 'is_mhsa_float32': True}, n_class=17, network='3d_fullres', network_trainer='nnUNetTrainerV2_DDP', npz=False, opt_betas=None, opt_eps=1e-08, optim_name='adamw', p='nnUNetPlansv2.1', plan_update='', pretrained=False, pretrained_config=None, pretrained_weights=None, rank=0, reclip=None, resume='auto', skip_grad_nan=False, task='Task500_BraTS2021', task_pretrained='Task801_WORD', total_batch_size=None, use_compressed_data=False, val_final=False, val_folder='validation_raw', valbest=False, validation_only=False, vallatest=False, warmup_epochs=10, weight_decay=3e-05, world_size=8)", + "base_num_features": "32", + "basic_generator_patch_size": "[182 182 182]", + "batch_dice": "True", + "batch_size": "2", + "best_MA_tr_loss_for_patience": "None", + "best_epoch_based_on_MA_tr_loss": "None", + "best_val_eval_criterion_MA": "None", + "classes": "[1, 2, 3]", + "conv_per_stage": "2", + "data_aug_params": "{'selected_data_channels': None, 'selected_seg_channels': [0], 'do_elastic': True, 'elastic_deform_alpha': (0.0, 900.0), 'elastic_deform_sigma': (9.0, 13.0), 'p_eldef': 0.3, 'do_scaling': True, 'scale_range': (0.65, 1.6), 'independent_scale_factor_for_each_axis': True, 'p_independent_scale_per_axis': 0.3, 'p_scale': 0.3, 'do_rotation': True, 'rotation_x': (-1.5707963267948966, 1.5707963267948966), 'rotation_y': (-1.5707963267948966, 1.5707963267948966), 'rotation_z': (-1.5707963267948966, 1.5707963267948966), 'rotation_p_per_axis': 1, 'p_rot': 0.3, 'random_crop': False, 'random_crop_dist_to_border': None, 'do_gamma': True, 'gamma_retain_stats': True, 'gamma_range': (0.5, 1.6), 'p_gamma': 0.3, 'do_mirror': True, 'mirror_axes': (0, 1, 2), 'dummy_2D': False, 'mask_was_used_for_normalization': OrderedDict([(0, True), (1, True), (2, True), (3, True)]), 'border_mode_data': 'constant', 'all_segmentation_labels': None, 'move_last_seg_chanel_to_data': False, 'cascade_do_cascade_augmentations': False, 'cascade_random_binary_transform_p': 0.4, 'cascade_random_binary_transform_p_per_label': 1, 'cascade_random_binary_transform_size': (1, 8), 'cascade_remove_conn_comp_p': 0.2, 'cascade_remove_conn_comp_max_size_percent_threshold': 0.15, 'cascade_remove_conn_comp_fill_with_other_class_p': 0.0, 'do_additive_brightness': True, 'additive_brightness_p_per_sample': 0.3, 'additive_brightness_p_per_channel': 0.5, 'additive_brightness_mu': 0, 'additive_brightness_sigma': 0.2, 'num_threads': 12, 'num_cached_per_thread': 4, 'patch_size_for_spatialtransform': array([128, 128, 128]), 'eldef_deformation_scale': (0, 0.25)}", + "dataset_directory": "/opt/tiger/project/data/nnUNet_preprocessed/Task500_BraTS2021", + "deep_supervision_scales": "[[1, 1, 1], [0.5, 0.5, 0.5], [0.25, 0.25, 0.25], [0.125, 0.125, 0.125], [0.0625, 0.0625, 0.0625]]", + "deterministic": "False", + "disable_ds": "True", + "distribute_batch_size": "False", + "dl_tr": "", + "dl_val": "", + "do_dummy_2D_aug": "False", + "ds_loss_weights": "[1. 0. 0. 0. 0.]", + "epoch": "0", + "experiment_name": "nnUNetTrainerV2_DDP", + "find_zero_weight_decay": "False", + "fold": "0", + "folder_with_preprocessed_data": "/opt/tiger/project/data/nnUNet_preprocessed/Task500_BraTS2021/nnUNetData_plans_v2.1_stage0", + "fp16": "True", + "global_batch_size": "16", + "gt_niftis_folder": "/opt/tiger/project/data/nnUNet_preprocessed/Task500_BraTS2021/gt_segmentations", + "inference_pad_border_mode": "constant", + "inference_pad_kwargs": "{'constant_values': 0}", + "init_args": "('/opt/tiger/project/data/nnUNet_preprocessed/Task500_BraTS2021/nnUNetPlansv2.1_plans_3D.pkl', 0, 0, '/opt/tiger/project/data/nnUNet_trained_models/UNet_IN_NANFang/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250', '/opt/tiger/project/data/nnUNet_preprocessed/Task500_BraTS2021', True, 0, True, False, False, True)", + "initial_lr": "0.0003", + "input_size": "[128, 128, 128]", + "is_spatial_aug_only": "False", + "is_ssl": "False", + "layer_decay": "1.0", + "local_rank": "0", + "log_file": "/opt/tiger/project/data/nnUNet_trained_models/UNet_IN_NANFang/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_0/training_log_2022_5_14_04_20_21.txt", + "loss": "None", + "lr_scheduler": "", + "lr_scheduler_eps": "0.001", + "lr_scheduler_name": "warmup_cosine", + "lr_scheduler_patience": "30", + "lr_threshold": "1e-06", + "max_num_epochs": "250", + "min_lr": "0", + "min_region_size_per_class": "None", + "min_size_per_class": "None", + "model": "Generic_TransUNet_max_ppbp", + "model_params": "{'is_masked_attn': True, 'max_dec_layers': 3, 'is_max_bottleneck_transformer': False, 'max_msda': '', 'is_max_ms': True, 'max_ms_idxs': [-4, -3, -2], 'max_hidden_dim': 192, 'mw': 1.0, 'is_max_ds': True, 'is_masking': True, 'is_max_hungarian': True, 'num_queries': 20, 'is_max_cls': True, 'is_mhsa_float32': True}", + "net_conv_kernel_sizes": "[[3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3]]", + "net_num_pool_op_kernel_sizes": "[[2, 2, 2], [2, 2, 2], [2, 2, 2], [2, 2, 2], [2, 2, 2]]", + "net_pool_per_axis": "[5, 5, 5]", + "normalization_schemes": "OrderedDict([(0, 'nonCT'), (1, 'nonCT'), (2, 'nonCT'), (3, 'nonCT')])", + "num_batches_per_epoch": "250", + "num_classes": "3", + "num_input_channels": "4", + "num_val_batches_per_epoch": "50", + "online_eval_fn": "[]", + "online_eval_foreground_dc": "[]", + "online_eval_fp": "[]", + "online_eval_tp": "[]", + "only_keep_largest_connected_component": "None", + "optim_name": "adamw", + "optimizer": "AdamW (\nParameter Group 0\n amsgrad: False\n betas: (0.9, 0.999)\n eps: 1e-08\n initial_lr: 0.0003\n lr: 0.0\n weight_decay: 3e-05\n)", + "output_folder": "/opt/tiger/project/data/nnUNet_trained_models/UNet_IN_NANFang/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_0", + "output_folder_base": "/opt/tiger/project/data/nnUNet_trained_models/UNet_IN_NANFang/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250", + "oversample_foreground_percent": "0.0", + "pad_all_sides": "None", + "patch_size": "[128 128 128]", + "patience": "50", + "pin_memory": "True", + "plans_file": "/opt/tiger/project/data/nnUNet_preprocessed/Task500_BraTS2021/nnUNetPlansv2.1_plans_3D.pkl", + "reclip": "None", + "regions": "{'whole tumor': (1, 2, 3), 'tumor core': (2, 3), 'enhancing tumor': (3,)}", + "regions_class_order": "(1, 2, 3)", + "resume": "auto", + "save_best_checkpoint": "True", + "save_every": "1", + "save_final_checkpoint": "True", + "save_intermediate_checkpoints": "True", + "save_latest_only": "True", + "stage": "0", + "threeD": "True", + "tr_gen": "", + "train_loss_MA": "None", + "train_loss_MA_alpha": "0.93", + "train_loss_MA_eps": "0.0005", + "transpose_backward": "[0, 1, 2]", + "transpose_forward": "[0, 1, 2]", + "unpack_data": "True", + "use_mask_for_norm": "OrderedDict([(0, True), (1, True), (2, True), (3, True)])", + "use_progress_bar": "False", + "val_eval_criterion_MA": "None", + "val_eval_criterion_alpha": "0.9", + "val_gen": "", + "warmup_epochs": "10", + "was_initialized": "True", + "weight_decay": "3e-05" +} \ No newline at end of file diff --git a/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_0/model_best.model b/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_0/model_best.model new file mode 100644 index 0000000000000000000000000000000000000000..80c5bd372183143c8d32c392b2fbaf7dbd499cad --- /dev/null +++ b/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_0/model_best.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:21f55c806780ce635c27d50b81a54164ad4e8892cafcdf660926f8ef2c5ce538 +size 409150945 diff --git a/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_0/model_best.model.pkl b/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_0/model_best.model.pkl new file mode 100644 index 0000000000000000000000000000000000000000..14260986a2fe6e2784abc7f5b67e3173b2a1eaca --- /dev/null +++ b/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_0/model_best.model.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ecc67a3df1e496e8155d1d5cef774c62efcd0b6a34c6e6dadb6c7091fd12f1a2 +size 393843 diff --git a/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_0/model_final_checkpoint.model b/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_0/model_final_checkpoint.model new file mode 100644 index 0000000000000000000000000000000000000000..5264b1011086a53683d293b6bc34b19098f7e02e --- /dev/null +++ b/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_0/model_final_checkpoint.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:772a410bef3df6e1b9caab6240938966555bc690692dcce8380742e86142cc65 +size 409163809 diff --git a/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_0/model_final_checkpoint.model.pkl b/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_0/model_final_checkpoint.model.pkl new file mode 100644 index 0000000000000000000000000000000000000000..14260986a2fe6e2784abc7f5b67e3173b2a1eaca --- /dev/null +++ b/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_0/model_final_checkpoint.model.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ecc67a3df1e496e8155d1d5cef774c62efcd0b6a34c6e6dadb6c7091fd12f1a2 +size 393843 diff --git a/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_0/model_latest.model b/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_0/model_latest.model new file mode 100644 index 0000000000000000000000000000000000000000..7cfb12f39917835d1ac5ff1b058786bb65527725 --- /dev/null +++ b/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_0/model_latest.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a4503ef029548807c3001a9de15b231efe9bca08ec00988fae9e9bb498473049 +size 409163809 diff --git a/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_0/model_latest.model.pkl b/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_0/model_latest.model.pkl new file mode 100644 index 0000000000000000000000000000000000000000..14260986a2fe6e2784abc7f5b67e3173b2a1eaca --- /dev/null +++ b/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_0/model_latest.model.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ecc67a3df1e496e8155d1d5cef774c62efcd0b6a34c6e6dadb6c7091fd12f1a2 +size 393843 diff --git a/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_0/progress.png b/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_0/progress.png new file mode 100644 index 0000000000000000000000000000000000000000..e3026435e8f7670fece27e5aa33411ba721080a8 --- /dev/null +++ b/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_0/progress.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c540bacba8c272e40fc5bf082b22feb25889b38a5ef3de6ae68e61890d45de50 +size 328385 diff --git a/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_0/training_log_2022_5_14_04_20_21.txt b/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_0/training_log_2022_5_14_04_20_21.txt new file mode 100644 index 0000000000000000000000000000000000000000..c064be197de971e55c632ee1e5b87acd68757af3 --- /dev/null +++ b/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_0/training_log_2022_5_14_04_20_21.txt @@ -0,0 +1,3446 @@ +Starting... +2022-05-14 04:20:21.547877: Using splits from existing split file: /opt/tiger/project/data/nnUNet_preprocessed/Task500_BraTS2021/splits_final.pkl +2022-05-14 04:20:21.548275: The split file contains 5 splits. +2022-05-14 04:20:21.548328: Desired fold for training: 0 +2022-05-14 04:20:21.548368: This split has 1000 training and 251 validation cases. +2022-05-14 04:21:48.063291: TRAINING KEYS: + odict_keys(['BraTS2021_00002', 'BraTS2021_00003', 'BraTS2021_00005', 'BraTS2021_00006', 'BraTS2021_00008', 'BraTS2021_00011', 'BraTS2021_00012', 'BraTS2021_00014', 'BraTS2021_00017', 'BraTS2021_00018', 'BraTS2021_00019', 'BraTS2021_00020', 'BraTS2021_00021', 'BraTS2021_00022', 'BraTS2021_00025', 'BraTS2021_00026', 'BraTS2021_00030', 'BraTS2021_00032', 'BraTS2021_00033', 'BraTS2021_00036', 'BraTS2021_00043', 'BraTS2021_00044', 'BraTS2021_00048', 'BraTS2021_00049', 'BraTS2021_00052', 'BraTS2021_00053', 'BraTS2021_00054', 'BraTS2021_00056', 'BraTS2021_00058', 'BraTS2021_00059', 'BraTS2021_00060', 'BraTS2021_00061', 'BraTS2021_00062', 'BraTS2021_00063', 'BraTS2021_00064', 'BraTS2021_00066', 'BraTS2021_00068', 'BraTS2021_00071', 'BraTS2021_00072', 'BraTS2021_00074', 'BraTS2021_00077', 'BraTS2021_00081', 'BraTS2021_00084', 'BraTS2021_00090', 'BraTS2021_00094', 'BraTS2021_00095', 'BraTS2021_00096', 'BraTS2021_00097', 'BraTS2021_00098', 'BraTS2021_00100', 'BraTS2021_00101', 'BraTS2021_00103', 'BraTS2021_00105', 'BraTS2021_00108', 'BraTS2021_00109', 'BraTS2021_00111', 'BraTS2021_00112', 'BraTS2021_00113', 'BraTS2021_00115', 'BraTS2021_00116', 'BraTS2021_00118', 'BraTS2021_00120', 'BraTS2021_00121', 'BraTS2021_00122', 'BraTS2021_00124', 'BraTS2021_00128', 'BraTS2021_00131', 'BraTS2021_00132', 'BraTS2021_00133', 'BraTS2021_00134', 'BraTS2021_00138', 'BraTS2021_00139', 'BraTS2021_00142', 'BraTS2021_00143', 'BraTS2021_00146', 'BraTS2021_00147', 'BraTS2021_00148', 'BraTS2021_00149', 'BraTS2021_00150', 'BraTS2021_00151', 'BraTS2021_00154', 'BraTS2021_00155', 'BraTS2021_00156', 'BraTS2021_00157', 'BraTS2021_00158', 'BraTS2021_00159', 'BraTS2021_00162', 'BraTS2021_00165', 'BraTS2021_00166', 'BraTS2021_00167', 'BraTS2021_00170', 'BraTS2021_00171', 'BraTS2021_00172', 'BraTS2021_00176', 'BraTS2021_00177', 'BraTS2021_00178', 'BraTS2021_00183', 'BraTS2021_00184', 'BraTS2021_00186', 'BraTS2021_00187', 'BraTS2021_00188', 'BraTS2021_00191', 'BraTS2021_00193', 'BraTS2021_00195', 'BraTS2021_00196', 'BraTS2021_00199', 'BraTS2021_00201', 'BraTS2021_00203', 'BraTS2021_00204', 'BraTS2021_00206', 'BraTS2021_00207', 'BraTS2021_00209', 'BraTS2021_00210', 'BraTS2021_00216', 'BraTS2021_00219', 'BraTS2021_00220', 'BraTS2021_00221', 'BraTS2021_00222', 'BraTS2021_00227', 'BraTS2021_00228', 'BraTS2021_00230', 'BraTS2021_00233', 'BraTS2021_00234', 'BraTS2021_00235', 'BraTS2021_00236', 'BraTS2021_00237', 'BraTS2021_00238', 'BraTS2021_00239', 'BraTS2021_00240', 'BraTS2021_00241', 'BraTS2021_00242', 'BraTS2021_00246', 'BraTS2021_00247', 'BraTS2021_00249', 'BraTS2021_00250', 'BraTS2021_00253', 'BraTS2021_00254', 'BraTS2021_00258', 'BraTS2021_00259', 'BraTS2021_00260', 'BraTS2021_00261', 'BraTS2021_00262', 'BraTS2021_00263', 'BraTS2021_00267', 'BraTS2021_00269', 'BraTS2021_00271', 'BraTS2021_00273', 'BraTS2021_00275', 'BraTS2021_00280', 'BraTS2021_00281', 'BraTS2021_00282', 'BraTS2021_00284', 'BraTS2021_00285', 'BraTS2021_00288', 'BraTS2021_00289', 'BraTS2021_00290', 'BraTS2021_00291', 'BraTS2021_00293', 'BraTS2021_00296', 'BraTS2021_00297', 'BraTS2021_00298', 'BraTS2021_00299', 'BraTS2021_00300', 'BraTS2021_00301', 'BraTS2021_00303', 'BraTS2021_00305', 'BraTS2021_00306', 'BraTS2021_00309', 'BraTS2021_00310', 'BraTS2021_00312', 'BraTS2021_00314', 'BraTS2021_00316', 'BraTS2021_00317', 'BraTS2021_00318', 'BraTS2021_00320', 'BraTS2021_00321', 'BraTS2021_00322', 'BraTS2021_00324', 'BraTS2021_00325', 'BraTS2021_00327', 'BraTS2021_00328', 'BraTS2021_00329', 'BraTS2021_00331', 'BraTS2021_00332', 'BraTS2021_00338', 'BraTS2021_00339', 'BraTS2021_00340', 'BraTS2021_00341', 'BraTS2021_00343', 'BraTS2021_00344', 'BraTS2021_00346', 'BraTS2021_00347', 'BraTS2021_00348', 'BraTS2021_00349', 'BraTS2021_00351', 'BraTS2021_00352', 'BraTS2021_00356', 'BraTS2021_00359', 'BraTS2021_00360', 'BraTS2021_00364', 'BraTS2021_00367', 'BraTS2021_00369', 'BraTS2021_00370', 'BraTS2021_00371', 'BraTS2021_00373', 'BraTS2021_00377', 'BraTS2021_00378', 'BraTS2021_00380', 'BraTS2021_00382', 'BraTS2021_00383', 'BraTS2021_00386', 'BraTS2021_00387', 'BraTS2021_00388', 'BraTS2021_00389', 'BraTS2021_00390', 'BraTS2021_00392', 'BraTS2021_00395', 'BraTS2021_00397', 'BraTS2021_00400', 'BraTS2021_00401', 'BraTS2021_00403', 'BraTS2021_00404', 'BraTS2021_00405', 'BraTS2021_00406', 'BraTS2021_00407', 'BraTS2021_00409', 'BraTS2021_00410', 'BraTS2021_00412', 'BraTS2021_00413', 'BraTS2021_00414', 'BraTS2021_00416', 'BraTS2021_00417', 'BraTS2021_00418', 'BraTS2021_00421', 'BraTS2021_00425', 'BraTS2021_00429', 'BraTS2021_00431', 'BraTS2021_00432', 'BraTS2021_00440', 'BraTS2021_00441', 'BraTS2021_00443', 'BraTS2021_00444', 'BraTS2021_00445', 'BraTS2021_00446', 'BraTS2021_00448', 'BraTS2021_00449', 'BraTS2021_00451', 'BraTS2021_00452', 'BraTS2021_00453', 'BraTS2021_00454', 'BraTS2021_00455', 'BraTS2021_00456', 'BraTS2021_00457', 'BraTS2021_00459', 'BraTS2021_00466', 'BraTS2021_00468', 'BraTS2021_00469', 'BraTS2021_00470', 'BraTS2021_00472', 'BraTS2021_00477', 'BraTS2021_00478', 'BraTS2021_00479', 'BraTS2021_00481', 'BraTS2021_00483', 'BraTS2021_00488', 'BraTS2021_00491', 'BraTS2021_00493', 'BraTS2021_00494', 'BraTS2021_00496', 'BraTS2021_00498', 'BraTS2021_00499', 'BraTS2021_00501', 'BraTS2021_00502', 'BraTS2021_00504', 'BraTS2021_00505', 'BraTS2021_00506', 'BraTS2021_00507', 'BraTS2021_00510', 'BraTS2021_00511', 'BraTS2021_00512', 'BraTS2021_00513', 'BraTS2021_00517', 'BraTS2021_00518', 'BraTS2021_00519', 'BraTS2021_00520', 'BraTS2021_00523', 'BraTS2021_00524', 'BraTS2021_00525', 'BraTS2021_00526', 'BraTS2021_00528', 'BraTS2021_00529', 'BraTS2021_00530', 'BraTS2021_00532', 'BraTS2021_00533', 'BraTS2021_00537', 'BraTS2021_00538', 'BraTS2021_00539', 'BraTS2021_00540', 'BraTS2021_00542', 'BraTS2021_00543', 'BraTS2021_00544', 'BraTS2021_00545', 'BraTS2021_00547', 'BraTS2021_00548', 'BraTS2021_00550', 'BraTS2021_00551', 'BraTS2021_00552', 'BraTS2021_00554', 'BraTS2021_00555', 'BraTS2021_00556', 'BraTS2021_00557', 'BraTS2021_00559', 'BraTS2021_00561', 'BraTS2021_00563', 'BraTS2021_00565', 'BraTS2021_00567', 'BraTS2021_00568', 'BraTS2021_00569', 'BraTS2021_00570', 'BraTS2021_00571', 'BraTS2021_00572', 'BraTS2021_00574', 'BraTS2021_00575', 'BraTS2021_00576', 'BraTS2021_00577', 'BraTS2021_00578', 'BraTS2021_00579', 'BraTS2021_00580', 'BraTS2021_00581', 'BraTS2021_00582', 'BraTS2021_00584', 'BraTS2021_00588', 'BraTS2021_00589', 'BraTS2021_00590', 'BraTS2021_00591', 'BraTS2021_00593', 'BraTS2021_00594', 'BraTS2021_00596', 'BraTS2021_00598', 'BraTS2021_00599', 'BraTS2021_00601', 'BraTS2021_00604', 'BraTS2021_00606', 'BraTS2021_00608', 'BraTS2021_00610', 'BraTS2021_00611', 'BraTS2021_00612', 'BraTS2021_00613', 'BraTS2021_00615', 'BraTS2021_00616', 'BraTS2021_00618', 'BraTS2021_00619', 'BraTS2021_00620', 'BraTS2021_00621', 'BraTS2021_00622', 'BraTS2021_00623', 'BraTS2021_00624', 'BraTS2021_00625', 'BraTS2021_00626', 'BraTS2021_00628', 'BraTS2021_00631', 'BraTS2021_00636', 'BraTS2021_00638', 'BraTS2021_00640', 'BraTS2021_00642', 'BraTS2021_00645', 'BraTS2021_00646', 'BraTS2021_00649', 'BraTS2021_00650', 'BraTS2021_00651', 'BraTS2021_00652', 'BraTS2021_00655', 'BraTS2021_00656', 'BraTS2021_00657', 'BraTS2021_00658', 'BraTS2021_00659', 'BraTS2021_00661', 'BraTS2021_00663', 'BraTS2021_00667', 'BraTS2021_00668', 'BraTS2021_00674', 'BraTS2021_00675', 'BraTS2021_00676', 'BraTS2021_00677', 'BraTS2021_00679', 'BraTS2021_00680', 'BraTS2021_00682', 'BraTS2021_00683', 'BraTS2021_00684', 'BraTS2021_00685', 'BraTS2021_00686', 'BraTS2021_00687', 'BraTS2021_00688', 'BraTS2021_00689', 'BraTS2021_00690', 'BraTS2021_00691', 'BraTS2021_00692', 'BraTS2021_00694', 'BraTS2021_00697', 'BraTS2021_00698', 'BraTS2021_00703', 'BraTS2021_00704', 'BraTS2021_00705', 'BraTS2021_00706', 'BraTS2021_00707', 'BraTS2021_00708', 'BraTS2021_00714', 'BraTS2021_00718', 'BraTS2021_00723', 'BraTS2021_00724', 'BraTS2021_00725', 'BraTS2021_00728', 'BraTS2021_00729', 'BraTS2021_00730', 'BraTS2021_00731', 'BraTS2021_00732', 'BraTS2021_00734', 'BraTS2021_00736', 'BraTS2021_00737', 'BraTS2021_00739', 'BraTS2021_00740', 'BraTS2021_00742', 'BraTS2021_00744', 'BraTS2021_00746', 'BraTS2021_00747', 'BraTS2021_00750', 'BraTS2021_00751', 'BraTS2021_00756', 'BraTS2021_00758', 'BraTS2021_00760', 'BraTS2021_00764', 'BraTS2021_00765', 'BraTS2021_00767', 'BraTS2021_00768', 'BraTS2021_00773', 'BraTS2021_00774', 'BraTS2021_00778', 'BraTS2021_00780', 'BraTS2021_00781', 'BraTS2021_00784', 'BraTS2021_00787', 'BraTS2021_00788', 'BraTS2021_00789', 'BraTS2021_00792', 'BraTS2021_00795', 'BraTS2021_00796', 'BraTS2021_00797', 'BraTS2021_00799', 'BraTS2021_00800', 'BraTS2021_00801', 'BraTS2021_00802', 'BraTS2021_00803', 'BraTS2021_00804', 'BraTS2021_00805', 'BraTS2021_00806', 'BraTS2021_00807', 'BraTS2021_00808', 'BraTS2021_00809', 'BraTS2021_00811', 'BraTS2021_00814', 'BraTS2021_00816', 'BraTS2021_00819', 'BraTS2021_00820', 'BraTS2021_00823', 'BraTS2021_00824', 'BraTS2021_00828', 'BraTS2021_00830', 'BraTS2021_00831', 'BraTS2021_00837', 'BraTS2021_00838', 'BraTS2021_00839', 'BraTS2021_00840', 'BraTS2021_00999', 'BraTS2021_01000', 'BraTS2021_01002', 'BraTS2021_01003', 'BraTS2021_01007', 'BraTS2021_01008', 'BraTS2021_01009', 'BraTS2021_01010', 'BraTS2021_01011', 'BraTS2021_01012', 'BraTS2021_01013', 'BraTS2021_01014', 'BraTS2021_01015', 'BraTS2021_01016', 'BraTS2021_01017', 'BraTS2021_01019', 'BraTS2021_01020', 'BraTS2021_01021', 'BraTS2021_01022', 'BraTS2021_01023', 'BraTS2021_01024', 'BraTS2021_01025', 'BraTS2021_01026', 'BraTS2021_01027', 'BraTS2021_01028', 'BraTS2021_01029', 'BraTS2021_01030', 'BraTS2021_01031', 'BraTS2021_01032', 'BraTS2021_01033', 'BraTS2021_01034', 'BraTS2021_01035', 'BraTS2021_01036', 'BraTS2021_01038', 'BraTS2021_01039', 'BraTS2021_01040', 'BraTS2021_01041', 'BraTS2021_01042', 'BraTS2021_01044', 'BraTS2021_01045', 'BraTS2021_01046', 'BraTS2021_01047', 'BraTS2021_01048', 'BraTS2021_01049', 'BraTS2021_01052', 'BraTS2021_01053', 'BraTS2021_01054', 'BraTS2021_01055', 'BraTS2021_01056', 'BraTS2021_01057', 'BraTS2021_01059', 'BraTS2021_01060', 'BraTS2021_01061', 'BraTS2021_01062', 'BraTS2021_01064', 'BraTS2021_01065', 'BraTS2021_01067', 'BraTS2021_01068', 'BraTS2021_01069', 'BraTS2021_01070', 'BraTS2021_01071', 'BraTS2021_01073', 'BraTS2021_01074', 'BraTS2021_01075', 'BraTS2021_01076', 'BraTS2021_01077', 'BraTS2021_01078', 'BraTS2021_01080', 'BraTS2021_01082', 'BraTS2021_01083', 'BraTS2021_01084', 'BraTS2021_01085', 'BraTS2021_01086', 'BraTS2021_01087', 'BraTS2021_01090', 'BraTS2021_01091', 'BraTS2021_01092', 'BraTS2021_01093', 'BraTS2021_01094', 'BraTS2021_01095', 'BraTS2021_01097', 'BraTS2021_01098', 'BraTS2021_01099', 'BraTS2021_01100', 'BraTS2021_01102', 'BraTS2021_01103', 'BraTS2021_01104', 'BraTS2021_01105', 'BraTS2021_01106', 'BraTS2021_01107', 'BraTS2021_01108', 'BraTS2021_01109', 'BraTS2021_01111', 'BraTS2021_01112', 'BraTS2021_01113', 'BraTS2021_01114', 'BraTS2021_01115', 'BraTS2021_01116', 'BraTS2021_01117', 'BraTS2021_01119', 'BraTS2021_01120', 'BraTS2021_01121', 'BraTS2021_01122', 'BraTS2021_01125', 'BraTS2021_01126', 'BraTS2021_01127', 'BraTS2021_01128', 'BraTS2021_01129', 'BraTS2021_01130', 'BraTS2021_01131', 'BraTS2021_01132', 'BraTS2021_01133', 'BraTS2021_01134', 'BraTS2021_01135', 'BraTS2021_01136', 'BraTS2021_01138', 'BraTS2021_01139', 'BraTS2021_01140', 'BraTS2021_01143', 'BraTS2021_01144', 'BraTS2021_01145', 'BraTS2021_01148', 'BraTS2021_01149', 'BraTS2021_01150', 'BraTS2021_01151', 'BraTS2021_01153', 'BraTS2021_01154', 'BraTS2021_01155', 'BraTS2021_01156', 'BraTS2021_01157', 'BraTS2021_01158', 'BraTS2021_01159', 'BraTS2021_01160', 'BraTS2021_01161', 'BraTS2021_01163', 'BraTS2021_01164', 'BraTS2021_01165', 'BraTS2021_01166', 'BraTS2021_01167', 'BraTS2021_01168', 'BraTS2021_01171', 'BraTS2021_01172', 'BraTS2021_01174', 'BraTS2021_01175', 'BraTS2021_01176', 'BraTS2021_01177', 'BraTS2021_01178', 'BraTS2021_01179', 'BraTS2021_01180', 'BraTS2021_01182', 'BraTS2021_01184', 'BraTS2021_01185', 'BraTS2021_01186', 'BraTS2021_01187', 'BraTS2021_01188', 'BraTS2021_01189', 'BraTS2021_01190', 'BraTS2021_01192', 'BraTS2021_01193', 'BraTS2021_01194', 'BraTS2021_01195', 'BraTS2021_01196', 'BraTS2021_01197', 'BraTS2021_01199', 'BraTS2021_01200', 'BraTS2021_01201', 'BraTS2021_01202', 'BraTS2021_01203', 'BraTS2021_01204', 'BraTS2021_01205', 'BraTS2021_01207', 'BraTS2021_01213', 'BraTS2021_01214', 'BraTS2021_01215', 'BraTS2021_01216', 'BraTS2021_01217', 'BraTS2021_01218', 'BraTS2021_01219', 'BraTS2021_01220', 'BraTS2021_01221', 'BraTS2021_01222', 'BraTS2021_01223', 'BraTS2021_01224', 'BraTS2021_01226', 'BraTS2021_01227', 'BraTS2021_01228', 'BraTS2021_01229', 'BraTS2021_01230', 'BraTS2021_01232', 'BraTS2021_01233', 'BraTS2021_01234', 'BraTS2021_01235', 'BraTS2021_01236', 'BraTS2021_01241', 'BraTS2021_01242', 'BraTS2021_01243', 'BraTS2021_01244', 'BraTS2021_01245', 'BraTS2021_01246', 'BraTS2021_01247', 'BraTS2021_01248', 'BraTS2021_01249', 'BraTS2021_01250', 'BraTS2021_01251', 'BraTS2021_01252', 'BraTS2021_01253', 'BraTS2021_01254', 'BraTS2021_01255', 'BraTS2021_01256', 'BraTS2021_01257', 'BraTS2021_01258', 'BraTS2021_01259', 'BraTS2021_01260', 'BraTS2021_01261', 'BraTS2021_01263', 'BraTS2021_01264', 'BraTS2021_01266', 'BraTS2021_01267', 'BraTS2021_01268', 'BraTS2021_01269', 'BraTS2021_01270', 'BraTS2021_01271', 'BraTS2021_01272', 'BraTS2021_01275', 'BraTS2021_01276', 'BraTS2021_01277', 'BraTS2021_01278', 'BraTS2021_01279', 'BraTS2021_01280', 'BraTS2021_01281', 'BraTS2021_01285', 'BraTS2021_01286', 'BraTS2021_01287', 'BraTS2021_01288', 'BraTS2021_01290', 'BraTS2021_01291', 'BraTS2021_01292', 'BraTS2021_01293', 'BraTS2021_01295', 'BraTS2021_01297', 'BraTS2021_01298', 'BraTS2021_01299', 'BraTS2021_01300', 'BraTS2021_01301', 'BraTS2021_01302', 'BraTS2021_01303', 'BraTS2021_01304', 'BraTS2021_01307', 'BraTS2021_01308', 'BraTS2021_01309', 'BraTS2021_01310', 'BraTS2021_01311', 'BraTS2021_01313', 'BraTS2021_01314', 'BraTS2021_01315', 'BraTS2021_01317', 'BraTS2021_01318', 'BraTS2021_01321', 'BraTS2021_01322', 'BraTS2021_01323', 'BraTS2021_01324', 'BraTS2021_01325', 'BraTS2021_01327', 'BraTS2021_01329', 'BraTS2021_01330', 'BraTS2021_01332', 'BraTS2021_01334', 'BraTS2021_01336', 'BraTS2021_01337', 'BraTS2021_01338', 'BraTS2021_01341', 'BraTS2021_01343', 'BraTS2021_01344', 'BraTS2021_01346', 'BraTS2021_01347', 'BraTS2021_01348', 'BraTS2021_01351', 'BraTS2021_01352', 'BraTS2021_01353', 'BraTS2021_01354', 'BraTS2021_01355', 'BraTS2021_01358', 'BraTS2021_01360', 'BraTS2021_01362', 'BraTS2021_01364', 'BraTS2021_01365', 'BraTS2021_01366', 'BraTS2021_01368', 'BraTS2021_01369', 'BraTS2021_01370', 'BraTS2021_01371', 'BraTS2021_01372', 'BraTS2021_01373', 'BraTS2021_01374', 'BraTS2021_01376', 'BraTS2021_01377', 'BraTS2021_01378', 'BraTS2021_01379', 'BraTS2021_01380', 'BraTS2021_01381', 'BraTS2021_01382', 'BraTS2021_01383', 'BraTS2021_01384', 'BraTS2021_01385', 'BraTS2021_01386', 'BraTS2021_01390', 'BraTS2021_01391', 'BraTS2021_01392', 'BraTS2021_01393', 'BraTS2021_01394', 'BraTS2021_01395', 'BraTS2021_01396', 'BraTS2021_01397', 'BraTS2021_01400', 'BraTS2021_01401', 'BraTS2021_01403', 'BraTS2021_01404', 'BraTS2021_01405', 'BraTS2021_01406', 'BraTS2021_01407', 'BraTS2021_01408', 'BraTS2021_01409', 'BraTS2021_01410', 'BraTS2021_01411', 'BraTS2021_01412', 'BraTS2021_01413', 'BraTS2021_01414', 'BraTS2021_01415', 'BraTS2021_01416', 'BraTS2021_01417', 'BraTS2021_01418', 'BraTS2021_01419', 'BraTS2021_01420', 'BraTS2021_01421', 'BraTS2021_01422', 'BraTS2021_01424', 'BraTS2021_01426', 'BraTS2021_01427', 'BraTS2021_01428', 'BraTS2021_01429', 'BraTS2021_01430', 'BraTS2021_01431', 'BraTS2021_01432', 'BraTS2021_01433', 'BraTS2021_01434', 'BraTS2021_01435', 'BraTS2021_01436', 'BraTS2021_01437', 'BraTS2021_01438', 'BraTS2021_01439', 'BraTS2021_01441', 'BraTS2021_01442', 'BraTS2021_01444', 'BraTS2021_01445', 'BraTS2021_01446', 'BraTS2021_01447', 'BraTS2021_01448', 'BraTS2021_01449', 'BraTS2021_01451', 'BraTS2021_01452', 'BraTS2021_01453', 'BraTS2021_01454', 'BraTS2021_01455', 'BraTS2021_01456', 'BraTS2021_01457', 'BraTS2021_01458', 'BraTS2021_01459', 'BraTS2021_01460', 'BraTS2021_01462', 'BraTS2021_01463', 'BraTS2021_01464', 'BraTS2021_01466', 'BraTS2021_01468', 'BraTS2021_01469', 'BraTS2021_01470', 'BraTS2021_01471', 'BraTS2021_01472', 'BraTS2021_01473', 'BraTS2021_01474', 'BraTS2021_01475', 'BraTS2021_01477', 'BraTS2021_01478', 'BraTS2021_01479', 'BraTS2021_01482', 'BraTS2021_01483', 'BraTS2021_01484', 'BraTS2021_01485', 'BraTS2021_01486', 'BraTS2021_01487', 'BraTS2021_01491', 'BraTS2021_01493', 'BraTS2021_01495', 'BraTS2021_01497', 'BraTS2021_01498', 'BraTS2021_01499', 'BraTS2021_01502', 'BraTS2021_01503', 'BraTS2021_01504', 'BraTS2021_01505', 'BraTS2021_01506', 'BraTS2021_01508', 'BraTS2021_01509', 'BraTS2021_01510', 'BraTS2021_01511', 'BraTS2021_01512', 'BraTS2021_01514', 'BraTS2021_01515', 'BraTS2021_01516', 'BraTS2021_01517', 'BraTS2021_01519', 'BraTS2021_01520', 'BraTS2021_01523', 'BraTS2021_01525', 'BraTS2021_01526', 'BraTS2021_01527', 'BraTS2021_01528', 'BraTS2021_01529', 'BraTS2021_01531', 'BraTS2021_01532', 'BraTS2021_01533', 'BraTS2021_01534', 'BraTS2021_01535', 'BraTS2021_01536', 'BraTS2021_01537', 'BraTS2021_01539', 'BraTS2021_01541', 'BraTS2021_01542', 'BraTS2021_01543', 'BraTS2021_01544', 'BraTS2021_01545', 'BraTS2021_01546', 'BraTS2021_01548', 'BraTS2021_01549', 'BraTS2021_01550', 'BraTS2021_01551', 'BraTS2021_01552', 'BraTS2021_01553', 'BraTS2021_01554', 'BraTS2021_01555', 'BraTS2021_01556', 'BraTS2021_01557', 'BraTS2021_01558', 'BraTS2021_01559', 'BraTS2021_01561', 'BraTS2021_01563', 'BraTS2021_01564', 'BraTS2021_01565', 'BraTS2021_01568', 'BraTS2021_01569', 'BraTS2021_01570', 'BraTS2021_01571', 'BraTS2021_01572', 'BraTS2021_01573', 'BraTS2021_01574', 'BraTS2021_01575', 'BraTS2021_01576', 'BraTS2021_01577', 'BraTS2021_01578', 'BraTS2021_01579', 'BraTS2021_01580', 'BraTS2021_01581', 'BraTS2021_01583', 'BraTS2021_01585', 'BraTS2021_01586', 'BraTS2021_01587', 'BraTS2021_01588', 'BraTS2021_01589', 'BraTS2021_01590', 'BraTS2021_01591', 'BraTS2021_01592', 'BraTS2021_01595', 'BraTS2021_01596', 'BraTS2021_01597', 'BraTS2021_01598', 'BraTS2021_01600', 'BraTS2021_01602', 'BraTS2021_01603', 'BraTS2021_01604', 'BraTS2021_01605', 'BraTS2021_01606', 'BraTS2021_01607', 'BraTS2021_01610', 'BraTS2021_01611', 'BraTS2021_01612', 'BraTS2021_01615', 'BraTS2021_01616', 'BraTS2021_01617', 'BraTS2021_01618', 'BraTS2021_01619', 'BraTS2021_01620', 'BraTS2021_01621', 'BraTS2021_01622', 'BraTS2021_01623', 'BraTS2021_01624', 'BraTS2021_01625', 'BraTS2021_01627', 'BraTS2021_01628', 'BraTS2021_01629', 'BraTS2021_01630', 'BraTS2021_01631', 'BraTS2021_01632', 'BraTS2021_01633', 'BraTS2021_01635', 'BraTS2021_01636', 'BraTS2021_01637', 'BraTS2021_01638', 'BraTS2021_01639', 'BraTS2021_01640', 'BraTS2021_01641', 'BraTS2021_01642', 'BraTS2021_01643', 'BraTS2021_01644', 'BraTS2021_01645', 'BraTS2021_01646', 'BraTS2021_01647', 'BraTS2021_01648', 'BraTS2021_01649', 'BraTS2021_01650', 'BraTS2021_01654', 'BraTS2021_01655', 'BraTS2021_01656', 'BraTS2021_01657', 'BraTS2021_01658', 'BraTS2021_01659', 'BraTS2021_01663', 'BraTS2021_01664', 'BraTS2021_01665', 'BraTS2021_01666']) +2022-05-14 04:21:48.063478: VALIDATION KEYS: + odict_keys(['BraTS2021_00000', 'BraTS2021_00009', 'BraTS2021_00016', 'BraTS2021_00024', 'BraTS2021_00028', 'BraTS2021_00031', 'BraTS2021_00035', 'BraTS2021_00045', 'BraTS2021_00046', 'BraTS2021_00051', 'BraTS2021_00070', 'BraTS2021_00078', 'BraTS2021_00085', 'BraTS2021_00087', 'BraTS2021_00088', 'BraTS2021_00089', 'BraTS2021_00099', 'BraTS2021_00102', 'BraTS2021_00104', 'BraTS2021_00106', 'BraTS2021_00107', 'BraTS2021_00110', 'BraTS2021_00117', 'BraTS2021_00123', 'BraTS2021_00126', 'BraTS2021_00127', 'BraTS2021_00130', 'BraTS2021_00136', 'BraTS2021_00137', 'BraTS2021_00140', 'BraTS2021_00144', 'BraTS2021_00152', 'BraTS2021_00160', 'BraTS2021_00185', 'BraTS2021_00192', 'BraTS2021_00194', 'BraTS2021_00211', 'BraTS2021_00212', 'BraTS2021_00214', 'BraTS2021_00217', 'BraTS2021_00218', 'BraTS2021_00231', 'BraTS2021_00243', 'BraTS2021_00251', 'BraTS2021_00266', 'BraTS2021_00270', 'BraTS2021_00274', 'BraTS2021_00283', 'BraTS2021_00286', 'BraTS2021_00292', 'BraTS2021_00294', 'BraTS2021_00304', 'BraTS2021_00311', 'BraTS2021_00313', 'BraTS2021_00334', 'BraTS2021_00336', 'BraTS2021_00350', 'BraTS2021_00353', 'BraTS2021_00366', 'BraTS2021_00375', 'BraTS2021_00376', 'BraTS2021_00379', 'BraTS2021_00391', 'BraTS2021_00399', 'BraTS2021_00402', 'BraTS2021_00419', 'BraTS2021_00423', 'BraTS2021_00426', 'BraTS2021_00430', 'BraTS2021_00433', 'BraTS2021_00436', 'BraTS2021_00442', 'BraTS2021_00464', 'BraTS2021_00480', 'BraTS2021_00485', 'BraTS2021_00495', 'BraTS2021_00500', 'BraTS2021_00514', 'BraTS2021_00516', 'BraTS2021_00549', 'BraTS2021_00558', 'BraTS2021_00583', 'BraTS2021_00586', 'BraTS2021_00587', 'BraTS2021_00597', 'BraTS2021_00602', 'BraTS2021_00605', 'BraTS2021_00607', 'BraTS2021_00630', 'BraTS2021_00639', 'BraTS2021_00641', 'BraTS2021_00654', 'BraTS2021_00693', 'BraTS2021_00709', 'BraTS2021_00715', 'BraTS2021_00716', 'BraTS2021_00727', 'BraTS2021_00733', 'BraTS2021_00735', 'BraTS2021_00753', 'BraTS2021_00757', 'BraTS2021_00759', 'BraTS2021_00772', 'BraTS2021_00775', 'BraTS2021_00777', 'BraTS2021_00782', 'BraTS2021_00791', 'BraTS2021_00793', 'BraTS2021_00810', 'BraTS2021_00818', 'BraTS2021_00834', 'BraTS2021_00836', 'BraTS2021_01001', 'BraTS2021_01004', 'BraTS2021_01005', 'BraTS2021_01018', 'BraTS2021_01037', 'BraTS2021_01043', 'BraTS2021_01050', 'BraTS2021_01051', 'BraTS2021_01058', 'BraTS2021_01063', 'BraTS2021_01066', 'BraTS2021_01072', 'BraTS2021_01079', 'BraTS2021_01081', 'BraTS2021_01088', 'BraTS2021_01089', 'BraTS2021_01096', 'BraTS2021_01101', 'BraTS2021_01110', 'BraTS2021_01118', 'BraTS2021_01123', 'BraTS2021_01124', 'BraTS2021_01137', 'BraTS2021_01141', 'BraTS2021_01142', 'BraTS2021_01146', 'BraTS2021_01147', 'BraTS2021_01152', 'BraTS2021_01162', 'BraTS2021_01169', 'BraTS2021_01170', 'BraTS2021_01173', 'BraTS2021_01181', 'BraTS2021_01183', 'BraTS2021_01191', 'BraTS2021_01198', 'BraTS2021_01206', 'BraTS2021_01208', 'BraTS2021_01209', 'BraTS2021_01210', 'BraTS2021_01211', 'BraTS2021_01212', 'BraTS2021_01225', 'BraTS2021_01231', 'BraTS2021_01237', 'BraTS2021_01238', 'BraTS2021_01239', 'BraTS2021_01240', 'BraTS2021_01262', 'BraTS2021_01265', 'BraTS2021_01273', 'BraTS2021_01274', 'BraTS2021_01282', 'BraTS2021_01283', 'BraTS2021_01284', 'BraTS2021_01289', 'BraTS2021_01294', 'BraTS2021_01296', 'BraTS2021_01305', 'BraTS2021_01306', 'BraTS2021_01312', 'BraTS2021_01316', 'BraTS2021_01319', 'BraTS2021_01320', 'BraTS2021_01326', 'BraTS2021_01328', 'BraTS2021_01331', 'BraTS2021_01333', 'BraTS2021_01335', 'BraTS2021_01339', 'BraTS2021_01340', 'BraTS2021_01342', 'BraTS2021_01345', 'BraTS2021_01349', 'BraTS2021_01350', 'BraTS2021_01356', 'BraTS2021_01357', 'BraTS2021_01359', 'BraTS2021_01361', 'BraTS2021_01363', 'BraTS2021_01367', 'BraTS2021_01375', 'BraTS2021_01387', 'BraTS2021_01388', 'BraTS2021_01389', 'BraTS2021_01398', 'BraTS2021_01399', 'BraTS2021_01402', 'BraTS2021_01423', 'BraTS2021_01425', 'BraTS2021_01440', 'BraTS2021_01443', 'BraTS2021_01450', 'BraTS2021_01461', 'BraTS2021_01465', 'BraTS2021_01467', 'BraTS2021_01476', 'BraTS2021_01480', 'BraTS2021_01481', 'BraTS2021_01488', 'BraTS2021_01489', 'BraTS2021_01490', 'BraTS2021_01492', 'BraTS2021_01494', 'BraTS2021_01496', 'BraTS2021_01500', 'BraTS2021_01501', 'BraTS2021_01507', 'BraTS2021_01513', 'BraTS2021_01518', 'BraTS2021_01521', 'BraTS2021_01522', 'BraTS2021_01524', 'BraTS2021_01530', 'BraTS2021_01538', 'BraTS2021_01540', 'BraTS2021_01547', 'BraTS2021_01560', 'BraTS2021_01562', 'BraTS2021_01566', 'BraTS2021_01567', 'BraTS2021_01582', 'BraTS2021_01584', 'BraTS2021_01593', 'BraTS2021_01594', 'BraTS2021_01599', 'BraTS2021_01601', 'BraTS2021_01608', 'BraTS2021_01609', 'BraTS2021_01613', 'BraTS2021_01614', 'BraTS2021_01626', 'BraTS2021_01634', 'BraTS2021_01651', 'BraTS2021_01652', 'BraTS2021_01653', 'BraTS2021_01660', 'BraTS2021_01661', 'BraTS2021_01662']) +2022-05-14 04:21:52.560964: lr: 3.3e-05 +2022-05-14 04:22:17.682653: +epoch: 0 +2022-05-14 04:29:49.334707: train loss : 0.3112 +2022-05-14 04:30:14.339277: validation loss: 0.1569 +2022-05-14 04:30:14.339949: Average global foreground Dice: [0.894, 0.8216, 0.8234] +2022-05-14 04:30:14.340045: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 04:30:14.672065: lr: 3.3e-05 +2022-05-14 04:30:14.672269: saving scheduled checkpoint file... +2022-05-14 04:30:14.720186: saving checkpoint... +2022-05-14 04:30:15.285822: done, saving took 0.61 seconds +2022-05-14 04:30:15.302295: done +2022-05-14 04:30:27.229711: This epoch took 489.546284 s + +2022-05-14 04:30:27.230044: +epoch: 1 +2022-05-14 04:37:43.536327: train loss : 0.1541 +2022-05-14 04:38:08.547851: validation loss: 0.1301 +2022-05-14 04:38:08.548512: Average global foreground Dice: [0.9127, 0.8468, 0.847] +2022-05-14 04:38:08.548598: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 04:38:08.941509: lr: 6.7e-05 +2022-05-14 04:38:08.941713: saving scheduled checkpoint file... +2022-05-14 04:38:08.991040: saving checkpoint... +2022-05-14 04:38:09.779329: done, saving took 0.84 seconds +2022-05-14 04:38:09.818769: done +2022-05-14 04:38:09.854283: saving checkpoint... +2022-05-14 04:38:10.375610: done, saving took 0.56 seconds +2022-05-14 04:38:32.985158: This epoch took 485.755001 s + +2022-05-14 04:38:32.985448: +epoch: 2 +2022-05-14 04:45:48.831101: train loss : 0.1181 +2022-05-14 04:46:13.857103: validation loss: 0.1055 +2022-05-14 04:46:13.857738: Average global foreground Dice: [0.9218, 0.8861, 0.8745] +2022-05-14 04:46:13.857843: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 04:46:14.256945: lr: 0.0001 +2022-05-14 04:46:14.257143: saving scheduled checkpoint file... +2022-05-14 04:46:14.303631: saving checkpoint... +2022-05-14 04:46:15.098059: done, saving took 0.84 seconds +2022-05-14 04:46:15.119099: done +2022-05-14 04:46:15.154187: saving checkpoint... +2022-05-14 04:46:15.849027: done, saving took 0.73 seconds +2022-05-14 04:46:35.562625: This epoch took 482.577081 s + +2022-05-14 04:46:35.562907: +epoch: 3 +2022-05-14 04:53:52.276014: train loss : 0.1037 +2022-05-14 04:54:17.338473: validation loss: 0.0807 +2022-05-14 04:54:17.339100: Average global foreground Dice: [0.9268, 0.8961, 0.886] +2022-05-14 04:54:17.339185: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 04:54:17.731920: lr: 0.000133 +2022-05-14 04:54:17.732120: saving scheduled checkpoint file... +2022-05-14 04:54:17.778707: saving checkpoint... +2022-05-14 04:54:18.573488: done, saving took 0.84 seconds +2022-05-14 04:54:18.594745: done +2022-05-14 04:54:18.630093: saving checkpoint... +2022-05-14 04:54:19.371422: done, saving took 0.78 seconds +2022-05-14 04:54:55.006617: This epoch took 499.443601 s + +2022-05-14 04:54:55.006930: +epoch: 4 +2022-05-14 05:02:11.619222: train loss : 0.0913 +2022-05-14 05:02:36.678671: validation loss: 0.0834 +2022-05-14 05:02:36.679334: Average global foreground Dice: [0.9255, 0.9044, 0.8951] +2022-05-14 05:02:36.679418: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 05:02:37.156547: lr: 0.000167 +2022-05-14 05:02:37.156729: saving scheduled checkpoint file... +2022-05-14 05:02:37.202531: saving checkpoint... +2022-05-14 05:02:37.969900: done, saving took 0.81 seconds +2022-05-14 05:02:37.987777: done +2022-05-14 05:02:38.023556: saving checkpoint... +2022-05-14 05:02:38.787804: done, saving took 0.80 seconds +2022-05-14 05:02:56.012624: This epoch took 481.005594 s + +2022-05-14 05:02:56.012906: +epoch: 5 +2022-05-14 05:10:12.780711: train loss : 0.0847 +2022-05-14 05:10:37.808244: validation loss: 0.0692 +2022-05-14 05:10:37.809020: Average global foreground Dice: [0.9325, 0.9009, 0.8866] +2022-05-14 05:10:37.809111: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 05:10:38.284482: lr: 0.0002 +2022-05-14 05:10:38.284671: saving scheduled checkpoint file... +2022-05-14 05:10:38.330705: saving checkpoint... +2022-05-14 05:10:39.098114: done, saving took 0.81 seconds +2022-05-14 05:10:39.116685: done +2022-05-14 05:10:39.153175: saving checkpoint... +2022-05-14 05:10:39.900839: done, saving took 0.78 seconds +2022-05-14 05:10:58.499530: This epoch took 482.486514 s + +2022-05-14 05:10:58.499846: +epoch: 6 +2022-05-14 05:18:15.124790: train loss : 0.0778 +2022-05-14 05:18:40.171201: validation loss: 0.0621 +2022-05-14 05:18:40.171905: Average global foreground Dice: [0.9378, 0.9237, 0.9014] +2022-05-14 05:18:40.171992: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 05:18:40.655962: lr: 0.000233 +2022-05-14 05:18:40.656157: saving scheduled checkpoint file... +2022-05-14 05:18:40.702194: saving checkpoint... +2022-05-14 05:18:41.471920: done, saving took 0.82 seconds +2022-05-14 05:18:41.492313: done +2022-05-14 05:18:41.532904: saving checkpoint... +2022-05-14 05:18:42.225664: done, saving took 0.73 seconds +2022-05-14 05:18:59.790694: This epoch took 481.290733 s + +2022-05-14 05:18:59.791035: +epoch: 7 +2022-05-14 05:26:16.672900: train loss : 0.0785 +2022-05-14 05:26:41.677150: validation loss: 0.0636 +2022-05-14 05:26:41.677870: Average global foreground Dice: [0.9217, 0.8796, 0.8993] +2022-05-14 05:26:41.677957: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 05:26:42.085428: lr: 0.000267 +2022-05-14 05:26:42.085624: saving scheduled checkpoint file... +2022-05-14 05:26:42.132668: saving checkpoint... +2022-05-14 05:26:42.931546: done, saving took 0.85 seconds +2022-05-14 05:26:42.955789: done +2022-05-14 05:26:42.991803: saving checkpoint... +2022-05-14 05:26:43.739642: done, saving took 0.78 seconds +2022-05-14 05:27:05.515979: This epoch took 485.724839 s + +2022-05-14 05:27:05.516264: +epoch: 8 +2022-05-14 05:34:21.738633: train loss : 0.0748 +2022-05-14 05:34:46.806837: validation loss: 0.0645 +2022-05-14 05:34:46.807508: Average global foreground Dice: [0.9158, 0.9027, 0.9107] +2022-05-14 05:34:46.807605: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 05:34:47.211041: lr: 0.0003 +2022-05-14 05:34:47.211238: saving scheduled checkpoint file... +2022-05-14 05:34:47.257690: saving checkpoint... +2022-05-14 05:34:48.060668: done, saving took 0.85 seconds +2022-05-14 05:34:48.081440: done +2022-05-14 05:34:48.116655: saving checkpoint... +2022-05-14 05:34:48.866418: done, saving took 0.78 seconds +2022-05-14 05:35:07.250259: This epoch took 481.733896 s + +2022-05-14 05:35:07.250551: +epoch: 9 +2022-05-14 05:42:23.475468: train loss : 0.0743 +2022-05-14 05:42:48.585928: validation loss: 0.0668 +2022-05-14 05:42:48.586619: Average global foreground Dice: [0.9353, 0.9161, 0.9089] +2022-05-14 05:42:48.586707: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 05:42:48.988509: lr: 0.0003 +2022-05-14 05:42:48.988707: saving scheduled checkpoint file... +2022-05-14 05:42:49.035794: saving checkpoint... +2022-05-14 05:42:49.811448: done, saving took 0.82 seconds +2022-05-14 05:42:49.831565: done +2022-05-14 05:42:49.872481: saving checkpoint... +2022-05-14 05:42:50.623984: done, saving took 0.79 seconds +2022-05-14 05:43:12.403634: This epoch took 485.152984 s + +2022-05-14 05:43:12.403928: +epoch: 10 +2022-05-14 05:50:28.434325: train loss : 0.0728 +2022-05-14 05:50:53.482607: validation loss: 0.0654 +2022-05-14 05:50:53.483284: Average global foreground Dice: [0.94, 0.928, 0.9134] +2022-05-14 05:50:53.483368: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 05:50:53.886598: lr: 0.0003 +2022-05-14 05:50:53.886804: saving scheduled checkpoint file... +2022-05-14 05:50:53.933394: saving checkpoint... +2022-05-14 05:50:54.747048: done, saving took 0.86 seconds +2022-05-14 05:50:54.848730: done +2022-05-14 05:50:54.880307: saving checkpoint... +2022-05-14 05:50:55.580984: done, saving took 0.73 seconds +2022-05-14 05:51:13.945658: This epoch took 481.541623 s + +2022-05-14 05:51:13.945964: +epoch: 11 +2022-05-14 05:58:30.371352: train loss : 0.0777 +2022-05-14 05:58:55.388366: validation loss: 0.0743 +2022-05-14 05:58:55.389056: Average global foreground Dice: [0.9248, 0.9112, 0.9077] +2022-05-14 05:58:55.389156: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 05:58:55.790535: lr: 0.0003 +2022-05-14 05:58:55.790716: saving scheduled checkpoint file... +2022-05-14 05:58:55.836700: saving checkpoint... +2022-05-14 05:58:56.636073: done, saving took 0.85 seconds +2022-05-14 05:58:56.658863: done +2022-05-14 05:58:56.699969: saving checkpoint... +2022-05-14 05:58:57.453721: done, saving took 0.79 seconds +2022-05-14 05:59:23.501191: This epoch took 489.555128 s + +2022-05-14 05:59:23.501488: +epoch: 12 +2022-05-14 06:06:39.881468: train loss : 0.0685 +2022-05-14 06:07:04.937686: validation loss: 0.0673 +2022-05-14 06:07:04.938432: Average global foreground Dice: [0.9368, 0.9119, 0.905] +2022-05-14 06:07:04.938518: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 06:07:05.345408: lr: 0.0003 +2022-05-14 06:07:05.345603: saving scheduled checkpoint file... +2022-05-14 06:07:05.391633: saving checkpoint... +2022-05-14 06:07:06.235716: done, saving took 0.89 seconds +2022-05-14 06:07:06.258015: done +2022-05-14 06:07:06.304036: saving checkpoint... +2022-05-14 06:07:07.100958: done, saving took 0.84 seconds +2022-05-14 06:07:22.507939: This epoch took 479.006351 s + +2022-05-14 06:07:22.508216: +epoch: 13 +2022-05-14 06:14:39.067359: train loss : 0.0654 +2022-05-14 06:15:04.115973: validation loss: 0.0549 +2022-05-14 06:15:04.116671: Average global foreground Dice: [0.9395, 0.9333, 0.9144] +2022-05-14 06:15:04.116758: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 06:15:04.516575: lr: 0.0003 +2022-05-14 06:15:04.516779: saving scheduled checkpoint file... +2022-05-14 06:15:04.563806: saving checkpoint... +2022-05-14 06:15:05.367255: done, saving took 0.85 seconds +2022-05-14 06:15:05.386652: done +2022-05-14 06:15:05.424004: saving checkpoint... +2022-05-14 06:15:06.144176: done, saving took 0.76 seconds +2022-05-14 06:15:23.117079: This epoch took 480.608761 s + +2022-05-14 06:15:23.117356: +epoch: 14 +2022-05-14 06:22:40.074800: train loss : 0.0618 +2022-05-14 06:23:05.100921: validation loss: 0.0516 +2022-05-14 06:23:05.101679: Average global foreground Dice: [0.9416, 0.9315, 0.9165] +2022-05-14 06:23:05.101773: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 06:23:05.500964: lr: 0.0003 +2022-05-14 06:23:05.501155: saving scheduled checkpoint file... +2022-05-14 06:23:05.549262: saving checkpoint... +2022-05-14 06:23:06.258231: done, saving took 0.76 seconds +2022-05-14 06:23:06.276396: done +2022-05-14 06:23:06.312798: saving checkpoint... +2022-05-14 06:23:07.008089: done, saving took 0.73 seconds +2022-05-14 06:23:29.588128: This epoch took 486.470658 s + +2022-05-14 06:23:29.588439: +epoch: 15 +2022-05-14 06:30:46.137189: train loss : 0.0657 +2022-05-14 06:31:11.164017: validation loss: 0.0702 +2022-05-14 06:31:11.164811: Average global foreground Dice: [0.9389, 0.9291, 0.9111] +2022-05-14 06:31:11.164901: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 06:31:11.577388: lr: 0.0003 +2022-05-14 06:31:11.577583: saving scheduled checkpoint file... +2022-05-14 06:31:11.626617: saving checkpoint... +2022-05-14 06:31:12.366525: done, saving took 0.79 seconds +2022-05-14 06:31:12.385446: done +2022-05-14 06:31:12.422390: saving checkpoint... +2022-05-14 06:31:13.120014: done, saving took 0.73 seconds +2022-05-14 06:31:31.984433: This epoch took 482.395890 s + +2022-05-14 06:31:31.984706: +epoch: 16 +2022-05-14 06:38:48.148856: train loss : 0.0634 +2022-05-14 06:39:13.166794: validation loss: 0.0555 +2022-05-14 06:39:13.167470: Average global foreground Dice: [0.9349, 0.9214, 0.905] +2022-05-14 06:39:13.167555: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 06:39:13.567736: lr: 0.000299 +2022-05-14 06:39:13.567945: saving scheduled checkpoint file... +2022-05-14 06:39:13.614295: saving checkpoint... +2022-05-14 06:39:14.319576: done, saving took 0.75 seconds +2022-05-14 06:39:14.337395: done +2022-05-14 06:39:14.372856: saving checkpoint... +2022-05-14 06:39:15.048301: done, saving took 0.71 seconds +2022-05-14 06:39:35.331798: This epoch took 483.346981 s + +2022-05-14 06:39:35.332077: +epoch: 17 +2022-05-14 06:46:52.122535: train loss : 0.0615 +2022-05-14 06:47:17.206103: validation loss: 0.0590 +2022-05-14 06:47:17.206708: Average global foreground Dice: [0.9359, 0.9228, 0.9159] +2022-05-14 06:47:17.206800: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 06:47:17.607969: lr: 0.000299 +2022-05-14 06:47:17.608168: saving scheduled checkpoint file... +2022-05-14 06:47:17.655134: saving checkpoint... +2022-05-14 06:47:18.371538: done, saving took 0.76 seconds +2022-05-14 06:47:18.390193: done +2022-05-14 06:47:18.424462: saving checkpoint... +2022-05-14 06:47:19.098816: done, saving took 0.71 seconds +2022-05-14 06:47:38.863986: This epoch took 483.531785 s + +2022-05-14 06:47:38.864360: +epoch: 18 +2022-05-14 06:54:55.379681: train loss : 0.0597 +2022-05-14 06:55:20.417582: validation loss: 0.0564 +2022-05-14 06:55:20.418225: Average global foreground Dice: [0.9436, 0.9186, 0.9197] +2022-05-14 06:55:20.418309: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 06:55:20.825238: lr: 0.000299 +2022-05-14 06:55:20.825441: saving scheduled checkpoint file... +2022-05-14 06:55:20.873212: saving checkpoint... +2022-05-14 06:55:21.623461: done, saving took 0.80 seconds +2022-05-14 06:55:21.659127: done +2022-05-14 06:55:21.693307: saving checkpoint... +2022-05-14 06:55:22.380575: done, saving took 0.72 seconds +2022-05-14 06:55:41.573755: This epoch took 482.709291 s + +2022-05-14 06:55:41.574068: +epoch: 19 +2022-05-14 07:02:58.352944: train loss : 0.0578 +2022-05-14 07:03:23.425884: validation loss: 0.0646 +2022-05-14 07:03:23.426568: Average global foreground Dice: [0.9416, 0.9238, 0.9159] +2022-05-14 07:03:23.426700: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 07:03:23.826986: lr: 0.000299 +2022-05-14 07:03:23.827192: saving scheduled checkpoint file... +2022-05-14 07:03:23.874424: saving checkpoint... +2022-05-14 07:03:24.575587: done, saving took 0.75 seconds +2022-05-14 07:03:24.593699: done +2022-05-14 07:03:24.627963: saving checkpoint... +2022-05-14 07:03:25.327611: done, saving took 0.73 seconds +2022-05-14 07:03:43.167840: This epoch took 481.593628 s + +2022-05-14 07:03:43.168130: +epoch: 20 +2022-05-14 07:10:59.629704: train loss : 0.0587 +2022-05-14 07:11:24.725866: validation loss: 0.0655 +2022-05-14 07:11:24.726591: Average global foreground Dice: [0.9409, 0.9287, 0.9129] +2022-05-14 07:11:24.726676: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 07:11:25.129279: lr: 0.000298 +2022-05-14 07:11:25.129470: saving scheduled checkpoint file... +2022-05-14 07:11:25.176111: saving checkpoint... +2022-05-14 07:11:25.905852: done, saving took 0.78 seconds +2022-05-14 07:11:25.924557: done +2022-05-14 07:11:25.959452: saving checkpoint... +2022-05-14 07:11:26.613981: done, saving took 0.69 seconds +2022-05-14 07:11:41.851212: This epoch took 478.682977 s + +2022-05-14 07:11:41.851513: +epoch: 21 +2022-05-14 07:18:58.749274: train loss : 0.0637 +2022-05-14 07:19:23.812097: validation loss: 0.0713 +2022-05-14 07:19:23.812819: Average global foreground Dice: [0.945, 0.9134, 0.9167] +2022-05-14 07:19:23.812905: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 07:19:24.207508: lr: 0.000298 +2022-05-14 07:19:24.207700: saving scheduled checkpoint file... +2022-05-14 07:19:24.254848: saving checkpoint... +2022-05-14 07:19:24.981598: done, saving took 0.77 seconds +2022-05-14 07:19:25.000038: done +2022-05-14 07:19:25.035856: saving checkpoint... +2022-05-14 07:19:25.746439: done, saving took 0.75 seconds +2022-05-14 07:19:43.512311: This epoch took 481.660700 s + +2022-05-14 07:19:43.512623: +epoch: 22 +2022-05-14 07:27:00.044794: train loss : 0.0616 +2022-05-14 07:27:25.113580: validation loss: 0.0711 +2022-05-14 07:27:25.114304: Average global foreground Dice: [0.9418, 0.9204, 0.9062] +2022-05-14 07:27:25.114403: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 07:27:25.514253: lr: 0.000298 +2022-05-14 07:27:25.514442: saving scheduled checkpoint file... +2022-05-14 07:27:25.560543: saving checkpoint... +2022-05-14 07:27:26.292530: done, saving took 0.78 seconds +2022-05-14 07:27:26.310614: done +2022-05-14 07:27:26.345080: saving checkpoint... +2022-05-14 07:27:27.021170: done, saving took 0.71 seconds +2022-05-14 07:27:49.296848: This epoch took 485.784123 s + +2022-05-14 07:27:49.297117: +epoch: 23 +2022-05-14 07:35:05.469582: train loss : 0.0606 +2022-05-14 07:35:30.544151: validation loss: 0.0637 +2022-05-14 07:35:30.544782: Average global foreground Dice: [0.9393, 0.9277, 0.918] +2022-05-14 07:35:30.544868: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 07:35:30.941555: lr: 0.000297 +2022-05-14 07:35:30.941758: saving scheduled checkpoint file... +2022-05-14 07:35:30.988539: saving checkpoint... +2022-05-14 07:35:31.708642: done, saving took 0.77 seconds +2022-05-14 07:35:31.726968: done +2022-05-14 07:35:31.761049: saving checkpoint... +2022-05-14 07:35:32.444510: done, saving took 0.72 seconds +2022-05-14 07:36:01.204624: This epoch took 491.907387 s + +2022-05-14 07:36:01.204994: +epoch: 24 +2022-05-14 07:43:17.960737: train loss : 0.0569 +2022-05-14 07:43:43.002231: validation loss: 0.0542 +2022-05-14 07:43:43.002947: Average global foreground Dice: [0.9448, 0.9266, 0.9134] +2022-05-14 07:43:43.003034: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 07:43:43.401050: lr: 0.000297 +2022-05-14 07:43:43.401247: saving scheduled checkpoint file... +2022-05-14 07:43:43.447551: saving checkpoint... +2022-05-14 07:43:44.239681: done, saving took 0.84 seconds +2022-05-14 07:43:44.258409: done +2022-05-14 07:43:44.294244: saving checkpoint... +2022-05-14 07:43:45.053028: done, saving took 0.79 seconds +2022-05-14 07:44:04.796534: This epoch took 483.591437 s + +2022-05-14 07:44:04.796823: +epoch: 25 +2022-05-14 07:51:21.436494: train loss : 0.0594 +2022-05-14 07:51:46.487422: validation loss: 0.0508 +2022-05-14 07:51:46.488192: Average global foreground Dice: [0.9463, 0.9171, 0.9208] +2022-05-14 07:51:46.488281: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 07:51:46.888453: lr: 0.000297 +2022-05-14 07:51:46.888647: saving scheduled checkpoint file... +2022-05-14 07:51:46.934117: saving checkpoint... +2022-05-14 07:51:47.725601: done, saving took 0.84 seconds +2022-05-14 07:51:47.743785: done +2022-05-14 07:51:47.779547: saving checkpoint... +2022-05-14 07:51:48.540557: done, saving took 0.80 seconds +2022-05-14 07:52:05.310191: This epoch took 480.513267 s + +2022-05-14 07:52:05.310483: +epoch: 26 +2022-05-14 07:59:22.055407: train loss : 0.0554 +2022-05-14 07:59:47.075532: validation loss: 0.0490 +2022-05-14 07:59:47.076182: Average global foreground Dice: [0.9466, 0.9412, 0.9208] +2022-05-14 07:59:47.076268: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 07:59:47.478508: lr: 0.000296 +2022-05-14 07:59:47.478710: saving scheduled checkpoint file... +2022-05-14 07:59:47.525873: saving checkpoint... +2022-05-14 07:59:48.329371: done, saving took 0.85 seconds +2022-05-14 07:59:48.349922: done +2022-05-14 07:59:48.391147: saving checkpoint... +2022-05-14 07:59:49.152967: done, saving took 0.80 seconds +2022-05-14 08:00:23.401191: This epoch took 498.090605 s + +2022-05-14 08:00:23.401478: +epoch: 27 +2022-05-14 08:07:40.810974: train loss : 0.0569 +2022-05-14 08:08:05.894502: validation loss: 0.0601 +2022-05-14 08:08:05.895120: Average global foreground Dice: [0.9466, 0.9363, 0.9224] +2022-05-14 08:08:05.895220: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 08:08:06.307164: lr: 0.000296 +2022-05-14 08:08:06.307349: saving scheduled checkpoint file... +2022-05-14 08:08:06.356413: saving checkpoint... +2022-05-14 08:08:07.149145: done, saving took 0.84 seconds +2022-05-14 08:08:07.169855: done +2022-05-14 08:08:07.211369: saving checkpoint... +2022-05-14 08:08:07.974440: done, saving took 0.80 seconds +2022-05-14 08:08:48.040385: This epoch took 504.638808 s + +2022-05-14 08:08:48.040680: +epoch: 28 +2022-05-14 08:16:05.054872: train loss : 0.0573 +2022-05-14 08:16:30.122547: validation loss: 0.0616 +2022-05-14 08:16:30.123444: Average global foreground Dice: [0.9421, 0.9305, 0.9277] +2022-05-14 08:16:30.123529: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 08:16:30.525853: lr: 0.000295 +2022-05-14 08:16:30.526053: saving scheduled checkpoint file... +2022-05-14 08:16:30.574851: saving checkpoint... +2022-05-14 08:16:31.365874: done, saving took 0.84 seconds +2022-05-14 08:16:31.383871: done +2022-05-14 08:16:31.419921: saving checkpoint... +2022-05-14 08:16:32.090880: done, saving took 0.71 seconds +2022-05-14 08:17:37.334151: This epoch took 529.293369 s + +2022-05-14 08:17:37.334427: +epoch: 29 +2022-05-14 08:24:53.466080: train loss : 0.0532 +2022-05-14 08:25:18.531254: validation loss: 0.0615 +2022-05-14 08:25:18.531924: Average global foreground Dice: [0.9368, 0.9339, 0.923] +2022-05-14 08:25:18.532009: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 08:25:18.935084: lr: 0.000295 +2022-05-14 08:25:18.935273: saving scheduled checkpoint file... +2022-05-14 08:25:18.981655: saving checkpoint... +2022-05-14 08:25:19.708319: done, saving took 0.77 seconds +2022-05-14 08:25:19.739133: done +2022-05-14 08:25:19.773960: saving checkpoint... +2022-05-14 08:25:20.419372: done, saving took 0.68 seconds +2022-05-14 08:25:44.466712: This epoch took 487.132169 s + +2022-05-14 08:25:44.467117: +epoch: 30 +2022-05-14 08:33:01.772955: train loss : 0.0561 +2022-05-14 08:33:26.935974: validation loss: 0.0607 +2022-05-14 08:33:26.936798: Average global foreground Dice: [0.9365, 0.9227, 0.9137] +2022-05-14 08:33:26.936888: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 08:33:27.357413: lr: 0.000294 +2022-05-14 08:33:27.357614: saving scheduled checkpoint file... +2022-05-14 08:33:27.404292: saving checkpoint... +2022-05-14 08:33:28.127787: done, saving took 0.77 seconds +2022-05-14 08:33:28.145920: done +2022-05-14 08:33:28.179863: saving checkpoint... +2022-05-14 08:33:28.886417: done, saving took 0.74 seconds +2022-05-14 08:33:46.436184: This epoch took 481.968970 s + +2022-05-14 08:33:46.436475: +epoch: 31 +2022-05-14 08:41:03.427266: train loss : 0.0522 +2022-05-14 08:41:28.458705: validation loss: 0.0579 +2022-05-14 08:41:28.459357: Average global foreground Dice: [0.9431, 0.9362, 0.9217] +2022-05-14 08:41:28.459442: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 08:41:28.860728: lr: 0.000294 +2022-05-14 08:41:28.860939: saving scheduled checkpoint file... +2022-05-14 08:41:28.907280: saving checkpoint... +2022-05-14 08:41:29.611738: done, saving took 0.75 seconds +2022-05-14 08:41:29.630613: done +2022-05-14 08:41:29.665250: saving checkpoint... +2022-05-14 08:41:30.360534: done, saving took 0.73 seconds +2022-05-14 08:41:48.281202: This epoch took 481.844609 s + +2022-05-14 08:41:48.281551: +epoch: 32 +2022-05-14 08:49:05.430301: train loss : 0.0545 +2022-05-14 08:49:30.482926: validation loss: 0.0530 +2022-05-14 08:49:30.483538: Average global foreground Dice: [0.9356, 0.9296, 0.9201] +2022-05-14 08:49:30.483622: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 08:49:30.884843: lr: 0.000293 +2022-05-14 08:49:30.885038: saving scheduled checkpoint file... +2022-05-14 08:49:30.932519: saving checkpoint... +2022-05-14 08:49:31.645872: done, saving took 0.76 seconds +2022-05-14 08:49:31.663089: done +2022-05-14 08:49:31.697320: saving checkpoint... +2022-05-14 08:49:32.406147: done, saving took 0.74 seconds +2022-05-14 08:50:10.107407: This epoch took 501.825754 s + +2022-05-14 08:50:10.107708: +epoch: 33 +2022-05-14 08:57:27.290452: train loss : 0.0530 +2022-05-14 08:57:52.290346: validation loss: 0.0659 +2022-05-14 08:57:52.290968: Average global foreground Dice: [0.9488, 0.9308, 0.9237] +2022-05-14 08:57:52.291055: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 08:57:52.692729: lr: 0.000293 +2022-05-14 08:57:52.692935: saving scheduled checkpoint file... +2022-05-14 08:57:52.739547: saving checkpoint... +2022-05-14 08:57:53.520668: done, saving took 0.83 seconds +2022-05-14 08:57:53.538844: done +2022-05-14 08:57:53.575746: saving checkpoint... +2022-05-14 08:57:54.349181: done, saving took 0.81 seconds +2022-05-14 08:58:11.614119: This epoch took 481.506290 s + +2022-05-14 08:58:11.614506: +epoch: 34 +2022-05-14 09:05:27.765172: train loss : 0.0542 +2022-05-14 09:05:52.819204: validation loss: 0.0632 +2022-05-14 09:05:52.819838: Average global foreground Dice: [0.9441, 0.9314, 0.9142] +2022-05-14 09:05:52.819925: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 09:05:53.223861: lr: 0.000292 +2022-05-14 09:05:53.224047: saving scheduled checkpoint file... +2022-05-14 09:05:53.271079: saving checkpoint... +2022-05-14 09:05:54.068361: done, saving took 0.84 seconds +2022-05-14 09:05:54.086383: done +2022-05-14 09:05:54.124254: saving checkpoint... +2022-05-14 09:05:54.900695: done, saving took 0.81 seconds +2022-05-14 09:06:25.522255: This epoch took 493.907643 s + +2022-05-14 09:06:25.522558: +epoch: 35 +2022-05-14 09:13:42.302405: train loss : 0.0509 +2022-05-14 09:14:07.314986: validation loss: 0.0502 +2022-05-14 09:14:07.315604: Average global foreground Dice: [0.9454, 0.942, 0.9194] +2022-05-14 09:14:07.315690: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 09:14:07.719542: lr: 0.000291 +2022-05-14 09:14:07.719733: saving scheduled checkpoint file... +2022-05-14 09:14:07.766184: saving checkpoint... +2022-05-14 09:14:08.560943: done, saving took 0.84 seconds +2022-05-14 09:14:08.579085: done +2022-05-14 09:14:08.615125: saving checkpoint... +2022-05-14 09:14:09.386034: done, saving took 0.81 seconds +2022-05-14 09:14:28.743991: This epoch took 483.221323 s + +2022-05-14 09:14:28.744271: +epoch: 36 +2022-05-14 09:21:45.475508: train loss : 0.0541 +2022-05-14 09:22:10.538679: validation loss: 0.0622 +2022-05-14 09:22:10.539361: Average global foreground Dice: [0.9465, 0.9307, 0.9178] +2022-05-14 09:22:10.539454: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 09:22:10.945094: lr: 0.000291 +2022-05-14 09:22:10.945292: saving scheduled checkpoint file... +2022-05-14 09:22:10.991731: saving checkpoint... +2022-05-14 09:22:11.784951: done, saving took 0.84 seconds +2022-05-14 09:22:11.805501: done +2022-05-14 09:22:11.846584: saving checkpoint... +2022-05-14 09:22:12.618447: done, saving took 0.81 seconds +2022-05-14 09:22:30.286609: This epoch took 481.542231 s + +2022-05-14 09:22:30.286888: +epoch: 37 +2022-05-14 09:29:47.922554: train loss : 0.0524 +2022-05-14 09:30:12.935416: validation loss: 0.0569 +2022-05-14 09:30:12.936169: Average global foreground Dice: [0.948, 0.9416, 0.9225] +2022-05-14 09:30:12.936255: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 09:30:13.341935: lr: 0.00029 +2022-05-14 09:30:13.342138: saving scheduled checkpoint file... +2022-05-14 09:30:13.388548: saving checkpoint... +2022-05-14 09:30:14.175102: done, saving took 0.83 seconds +2022-05-14 09:30:14.193019: done +2022-05-14 09:30:14.229104: saving checkpoint... +2022-05-14 09:30:15.054093: done, saving took 0.86 seconds +2022-05-14 09:30:39.770210: This epoch took 489.483202 s + +2022-05-14 09:30:39.770599: +epoch: 38 +2022-05-14 09:37:56.575041: train loss : 0.0506 +2022-05-14 09:38:21.716583: validation loss: 0.0527 +2022-05-14 09:38:21.717181: Average global foreground Dice: [0.9439, 0.9356, 0.9228] +2022-05-14 09:38:21.717265: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 09:38:22.123398: lr: 0.000289 +2022-05-14 09:38:22.123589: saving scheduled checkpoint file... +2022-05-14 09:38:22.169983: saving checkpoint... +2022-05-14 09:38:22.971037: done, saving took 0.85 seconds +2022-05-14 09:38:22.989161: done +2022-05-14 09:38:23.026124: saving checkpoint... +2022-05-14 09:38:23.800986: done, saving took 0.81 seconds +2022-05-14 09:39:20.422873: This epoch took 520.652163 s + +2022-05-14 09:39:20.423202: +epoch: 39 +2022-05-14 09:46:37.551255: train loss : 0.0534 +2022-05-14 09:47:02.591238: validation loss: 0.0549 +2022-05-14 09:47:02.591904: Average global foreground Dice: [0.9456, 0.9392, 0.92] +2022-05-14 09:47:02.591989: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 09:47:02.996266: lr: 0.000289 +2022-05-14 09:47:02.996469: saving scheduled checkpoint file... +2022-05-14 09:47:03.043116: saving checkpoint... +2022-05-14 09:47:03.840191: done, saving took 0.84 seconds +2022-05-14 09:47:03.859180: done +2022-05-14 09:47:03.895754: saving checkpoint... +2022-05-14 09:47:04.683501: done, saving took 0.82 seconds +2022-05-14 09:47:25.979488: This epoch took 485.556184 s + +2022-05-14 09:47:25.979749: +epoch: 40 +2022-05-14 09:54:42.501166: train loss : 0.0543 +2022-05-14 09:55:07.558404: validation loss: 0.0563 +2022-05-14 09:55:07.559247: Average global foreground Dice: [0.9466, 0.9358, 0.928] +2022-05-14 09:55:07.559333: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 09:55:07.968324: lr: 0.000288 +2022-05-14 09:55:07.968522: saving scheduled checkpoint file... +2022-05-14 09:55:08.015654: saving checkpoint... +2022-05-14 09:55:08.758168: done, saving took 0.79 seconds +2022-05-14 09:55:08.775315: done +2022-05-14 09:55:08.809047: saving checkpoint... +2022-05-14 09:55:09.551165: done, saving took 0.78 seconds +2022-05-14 09:55:36.761260: This epoch took 490.781386 s + +2022-05-14 09:55:36.761566: +epoch: 41 +2022-05-14 10:02:53.444741: train loss : 0.0553 +2022-05-14 10:03:18.480296: validation loss: 0.0545 +2022-05-14 10:03:18.480909: Average global foreground Dice: [0.9482, 0.9446, 0.9237] +2022-05-14 10:03:18.480994: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 10:03:18.883038: lr: 0.000287 +2022-05-14 10:03:18.883224: saving scheduled checkpoint file... +2022-05-14 10:03:18.929124: saving checkpoint... +2022-05-14 10:03:19.658987: done, saving took 0.78 seconds +2022-05-14 10:03:19.676592: done +2022-05-14 10:03:19.710711: saving checkpoint... +2022-05-14 10:03:20.389473: done, saving took 0.71 seconds +2022-05-14 10:03:41.625942: This epoch took 484.864253 s + +2022-05-14 10:03:41.626340: +epoch: 42 +2022-05-14 10:10:58.421671: train loss : 0.0520 +2022-05-14 10:11:23.420776: validation loss: 0.0551 +2022-05-14 10:11:23.421438: Average global foreground Dice: [0.948, 0.9422, 0.9201] +2022-05-14 10:11:23.421525: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 10:11:23.821238: lr: 0.000286 +2022-05-14 10:11:23.821450: saving scheduled checkpoint file... +2022-05-14 10:11:23.868383: saving checkpoint... +2022-05-14 10:11:24.598813: done, saving took 0.78 seconds +2022-05-14 10:11:24.617460: done +2022-05-14 10:11:24.651653: saving checkpoint... +2022-05-14 10:11:25.306984: done, saving took 0.69 seconds +2022-05-14 10:11:46.643270: This epoch took 485.016824 s + +2022-05-14 10:11:46.643550: +epoch: 43 +2022-05-14 10:19:03.088048: train loss : 0.0509 +2022-05-14 10:19:28.076690: validation loss: 0.0556 +2022-05-14 10:19:28.077361: Average global foreground Dice: [0.9439, 0.9225, 0.918] +2022-05-14 10:19:28.077458: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 10:19:28.476040: lr: 0.000285 +2022-05-14 10:19:28.476243: saving scheduled checkpoint file... +2022-05-14 10:19:28.523156: saving checkpoint... +2022-05-14 10:19:29.243976: done, saving took 0.77 seconds +2022-05-14 10:19:29.261628: done +2022-05-14 10:19:51.224488: This epoch took 484.580813 s + +2022-05-14 10:19:51.224891: +epoch: 44 +2022-05-14 10:27:07.740507: train loss : 0.0487 +2022-05-14 10:27:32.787065: validation loss: 0.0524 +2022-05-14 10:27:32.787646: Average global foreground Dice: [0.9488, 0.9392, 0.9262] +2022-05-14 10:27:32.787730: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 10:27:33.191110: lr: 0.000285 +2022-05-14 10:27:33.191305: saving scheduled checkpoint file... +2022-05-14 10:27:33.237826: saving checkpoint... +2022-05-14 10:27:33.941662: done, saving took 0.75 seconds +2022-05-14 10:27:33.960303: done +2022-05-14 10:27:33.995056: saving checkpoint... +2022-05-14 10:27:34.708126: done, saving took 0.75 seconds +2022-05-14 10:27:59.226175: This epoch took 488.001182 s + +2022-05-14 10:27:59.226442: +epoch: 45 +2022-05-14 10:35:16.071601: train loss : 0.0533 +2022-05-14 10:35:41.102419: validation loss: 0.0580 +2022-05-14 10:35:41.103057: Average global foreground Dice: [0.942, 0.9362, 0.9234] +2022-05-14 10:35:41.103143: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 10:35:41.500110: lr: 0.000284 +2022-05-14 10:35:41.500313: saving scheduled checkpoint file... +2022-05-14 10:35:41.547077: saving checkpoint... +2022-05-14 10:35:42.338455: done, saving took 0.84 seconds +2022-05-14 10:35:42.356770: done +2022-05-14 10:35:42.392639: saving checkpoint... +2022-05-14 10:35:43.162490: done, saving took 0.81 seconds +2022-05-14 10:36:00.244584: This epoch took 481.018031 s + +2022-05-14 10:36:00.244910: +epoch: 46 +2022-05-14 10:43:17.555372: train loss : 0.0515 +2022-05-14 10:43:42.611300: validation loss: 0.0471 +2022-05-14 10:43:42.611969: Average global foreground Dice: [0.9487, 0.938, 0.9301] +2022-05-14 10:43:42.612053: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 10:43:43.010304: lr: 0.000283 +2022-05-14 10:43:43.010484: saving scheduled checkpoint file... +2022-05-14 10:43:43.056549: saving checkpoint... +2022-05-14 10:43:43.858382: done, saving took 0.85 seconds +2022-05-14 10:43:43.876149: done +2022-05-14 10:43:43.911827: saving checkpoint... +2022-05-14 10:43:44.679422: done, saving took 0.80 seconds +2022-05-14 10:44:05.313593: This epoch took 485.068580 s + +2022-05-14 10:44:05.313931: +epoch: 47 +2022-05-14 10:51:21.728561: train loss : 0.0510 +2022-05-14 10:51:46.714863: validation loss: 0.0506 +2022-05-14 10:51:46.715491: Average global foreground Dice: [0.9443, 0.9372, 0.9219] +2022-05-14 10:51:46.715579: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 10:51:47.116537: lr: 0.000282 +2022-05-14 10:51:47.116738: saving scheduled checkpoint file... +2022-05-14 10:51:47.165892: saving checkpoint... +2022-05-14 10:51:47.961430: done, saving took 0.84 seconds +2022-05-14 10:51:47.981422: done +2022-05-14 10:51:48.022807: saving checkpoint... +2022-05-14 10:51:48.796338: done, saving took 0.81 seconds +2022-05-14 10:52:09.065638: This epoch took 483.751603 s + +2022-05-14 10:52:09.065938: +epoch: 48 +2022-05-14 10:59:25.895748: train loss : 0.0515 +2022-05-14 10:59:50.927434: validation loss: 0.0473 +2022-05-14 10:59:50.928303: Average global foreground Dice: [0.9484, 0.9384, 0.9241] +2022-05-14 10:59:50.928389: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 10:59:51.330499: lr: 0.000281 +2022-05-14 10:59:51.330688: saving scheduled checkpoint file... +2022-05-14 10:59:51.376816: saving checkpoint... +2022-05-14 10:59:52.172287: done, saving took 0.84 seconds +2022-05-14 10:59:52.190266: done +2022-05-14 10:59:52.226578: saving checkpoint... +2022-05-14 10:59:52.991404: done, saving took 0.80 seconds +2022-05-14 11:00:24.360062: This epoch took 495.294022 s + +2022-05-14 11:00:24.360347: +epoch: 49 +2022-05-14 11:07:40.769397: train loss : 0.0505 +2022-05-14 11:08:05.815184: validation loss: 0.0509 +2022-05-14 11:08:05.815843: Average global foreground Dice: [0.95, 0.9376, 0.922] +2022-05-14 11:08:05.815931: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 11:08:06.217956: lr: 0.00028 +2022-05-14 11:08:06.218137: saving scheduled checkpoint file... +2022-05-14 11:08:06.264487: saving checkpoint... +2022-05-14 11:08:07.066524: done, saving took 0.85 seconds +2022-05-14 11:08:07.084983: done +2022-05-14 11:08:07.121529: saving checkpoint... +2022-05-14 11:08:07.871802: done, saving took 0.79 seconds +2022-05-14 11:08:24.063660: This epoch took 479.703210 s + +2022-05-14 11:08:24.063945: +epoch: 50 +2022-05-14 11:15:41.235873: train loss : 0.0525 +2022-05-14 11:16:06.305725: validation loss: 0.0543 +2022-05-14 11:16:06.306468: Average global foreground Dice: [0.9453, 0.9383, 0.929] +2022-05-14 11:16:06.306554: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 11:16:06.711572: lr: 0.000279 +2022-05-14 11:16:06.711762: saving scheduled checkpoint file... +2022-05-14 11:16:06.758167: saving checkpoint... +2022-05-14 11:16:07.554892: done, saving took 0.84 seconds +2022-05-14 11:16:07.572872: done +2022-05-14 11:16:07.608892: saving checkpoint... +2022-05-14 11:16:08.374214: done, saving took 0.80 seconds +2022-05-14 11:16:23.505456: This epoch took 479.441413 s + +2022-05-14 11:16:23.505730: +epoch: 51 +2022-05-14 11:23:40.127583: train loss : 0.0509 +2022-05-14 11:24:05.197427: validation loss: 0.0491 +2022-05-14 11:24:05.198068: Average global foreground Dice: [0.9485, 0.9405, 0.9282] +2022-05-14 11:24:05.198155: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 11:24:05.599713: lr: 0.000278 +2022-05-14 11:24:05.599919: saving scheduled checkpoint file... +2022-05-14 11:24:05.646518: saving checkpoint... +2022-05-14 11:24:06.356779: done, saving took 0.76 seconds +2022-05-14 11:24:06.374423: done +2022-05-14 11:24:06.408163: saving checkpoint... +2022-05-14 11:24:07.099368: done, saving took 0.72 seconds +2022-05-14 11:24:26.793154: This epoch took 483.287302 s + +2022-05-14 11:24:26.793450: +epoch: 52 +2022-05-14 11:31:42.532845: train loss : 0.0488 +2022-05-14 11:32:07.530963: validation loss: 0.0474 +2022-05-14 11:32:07.531605: Average global foreground Dice: [0.9481, 0.9291, 0.9254] +2022-05-14 11:32:07.531694: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 11:32:07.933429: lr: 0.000277 +2022-05-14 11:32:07.933694: saving scheduled checkpoint file... +2022-05-14 11:32:07.980870: saving checkpoint... +2022-05-14 11:32:08.780127: done, saving took 0.85 seconds +2022-05-14 11:32:08.798281: done +2022-05-14 11:32:29.697152: This epoch took 482.903602 s + +2022-05-14 11:32:29.697422: +epoch: 53 +2022-05-14 11:39:45.999619: train loss : 0.0524 +2022-05-14 11:40:11.015318: validation loss: 0.0536 +2022-05-14 11:40:11.015915: Average global foreground Dice: [0.9501, 0.9338, 0.9206] +2022-05-14 11:40:11.016002: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 11:40:11.420074: lr: 0.000276 +2022-05-14 11:40:11.420268: saving scheduled checkpoint file... +2022-05-14 11:40:11.466733: saving checkpoint... +2022-05-14 11:40:12.265772: done, saving took 0.85 seconds +2022-05-14 11:40:12.283924: done +2022-05-14 11:40:29.399230: This epoch took 479.701710 s + +2022-05-14 11:40:29.399506: +epoch: 54 +2022-05-14 11:47:46.338642: train loss : 0.0510 +2022-05-14 11:48:11.447127: validation loss: 0.0510 +2022-05-14 11:48:11.447891: Average global foreground Dice: [0.95, 0.9361, 0.9303] +2022-05-14 11:48:11.447978: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 11:48:11.851034: lr: 0.000275 +2022-05-14 11:48:11.851230: saving scheduled checkpoint file... +2022-05-14 11:48:11.897654: saving checkpoint... +2022-05-14 11:48:12.712813: done, saving took 0.86 seconds +2022-05-14 11:48:12.731323: done +2022-05-14 11:48:12.768013: saving checkpoint... +2022-05-14 11:48:13.533347: done, saving took 0.80 seconds +2022-05-14 11:48:31.060464: This epoch took 481.660859 s + +2022-05-14 11:48:31.060739: +epoch: 55 +2022-05-14 11:55:48.187547: train loss : 0.0457 +2022-05-14 11:56:13.248245: validation loss: 0.0549 +2022-05-14 11:56:13.248924: Average global foreground Dice: [0.9467, 0.9295, 0.9232] +2022-05-14 11:56:13.249011: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 11:56:13.650577: lr: 0.000274 +2022-05-14 11:56:13.650778: saving scheduled checkpoint file... +2022-05-14 11:56:13.697161: saving checkpoint... +2022-05-14 11:56:14.491910: done, saving took 0.84 seconds +2022-05-14 11:56:14.510259: done +2022-05-14 11:56:33.491734: This epoch took 482.430874 s + +2022-05-14 11:56:33.496289: +epoch: 56 +2022-05-14 12:03:50.152731: train loss : 0.0527 +2022-05-14 12:04:15.158992: validation loss: 0.0522 +2022-05-14 12:04:15.159621: Average global foreground Dice: [0.9488, 0.9417, 0.924] +2022-05-14 12:04:15.159707: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 12:04:15.562023: lr: 0.000272 +2022-05-14 12:04:15.562220: saving scheduled checkpoint file... +2022-05-14 12:04:15.608904: saving checkpoint... +2022-05-14 12:04:16.398117: done, saving took 0.84 seconds +2022-05-14 12:04:16.418181: done +2022-05-14 12:04:16.459261: saving checkpoint... +2022-05-14 12:04:17.232091: done, saving took 0.81 seconds +2022-05-14 12:04:33.171213: This epoch took 479.674818 s + +2022-05-14 12:04:33.171498: +epoch: 57 +2022-05-14 12:11:50.051971: train loss : 0.0484 +2022-05-14 12:12:15.145199: validation loss: 0.0652 +2022-05-14 12:12:15.145838: Average global foreground Dice: [0.9434, 0.9328, 0.9209] +2022-05-14 12:12:15.145924: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 12:12:15.555440: lr: 0.000271 +2022-05-14 12:12:15.555645: saving scheduled checkpoint file... +2022-05-14 12:12:15.602165: saving checkpoint... +2022-05-14 12:12:16.388486: done, saving took 0.83 seconds +2022-05-14 12:12:16.409074: done +2022-05-14 12:12:32.327603: This epoch took 479.156010 s + +2022-05-14 12:12:32.327875: +epoch: 58 +2022-05-14 12:19:49.840179: train loss : 0.0504 +2022-05-14 12:20:14.888983: validation loss: 0.0492 +2022-05-14 12:20:14.889612: Average global foreground Dice: [0.9479, 0.9366, 0.9296] +2022-05-14 12:20:14.889699: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 12:20:15.293953: lr: 0.00027 +2022-05-14 12:20:15.294152: saving scheduled checkpoint file... +2022-05-14 12:20:15.340832: saving checkpoint... +2022-05-14 12:20:16.132562: done, saving took 0.84 seconds +2022-05-14 12:20:16.152698: done +2022-05-14 12:20:16.193752: saving checkpoint... +2022-05-14 12:20:16.979574: done, saving took 0.83 seconds +2022-05-14 12:20:34.692004: This epoch took 482.364017 s + +2022-05-14 12:20:34.692342: +epoch: 59 +2022-05-14 12:27:51.905892: train loss : 0.0505 +2022-05-14 12:28:16.969378: validation loss: 0.0562 +2022-05-14 12:28:16.970024: Average global foreground Dice: [0.9475, 0.9339, 0.9207] +2022-05-14 12:28:16.970112: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 12:28:17.375118: lr: 0.000269 +2022-05-14 12:28:17.375305: saving scheduled checkpoint file... +2022-05-14 12:28:17.421547: saving checkpoint... +2022-05-14 12:28:18.213092: done, saving took 0.84 seconds +2022-05-14 12:28:18.231436: done +2022-05-14 12:28:41.500587: This epoch took 486.808121 s + +2022-05-14 12:28:41.500975: +epoch: 60 +2022-05-14 12:35:58.260606: train loss : 0.0537 +2022-05-14 12:36:23.326588: validation loss: 0.0551 +2022-05-14 12:36:23.327200: Average global foreground Dice: [0.9459, 0.936, 0.9211] +2022-05-14 12:36:23.327285: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 12:36:23.736533: lr: 0.000268 +2022-05-14 12:36:23.736713: saving scheduled checkpoint file... +2022-05-14 12:36:23.783858: saving checkpoint... +2022-05-14 12:36:24.593248: done, saving took 0.86 seconds +2022-05-14 12:36:24.611257: done +2022-05-14 12:36:47.048033: This epoch took 485.546946 s + +2022-05-14 12:36:47.048431: +epoch: 61 +2022-05-14 12:44:04.237739: train loss : 0.0508 +2022-05-14 12:44:29.351750: validation loss: 0.0621 +2022-05-14 12:44:29.352445: Average global foreground Dice: [0.9422, 0.9289, 0.9243] +2022-05-14 12:44:29.352531: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 12:44:29.755594: lr: 0.000267 +2022-05-14 12:44:29.755803: saving scheduled checkpoint file... +2022-05-14 12:44:29.802555: saving checkpoint... +2022-05-14 12:44:30.596023: done, saving took 0.84 seconds +2022-05-14 12:44:30.613973: done +2022-05-14 12:45:34.438482: This epoch took 527.389949 s + +2022-05-14 12:45:34.438789: +epoch: 62 +2022-05-14 12:52:51.101916: train loss : 0.0506 +2022-05-14 12:53:16.143583: validation loss: 0.0623 +2022-05-14 12:53:16.144283: Average global foreground Dice: [0.9452, 0.9282, 0.9248] +2022-05-14 12:53:16.144369: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 12:53:16.549357: lr: 0.000265 +2022-05-14 12:53:16.549553: saving scheduled checkpoint file... +2022-05-14 12:53:16.596534: saving checkpoint... +2022-05-14 12:53:17.304192: done, saving took 0.75 seconds +2022-05-14 12:53:17.323347: done +2022-05-14 12:53:35.227108: This epoch took 480.788212 s + +2022-05-14 12:53:35.227402: +epoch: 63 +2022-05-14 13:00:51.630589: train loss : 0.0485 +2022-05-14 13:01:16.631651: validation loss: 0.0563 +2022-05-14 13:01:16.632301: Average global foreground Dice: [0.9459, 0.937, 0.9251] +2022-05-14 13:01:16.632386: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 13:01:17.037050: lr: 0.000264 +2022-05-14 13:01:17.037246: saving scheduled checkpoint file... +2022-05-14 13:01:17.083576: saving checkpoint... +2022-05-14 13:01:17.777046: done, saving took 0.74 seconds +2022-05-14 13:01:17.795204: done +2022-05-14 13:01:44.224283: This epoch took 488.996778 s + +2022-05-14 13:01:44.224567: +epoch: 64 +2022-05-14 13:09:01.105247: train loss : 0.0487 +2022-05-14 13:09:26.100133: validation loss: 0.0490 +2022-05-14 13:09:26.100830: Average global foreground Dice: [0.9477, 0.9268, 0.9216] +2022-05-14 13:09:26.100918: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 13:09:26.509105: lr: 0.000263 +2022-05-14 13:09:26.509313: saving scheduled checkpoint file... +2022-05-14 13:09:26.556577: saving checkpoint... +2022-05-14 13:09:27.267720: done, saving took 0.76 seconds +2022-05-14 13:09:27.285578: done +2022-05-14 13:09:46.077051: This epoch took 481.852374 s + +2022-05-14 13:09:46.077389: +epoch: 65 +2022-05-14 13:17:03.434333: train loss : 0.0486 +2022-05-14 13:17:28.546785: validation loss: 0.0492 +2022-05-14 13:17:28.547428: Average global foreground Dice: [0.9471, 0.9341, 0.9277] +2022-05-14 13:17:28.547514: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 13:17:28.950339: lr: 0.000261 +2022-05-14 13:17:28.950535: saving scheduled checkpoint file... +2022-05-14 13:17:29.001323: saving checkpoint... +2022-05-14 13:17:29.762752: done, saving took 0.81 seconds +2022-05-14 13:17:29.780472: done +2022-05-14 13:17:46.808388: This epoch took 480.730899 s + +2022-05-14 13:17:46.808671: +epoch: 66 +2022-05-14 13:25:04.065130: train loss : 0.0503 +2022-05-14 13:25:29.107133: validation loss: 0.0517 +2022-05-14 13:25:29.107792: Average global foreground Dice: [0.9498, 0.9367, 0.9238] +2022-05-14 13:25:29.107878: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 13:25:29.515916: lr: 0.00026 +2022-05-14 13:25:29.516109: saving scheduled checkpoint file... +2022-05-14 13:25:29.562532: saving checkpoint... +2022-05-14 13:25:30.271166: done, saving took 0.75 seconds +2022-05-14 13:25:30.288761: done +2022-05-14 13:25:49.587086: This epoch took 482.778312 s + +2022-05-14 13:25:49.587404: +epoch: 67 +2022-05-14 13:33:06.292819: train loss : 0.0486 +2022-05-14 13:33:31.260273: validation loss: 0.0579 +2022-05-14 13:33:31.260961: Average global foreground Dice: [0.9469, 0.937, 0.9238] +2022-05-14 13:33:31.261047: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 13:33:31.666424: lr: 0.000259 +2022-05-14 13:33:31.666627: saving scheduled checkpoint file... +2022-05-14 13:33:31.713405: saving checkpoint... +2022-05-14 13:33:32.441174: done, saving took 0.77 seconds +2022-05-14 13:33:32.458797: done +2022-05-14 13:33:54.564132: This epoch took 484.976630 s + +2022-05-14 13:33:54.564415: +epoch: 68 +2022-05-14 13:41:11.774554: train loss : 0.0473 +2022-05-14 13:41:36.788803: validation loss: 0.0527 +2022-05-14 13:41:36.789462: Average global foreground Dice: [0.9489, 0.9428, 0.9275] +2022-05-14 13:41:36.789544: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 13:41:37.196276: lr: 0.000257 +2022-05-14 13:41:37.196465: saving scheduled checkpoint file... +2022-05-14 13:41:37.242194: saving checkpoint... +2022-05-14 13:41:38.032870: done, saving took 0.84 seconds +2022-05-14 13:41:38.050784: done +2022-05-14 13:41:38.086503: saving checkpoint... +2022-05-14 13:41:38.857681: done, saving took 0.81 seconds +2022-05-14 13:41:56.923212: This epoch took 482.358692 s + +2022-05-14 13:41:56.923501: +epoch: 69 +2022-05-14 13:49:13.786942: train loss : 0.0464 +2022-05-14 13:49:38.881817: validation loss: 0.0415 +2022-05-14 13:49:38.882434: Average global foreground Dice: [0.9454, 0.941, 0.9232] +2022-05-14 13:49:38.882526: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 13:49:39.290560: lr: 0.000256 +2022-05-14 13:49:39.290763: saving scheduled checkpoint file... +2022-05-14 13:49:39.337464: saving checkpoint... +2022-05-14 13:49:40.127090: done, saving took 0.84 seconds +2022-05-14 13:49:40.145494: done +2022-05-14 13:49:40.182212: saving checkpoint... +2022-05-14 13:49:40.957411: done, saving took 0.81 seconds +2022-05-14 13:49:55.017400: This epoch took 478.093780 s + +2022-05-14 13:49:55.017789: +epoch: 70 +2022-05-14 13:57:11.842987: train loss : 0.0491 +2022-05-14 13:57:36.888284: validation loss: 0.0542 +2022-05-14 13:57:36.888898: Average global foreground Dice: [0.9491, 0.9383, 0.9273] +2022-05-14 13:57:36.888984: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 13:57:37.297204: lr: 0.000255 +2022-05-14 13:57:37.297399: saving scheduled checkpoint file... +2022-05-14 13:57:37.343780: saving checkpoint... +2022-05-14 13:57:38.076583: done, saving took 0.78 seconds +2022-05-14 13:57:38.094717: done +2022-05-14 13:57:38.128638: saving checkpoint... +2022-05-14 13:57:38.873179: done, saving took 0.78 seconds +2022-05-14 13:57:55.386641: This epoch took 480.368748 s + +2022-05-14 13:57:55.386940: +epoch: 71 +2022-05-14 14:05:12.068870: train loss : 0.0485 +2022-05-14 14:05:37.091094: validation loss: 0.0517 +2022-05-14 14:05:37.091805: Average global foreground Dice: [0.9479, 0.9399, 0.9275] +2022-05-14 14:05:37.091893: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 14:05:37.497932: lr: 0.000253 +2022-05-14 14:05:37.498121: saving scheduled checkpoint file... +2022-05-14 14:05:37.544245: saving checkpoint... +2022-05-14 14:05:38.342051: done, saving took 0.84 seconds +2022-05-14 14:05:38.360499: done +2022-05-14 14:05:38.397162: saving checkpoint... +2022-05-14 14:05:39.177499: done, saving took 0.82 seconds +2022-05-14 14:05:55.195325: This epoch took 479.808278 s + +2022-05-14 14:05:55.195628: +epoch: 72 +2022-05-14 14:13:11.170667: train loss : 0.0514 +2022-05-14 14:13:36.293930: validation loss: 0.0565 +2022-05-14 14:13:36.294586: Average global foreground Dice: [0.9472, 0.9255, 0.9258] +2022-05-14 14:13:36.294672: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 14:13:36.701643: lr: 0.000252 +2022-05-14 14:13:36.701848: saving scheduled checkpoint file... +2022-05-14 14:13:36.748197: saving checkpoint... +2022-05-14 14:13:37.538530: done, saving took 0.84 seconds +2022-05-14 14:13:37.556498: done +2022-05-14 14:13:59.106209: This epoch took 483.910477 s + +2022-05-14 14:13:59.106503: +epoch: 73 +2022-05-14 14:21:15.715818: train loss : 0.0480 +2022-05-14 14:21:40.783736: validation loss: 0.0536 +2022-05-14 14:21:40.784393: Average global foreground Dice: [0.9458, 0.9372, 0.9253] +2022-05-14 14:21:40.784477: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 14:21:41.190716: lr: 0.00025 +2022-05-14 14:21:41.190919: saving scheduled checkpoint file... +2022-05-14 14:21:41.237929: saving checkpoint... +2022-05-14 14:21:42.035247: done, saving took 0.84 seconds +2022-05-14 14:21:42.054717: done +2022-05-14 14:21:59.760798: This epoch took 480.654174 s + +2022-05-14 14:21:59.761169: +epoch: 74 +2022-05-14 14:29:16.546969: train loss : 0.0480 +2022-05-14 14:29:41.579836: validation loss: 0.0488 +2022-05-14 14:29:41.580424: Average global foreground Dice: [0.9495, 0.9356, 0.9243] +2022-05-14 14:29:41.580507: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 14:29:41.990507: lr: 0.000249 +2022-05-14 14:29:41.990697: saving scheduled checkpoint file... +2022-05-14 14:29:42.036742: saving checkpoint... +2022-05-14 14:29:42.833151: done, saving took 0.84 seconds +2022-05-14 14:29:42.852339: done +2022-05-14 14:30:03.994743: This epoch took 484.233462 s + +2022-05-14 14:30:03.995066: +epoch: 75 +2022-05-14 14:37:20.937859: train loss : 0.0423 +2022-05-14 14:37:45.995762: validation loss: 0.0415 +2022-05-14 14:37:45.996390: Average global foreground Dice: [0.9477, 0.9331, 0.9257] +2022-05-14 14:37:45.996474: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 14:37:46.404310: lr: 0.000247 +2022-05-14 14:37:46.404508: saving scheduled checkpoint file... +2022-05-14 14:37:46.450918: saving checkpoint... +2022-05-14 14:37:47.243715: done, saving took 0.84 seconds +2022-05-14 14:37:47.261545: done +2022-05-14 14:38:05.961200: This epoch took 481.966031 s + +2022-05-14 14:38:05.961520: +epoch: 76 +2022-05-14 14:45:22.699585: train loss : 0.0449 +2022-05-14 14:45:47.781787: validation loss: 0.0572 +2022-05-14 14:45:47.782399: Average global foreground Dice: [0.9504, 0.943, 0.9294] +2022-05-14 14:45:47.782483: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 14:45:48.194160: lr: 0.000246 +2022-05-14 14:45:48.194353: saving scheduled checkpoint file... +2022-05-14 14:45:48.241367: saving checkpoint... +2022-05-14 14:45:49.040379: done, saving took 0.85 seconds +2022-05-14 14:45:49.060683: done +2022-05-14 14:45:49.102306: saving checkpoint... +2022-05-14 14:45:49.875440: done, saving took 0.81 seconds +2022-05-14 14:46:08.132240: This epoch took 482.170621 s + +2022-05-14 14:46:08.132551: +epoch: 77 +2022-05-14 14:53:25.334657: train loss : 0.0464 +2022-05-14 14:53:50.371830: validation loss: 0.0601 +2022-05-14 14:53:50.372488: Average global foreground Dice: [0.9453, 0.9364, 0.9152] +2022-05-14 14:53:50.372575: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 14:53:50.785430: lr: 0.000244 +2022-05-14 14:53:50.785619: saving scheduled checkpoint file... +2022-05-14 14:53:50.832718: saving checkpoint... +2022-05-14 14:53:51.630281: done, saving took 0.84 seconds +2022-05-14 14:53:51.648164: done +2022-05-14 14:54:08.592257: This epoch took 480.459588 s + +2022-05-14 14:54:08.592631: +epoch: 78 +2022-05-14 15:01:25.314291: train loss : 0.0498 +2022-05-14 15:01:50.388975: validation loss: 0.0537 +2022-05-14 15:01:50.389615: Average global foreground Dice: [0.948, 0.9436, 0.928] +2022-05-14 15:01:50.389704: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 15:01:50.801294: lr: 0.000243 +2022-05-14 15:01:50.801501: saving scheduled checkpoint file... +2022-05-14 15:01:50.849161: saving checkpoint... +2022-05-14 15:01:51.655019: done, saving took 0.85 seconds +2022-05-14 15:01:51.672871: done +2022-05-14 15:02:06.225644: This epoch took 477.632905 s + +2022-05-14 15:02:06.225995: +epoch: 79 +2022-05-14 15:09:23.036924: train loss : 0.0486 +2022-05-14 15:09:48.003627: validation loss: 0.0586 +2022-05-14 15:09:48.004281: Average global foreground Dice: [0.9501, 0.9417, 0.9286] +2022-05-14 15:09:48.004368: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 15:09:48.414195: lr: 0.000241 +2022-05-14 15:09:48.414405: saving scheduled checkpoint file... +2022-05-14 15:09:48.461185: saving checkpoint... +2022-05-14 15:09:49.268605: done, saving took 0.85 seconds +2022-05-14 15:09:49.286739: done +2022-05-14 15:09:49.322946: saving checkpoint... +2022-05-14 15:09:50.089761: done, saving took 0.80 seconds +2022-05-14 15:10:25.258225: This epoch took 499.032128 s + +2022-05-14 15:10:25.258519: +epoch: 80 +2022-05-14 15:17:41.625415: train loss : 0.0456 +2022-05-14 15:18:06.626805: validation loss: 0.0510 +2022-05-14 15:18:06.627476: Average global foreground Dice: [0.9469, 0.9412, 0.9267] +2022-05-14 15:18:06.627562: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 15:18:07.036651: lr: 0.00024 +2022-05-14 15:18:07.036854: saving scheduled checkpoint file... +2022-05-14 15:18:07.082299: saving checkpoint... +2022-05-14 15:18:07.880396: done, saving took 0.84 seconds +2022-05-14 15:18:07.898316: done +2022-05-14 15:18:07.934994: saving checkpoint... +2022-05-14 15:18:08.725908: done, saving took 0.83 seconds +2022-05-14 15:18:25.680133: This epoch took 480.421494 s + +2022-05-14 15:18:25.680487: +epoch: 81 +2022-05-14 15:25:42.582800: train loss : 0.0442 +2022-05-14 15:26:07.628408: validation loss: 0.0498 +2022-05-14 15:26:07.629041: Average global foreground Dice: [0.9484, 0.9379, 0.9259] +2022-05-14 15:26:07.629126: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 15:26:08.037165: lr: 0.000238 +2022-05-14 15:26:08.037355: saving scheduled checkpoint file... +2022-05-14 15:26:08.083751: saving checkpoint... +2022-05-14 15:26:08.855254: done, saving took 0.82 seconds +2022-05-14 15:26:08.873104: done +2022-05-14 15:26:08.909306: saving checkpoint... +2022-05-14 15:26:09.696094: done, saving took 0.82 seconds +2022-05-14 15:26:27.373111: This epoch took 481.692501 s + +2022-05-14 15:26:27.373429: +epoch: 82 +2022-05-14 15:33:43.852476: train loss : 0.0482 +2022-05-14 15:34:08.850779: validation loss: 0.0456 +2022-05-14 15:34:08.851466: Average global foreground Dice: [0.9507, 0.938, 0.9279] +2022-05-14 15:34:08.851552: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 15:34:09.256575: lr: 0.000237 +2022-05-14 15:34:09.256778: saving scheduled checkpoint file... +2022-05-14 15:34:09.302988: saving checkpoint... +2022-05-14 15:34:10.106227: done, saving took 0.85 seconds +2022-05-14 15:34:10.124552: done +2022-05-14 15:34:10.160520: saving checkpoint... +2022-05-14 15:34:10.949723: done, saving took 0.82 seconds +2022-05-14 15:34:31.665858: This epoch took 484.292329 s + +2022-05-14 15:34:31.666125: +epoch: 83 +2022-05-14 15:41:47.972713: train loss : 0.0453 +2022-05-14 15:42:13.037799: validation loss: 0.0456 +2022-05-14 15:42:13.038476: Average global foreground Dice: [0.9473, 0.9439, 0.926] +2022-05-14 15:42:13.038561: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 15:42:13.445993: lr: 0.000235 +2022-05-14 15:42:13.446249: saving scheduled checkpoint file... +2022-05-14 15:42:13.492553: saving checkpoint... +2022-05-14 15:42:14.291160: done, saving took 0.84 seconds +2022-05-14 15:42:14.309597: done +2022-05-14 15:42:14.345834: saving checkpoint... +2022-05-14 15:42:15.118478: done, saving took 0.81 seconds +2022-05-14 15:42:35.399677: This epoch took 483.733436 s + +2022-05-14 15:42:35.400042: +epoch: 84 +2022-05-14 15:49:51.676176: train loss : 0.0434 +2022-05-14 15:50:16.719812: validation loss: 0.0543 +2022-05-14 15:50:16.720546: Average global foreground Dice: [0.9469, 0.9345, 0.9211] +2022-05-14 15:50:16.720630: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 15:50:17.122133: lr: 0.000233 +2022-05-14 15:50:17.122330: saving scheduled checkpoint file... +2022-05-14 15:50:17.168538: saving checkpoint... +2022-05-14 15:50:17.968487: done, saving took 0.85 seconds +2022-05-14 15:50:17.986568: done +2022-05-14 15:50:36.822254: This epoch took 481.422107 s + +2022-05-14 15:50:36.822560: +epoch: 85 +2022-05-14 15:57:53.518159: train loss : 0.0434 +2022-05-14 15:58:18.596649: validation loss: 0.0590 +2022-05-14 15:58:18.597303: Average global foreground Dice: [0.9484, 0.941, 0.9271] +2022-05-14 15:58:18.597389: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 15:58:19.006099: lr: 0.000232 +2022-05-14 15:58:19.006297: saving scheduled checkpoint file... +2022-05-14 15:58:19.054116: saving checkpoint... +2022-05-14 15:58:19.868351: done, saving took 0.86 seconds +2022-05-14 15:58:19.887057: done +2022-05-14 15:59:02.303090: This epoch took 505.480436 s + +2022-05-14 15:59:02.303381: +epoch: 86 +2022-05-14 16:06:18.993953: train loss : 0.0465 +2022-05-14 16:06:43.980681: validation loss: 0.0534 +2022-05-14 16:06:43.981398: Average global foreground Dice: [0.9466, 0.9408, 0.9247] +2022-05-14 16:06:43.981485: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 16:06:44.387093: lr: 0.00023 +2022-05-14 16:06:44.387298: saving scheduled checkpoint file... +2022-05-14 16:06:44.433473: saving checkpoint... +2022-05-14 16:06:45.229412: done, saving took 0.84 seconds +2022-05-14 16:06:45.247712: done +2022-05-14 16:07:03.529322: This epoch took 481.225840 s + +2022-05-14 16:07:03.529589: +epoch: 87 +2022-05-14 16:14:19.660372: train loss : 0.0471 +2022-05-14 16:14:44.635872: validation loss: 0.0438 +2022-05-14 16:14:44.636456: Average global foreground Dice: [0.9463, 0.942, 0.9272] +2022-05-14 16:14:44.636549: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 16:14:45.040957: lr: 0.000228 +2022-05-14 16:14:45.041143: saving scheduled checkpoint file... +2022-05-14 16:14:45.087899: saving checkpoint... +2022-05-14 16:14:45.889922: done, saving took 0.85 seconds +2022-05-14 16:14:45.908041: done +2022-05-14 16:14:45.944662: saving checkpoint... +2022-05-14 16:14:46.723861: done, saving took 0.82 seconds +2022-05-14 16:15:08.145724: This epoch took 484.616035 s + +2022-05-14 16:15:08.146031: +epoch: 88 +2022-05-14 16:22:24.935351: train loss : 0.0452 +2022-05-14 16:22:50.076181: validation loss: 0.0476 +2022-05-14 16:22:50.076843: Average global foreground Dice: [0.9484, 0.9479, 0.9269] +2022-05-14 16:22:50.076933: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 16:22:50.484047: lr: 0.000227 +2022-05-14 16:22:50.484227: saving scheduled checkpoint file... +2022-05-14 16:22:50.530889: saving checkpoint... +2022-05-14 16:22:51.330705: done, saving took 0.85 seconds +2022-05-14 16:22:51.352881: done +2022-05-14 16:22:51.396585: saving checkpoint... +2022-05-14 16:22:52.168271: done, saving took 0.82 seconds +2022-05-14 16:23:10.474334: This epoch took 482.328207 s + +2022-05-14 16:23:10.474605: +epoch: 89 +2022-05-14 16:30:27.280281: train loss : 0.0453 +2022-05-14 16:30:52.344904: validation loss: 0.0618 +2022-05-14 16:30:52.345507: Average global foreground Dice: [0.9487, 0.9416, 0.9277] +2022-05-14 16:30:52.345727: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 16:30:52.750517: lr: 0.000225 +2022-05-14 16:30:52.750720: saving scheduled checkpoint file... +2022-05-14 16:30:52.797313: saving checkpoint... +2022-05-14 16:30:53.612807: done, saving took 0.86 seconds +2022-05-14 16:30:53.630749: done +2022-05-14 16:30:53.666737: saving checkpoint... +2022-05-14 16:30:54.428051: done, saving took 0.80 seconds +2022-05-14 16:31:16.350833: This epoch took 485.876126 s + +2022-05-14 16:31:16.351115: +epoch: 90 +2022-05-14 16:38:32.935409: train loss : 0.0424 +2022-05-14 16:38:57.963707: validation loss: 0.0579 +2022-05-14 16:38:57.964354: Average global foreground Dice: [0.9406, 0.9245, 0.9158] +2022-05-14 16:38:57.964440: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 16:38:58.371185: lr: 0.000223 +2022-05-14 16:38:58.371620: saving scheduled checkpoint file... +2022-05-14 16:38:58.418388: saving checkpoint... +2022-05-14 16:38:59.241405: done, saving took 0.87 seconds +2022-05-14 16:38:59.260138: done +2022-05-14 16:39:41.357797: This epoch took 505.006564 s + +2022-05-14 16:39:41.358073: +epoch: 91 +2022-05-14 16:46:58.089940: train loss : 0.0447 +2022-05-14 16:47:23.119437: validation loss: 0.0492 +2022-05-14 16:47:23.120060: Average global foreground Dice: [0.9483, 0.9398, 0.928] +2022-05-14 16:47:23.120145: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 16:47:23.523481: lr: 0.000222 +2022-05-14 16:47:23.523680: saving scheduled checkpoint file... +2022-05-14 16:47:23.571069: saving checkpoint... +2022-05-14 16:47:24.283151: done, saving took 0.76 seconds +2022-05-14 16:47:24.301173: done +2022-05-14 16:47:42.018726: This epoch took 480.660551 s + +2022-05-14 16:47:42.019062: +epoch: 92 +2022-05-14 16:54:58.165035: train loss : 0.0438 +2022-05-14 16:55:23.241873: validation loss: 0.0481 +2022-05-14 16:55:23.242596: Average global foreground Dice: [0.9449, 0.9453, 0.9256] +2022-05-14 16:55:23.242683: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 16:55:23.648872: lr: 0.00022 +2022-05-14 16:55:23.649064: saving scheduled checkpoint file... +2022-05-14 16:55:23.695612: saving checkpoint... +2022-05-14 16:55:24.425632: done, saving took 0.78 seconds +2022-05-14 16:55:24.462529: done +2022-05-14 16:55:43.433362: This epoch took 481.414197 s + +2022-05-14 16:55:43.433652: +epoch: 93 +2022-05-14 17:02:59.975917: train loss : 0.0456 +2022-05-14 17:03:25.028884: validation loss: 0.0480 +2022-05-14 17:03:25.029645: Average global foreground Dice: [0.9476, 0.9366, 0.9255] +2022-05-14 17:03:25.029733: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 17:03:25.438997: lr: 0.000218 +2022-05-14 17:03:25.439199: saving scheduled checkpoint file... +2022-05-14 17:03:25.485387: saving checkpoint... +2022-05-14 17:03:26.207851: done, saving took 0.77 seconds +2022-05-14 17:03:26.227374: done +2022-05-14 17:03:42.890342: This epoch took 479.456586 s + +2022-05-14 17:03:42.890639: +epoch: 94 +2022-05-14 17:10:59.569948: train loss : 0.0443 +2022-05-14 17:11:24.607199: validation loss: 0.0572 +2022-05-14 17:11:24.607833: Average global foreground Dice: [0.9502, 0.9402, 0.9307] +2022-05-14 17:11:24.607919: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 17:11:25.014446: lr: 0.000216 +2022-05-14 17:11:25.014638: saving scheduled checkpoint file... +2022-05-14 17:11:25.061245: saving checkpoint... +2022-05-14 17:11:25.772548: done, saving took 0.76 seconds +2022-05-14 17:11:25.791708: done +2022-05-14 17:11:40.455314: This epoch took 477.564569 s + +2022-05-14 17:11:40.455596: +epoch: 95 +2022-05-14 17:18:56.531961: train loss : 0.0460 +2022-05-14 17:19:21.658290: validation loss: 0.0601 +2022-05-14 17:19:21.658925: Average global foreground Dice: [0.9498, 0.9426, 0.9279] +2022-05-14 17:19:21.659028: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 17:19:22.062709: lr: 0.000215 +2022-05-14 17:19:22.062910: saving scheduled checkpoint file... +2022-05-14 17:19:22.109106: saving checkpoint... +2022-05-14 17:19:22.837612: done, saving took 0.77 seconds +2022-05-14 17:19:22.854438: done +2022-05-14 17:19:42.102194: This epoch took 481.646491 s + +2022-05-14 17:19:42.102453: +epoch: 96 +2022-05-14 17:26:58.562485: train loss : 0.0434 +2022-05-14 17:27:23.531393: validation loss: 0.0579 +2022-05-14 17:27:23.532173: Average global foreground Dice: [0.9499, 0.9363, 0.9246] +2022-05-14 17:27:23.532257: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 17:27:23.940233: lr: 0.000213 +2022-05-14 17:27:23.940424: saving scheduled checkpoint file... +2022-05-14 17:27:23.986921: saving checkpoint... +2022-05-14 17:27:24.723968: done, saving took 0.78 seconds +2022-05-14 17:27:24.760190: done +2022-05-14 17:27:46.979264: This epoch took 484.876711 s + +2022-05-14 17:27:46.979546: +epoch: 97 +2022-05-14 17:35:03.208236: train loss : 0.0439 +2022-05-14 17:35:28.188917: validation loss: 0.0432 +2022-05-14 17:35:28.189559: Average global foreground Dice: [0.9487, 0.9459, 0.9312] +2022-05-14 17:35:28.189644: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 17:35:28.617378: lr: 0.000211 +2022-05-14 17:35:28.617576: saving scheduled checkpoint file... +2022-05-14 17:35:28.666094: saving checkpoint... +2022-05-14 17:35:29.376898: done, saving took 0.76 seconds +2022-05-14 17:35:29.394585: done +2022-05-14 17:35:29.428867: saving checkpoint... +2022-05-14 17:35:30.103308: done, saving took 0.71 seconds +2022-05-14 17:35:49.098874: This epoch took 482.119226 s + +2022-05-14 17:35:49.099162: +epoch: 98 +2022-05-14 17:43:05.675593: train loss : 0.0452 +2022-05-14 17:43:30.666074: validation loss: 0.0461 +2022-05-14 17:43:30.666711: Average global foreground Dice: [0.9455, 0.9399, 0.9221] +2022-05-14 17:43:30.666806: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 17:43:31.076311: lr: 0.000209 +2022-05-14 17:43:31.076504: saving scheduled checkpoint file... +2022-05-14 17:43:31.123079: saving checkpoint... +2022-05-14 17:43:31.889466: done, saving took 0.81 seconds +2022-05-14 17:43:31.907997: done +2022-05-14 17:43:46.302352: This epoch took 477.203093 s + +2022-05-14 17:43:46.302654: +epoch: 99 +2022-05-14 17:51:02.487709: train loss : 0.0416 +2022-05-14 17:51:27.543350: validation loss: 0.0553 +2022-05-14 17:51:27.544030: Average global foreground Dice: [0.9474, 0.938, 0.9239] +2022-05-14 17:51:27.544115: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 17:51:27.948971: lr: 0.000207 +2022-05-14 17:51:27.949165: saving scheduled checkpoint file... +2022-05-14 17:51:27.995902: saving checkpoint... +2022-05-14 17:51:28.719274: done, saving took 0.77 seconds +2022-05-14 17:51:28.737161: done +2022-05-14 17:52:07.323206: This epoch took 501.020452 s + +2022-05-14 17:52:07.323509: +epoch: 100 +2022-05-14 17:59:24.116554: train loss : 0.0425 +2022-05-14 17:59:49.101592: validation loss: 0.0564 +2022-05-14 17:59:49.102196: Average global foreground Dice: [0.9489, 0.9422, 0.9304] +2022-05-14 17:59:49.102280: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 17:59:49.526223: lr: 0.000206 +2022-05-14 17:59:49.526423: saving scheduled checkpoint file... +2022-05-14 17:59:49.573118: saving checkpoint... +2022-05-14 17:59:50.377705: done, saving took 0.85 seconds +2022-05-14 17:59:50.396698: done +2022-05-14 18:00:11.383366: This epoch took 484.059751 s + +2022-05-14 18:00:11.383650: +epoch: 101 +2022-05-14 18:07:27.730817: train loss : 0.0437 +2022-05-14 18:07:52.791548: validation loss: 0.0560 +2022-05-14 18:07:52.792238: Average global foreground Dice: [0.9468, 0.9488, 0.9267] +2022-05-14 18:07:52.792325: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 18:07:53.202083: lr: 0.000204 +2022-05-14 18:07:53.202290: saving scheduled checkpoint file... +2022-05-14 18:07:53.249054: saving checkpoint... +2022-05-14 18:07:54.071798: done, saving took 0.87 seconds +2022-05-14 18:07:54.089900: done +2022-05-14 18:07:54.128376: saving checkpoint... +2022-05-14 18:07:54.919992: done, saving took 0.83 seconds +2022-05-14 18:08:15.559632: This epoch took 484.175881 s + +2022-05-14 18:08:15.559937: +epoch: 102 +2022-05-14 18:15:32.574834: train loss : 0.0429 +2022-05-14 18:15:57.610466: validation loss: 0.0610 +2022-05-14 18:15:57.611112: Average global foreground Dice: [0.9496, 0.9386, 0.9241] +2022-05-14 18:15:57.611202: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 18:15:58.020770: lr: 0.000202 +2022-05-14 18:15:58.020963: saving scheduled checkpoint file... +2022-05-14 18:15:58.067135: saving checkpoint... +2022-05-14 18:15:58.936520: done, saving took 0.92 seconds +2022-05-14 18:15:58.959464: done +2022-05-14 18:16:13.648309: This epoch took 478.088265 s + +2022-05-14 18:16:13.648628: +epoch: 103 +2022-05-14 18:23:30.385847: train loss : 0.0443 +2022-05-14 18:23:55.311384: validation loss: 0.0472 +2022-05-14 18:23:55.312094: Average global foreground Dice: [0.9493, 0.9346, 0.9275] +2022-05-14 18:23:55.312181: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 18:23:55.732100: lr: 0.0002 +2022-05-14 18:23:55.732297: saving scheduled checkpoint file... +2022-05-14 18:23:55.779149: saving checkpoint... +2022-05-14 18:23:56.581305: done, saving took 0.85 seconds +2022-05-14 18:23:56.602181: done +2022-05-14 18:24:21.658868: This epoch took 488.010134 s + +2022-05-14 18:24:21.659155: +epoch: 104 +2022-05-14 18:31:38.479391: train loss : 0.0439 +2022-05-14 18:32:03.508077: validation loss: 0.0544 +2022-05-14 18:32:03.508781: Average global foreground Dice: [0.9491, 0.928, 0.9262] +2022-05-14 18:32:03.508868: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 18:32:03.916447: lr: 0.000198 +2022-05-14 18:32:03.916637: saving scheduled checkpoint file... +2022-05-14 18:32:03.963513: saving checkpoint... +2022-05-14 18:32:04.754929: done, saving took 0.84 seconds +2022-05-14 18:32:04.794313: done +2022-05-14 18:32:19.277180: This epoch took 477.617907 s + +2022-05-14 18:32:19.277570: +epoch: 105 +2022-05-14 18:39:36.189784: train loss : 0.0384 +2022-05-14 18:40:01.208089: validation loss: 0.0422 +2022-05-14 18:40:01.208699: Average global foreground Dice: [0.9499, 0.9435, 0.9288] +2022-05-14 18:40:01.208794: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 18:40:01.626907: lr: 0.000196 +2022-05-14 18:40:01.627111: saving scheduled checkpoint file... +2022-05-14 18:40:01.674465: saving checkpoint... +2022-05-14 18:40:02.474902: done, saving took 0.85 seconds +2022-05-14 18:40:02.493119: done +2022-05-14 18:40:20.066868: This epoch took 480.789175 s + +2022-05-14 18:40:20.067237: +epoch: 106 +2022-05-14 18:47:37.063163: train loss : 0.0418 +2022-05-14 18:48:02.052758: validation loss: 0.0763 +2022-05-14 18:48:02.053508: Average global foreground Dice: [0.9476, 0.9332, 0.923] +2022-05-14 18:48:02.053594: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 18:48:02.459926: lr: 0.000194 +2022-05-14 18:48:02.460127: saving scheduled checkpoint file... +2022-05-14 18:48:02.506586: saving checkpoint... +2022-05-14 18:48:03.175429: done, saving took 0.72 seconds +2022-05-14 18:48:03.192484: done +2022-05-14 18:48:20.101249: This epoch took 480.033909 s + +2022-05-14 18:48:20.101526: +epoch: 107 +2022-05-14 18:55:36.302125: train loss : 0.0429 +2022-05-14 18:56:01.301921: validation loss: 0.0515 +2022-05-14 18:56:01.302511: Average global foreground Dice: [0.9487, 0.9421, 0.9314] +2022-05-14 18:56:01.302595: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 18:56:01.706306: lr: 0.000193 +2022-05-14 18:56:01.706495: saving scheduled checkpoint file... +2022-05-14 18:56:01.753595: saving checkpoint... +2022-05-14 18:56:02.525701: done, saving took 0.82 seconds +2022-05-14 18:56:02.543586: done +2022-05-14 18:56:22.280934: This epoch took 482.179292 s + +2022-05-14 18:56:22.281273: +epoch: 108 +2022-05-14 19:03:38.857712: train loss : 0.0422 +2022-05-14 19:04:03.888476: validation loss: 0.0486 +2022-05-14 19:04:03.889272: Average global foreground Dice: [0.9483, 0.945, 0.9288] +2022-05-14 19:04:03.889359: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 19:04:04.295534: lr: 0.000191 +2022-05-14 19:04:04.295730: saving scheduled checkpoint file... +2022-05-14 19:04:04.342466: saving checkpoint... +2022-05-14 19:04:05.092892: done, saving took 0.80 seconds +2022-05-14 19:04:05.110606: done +2022-05-14 19:04:05.144734: saving checkpoint... +2022-05-14 19:04:05.869992: done, saving took 0.76 seconds +2022-05-14 19:04:22.676343: This epoch took 480.394967 s + +2022-05-14 19:04:22.676623: +epoch: 109 +2022-05-14 19:11:39.481799: train loss : 0.0427 +2022-05-14 19:12:04.497938: validation loss: 0.0556 +2022-05-14 19:12:04.498654: Average global foreground Dice: [0.9495, 0.9452, 0.929] +2022-05-14 19:12:04.498739: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 19:12:04.903801: lr: 0.000189 +2022-05-14 19:12:04.903987: saving scheduled checkpoint file... +2022-05-14 19:12:04.950106: saving checkpoint... +2022-05-14 19:12:05.749424: done, saving took 0.85 seconds +2022-05-14 19:12:05.767387: done +2022-05-14 19:12:05.803433: saving checkpoint... +2022-05-14 19:12:06.585793: done, saving took 0.82 seconds +2022-05-14 19:12:22.482760: This epoch took 479.806033 s + +2022-05-14 19:12:22.483046: +epoch: 110 +2022-05-14 19:19:39.061116: train loss : 0.0438 +2022-05-14 19:20:04.052951: validation loss: 0.0578 +2022-05-14 19:20:04.053601: Average global foreground Dice: [0.9462, 0.9439, 0.9289] +2022-05-14 19:20:04.053685: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 19:20:04.463420: lr: 0.000187 +2022-05-14 19:20:04.463601: saving scheduled checkpoint file... +2022-05-14 19:20:04.510079: saving checkpoint... +2022-05-14 19:20:05.313346: done, saving took 0.85 seconds +2022-05-14 19:20:05.331830: done +2022-05-14 19:20:05.368158: saving checkpoint... +2022-05-14 19:20:06.126591: done, saving took 0.79 seconds +2022-05-14 19:20:24.799630: This epoch took 482.316483 s + +2022-05-14 19:20:24.799923: +epoch: 111 +2022-05-14 19:27:41.382790: train loss : 0.0438 +2022-05-14 19:28:06.315031: validation loss: 0.0538 +2022-05-14 19:28:06.315697: Average global foreground Dice: [0.9496, 0.9344, 0.9233] +2022-05-14 19:28:06.315798: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 19:28:06.823037: lr: 0.000185 +2022-05-14 19:28:06.823235: saving scheduled checkpoint file... +2022-05-14 19:28:06.870220: saving checkpoint... +2022-05-14 19:28:07.658627: done, saving took 0.84 seconds +2022-05-14 19:28:07.676435: done +2022-05-14 19:28:24.449315: This epoch took 479.649293 s + +2022-05-14 19:28:24.449611: +epoch: 112 +2022-05-14 19:35:41.277456: train loss : 0.0416 +2022-05-14 19:36:06.230867: validation loss: 0.0536 +2022-05-14 19:36:06.231458: Average global foreground Dice: [0.9472, 0.9278, 0.9253] +2022-05-14 19:36:06.231540: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 19:36:06.637199: lr: 0.000183 +2022-05-14 19:36:06.637397: saving scheduled checkpoint file... +2022-05-14 19:36:06.683313: saving checkpoint... +2022-05-14 19:36:07.400098: done, saving took 0.76 seconds +2022-05-14 19:36:07.419747: done +2022-05-14 19:36:27.457736: This epoch took 483.008001 s + +2022-05-14 19:36:27.458091: +epoch: 113 +2022-05-14 19:43:44.365885: train loss : 0.0417 +2022-05-14 19:44:09.442472: validation loss: 0.0588 +2022-05-14 19:44:09.443089: Average global foreground Dice: [0.9501, 0.9373, 0.9302] +2022-05-14 19:44:09.443177: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 19:44:09.853894: lr: 0.000181 +2022-05-14 19:44:09.854087: saving scheduled checkpoint file... +2022-05-14 19:44:09.900029: saving checkpoint... +2022-05-14 19:44:10.717945: done, saving took 0.86 seconds +2022-05-14 19:44:10.739348: done +2022-05-14 19:44:28.763421: This epoch took 481.305225 s + +2022-05-14 19:44:28.763716: +epoch: 114 +2022-05-14 19:51:45.695079: train loss : 0.0440 +2022-05-14 19:52:10.769096: validation loss: 0.0586 +2022-05-14 19:52:10.769741: Average global foreground Dice: [0.9446, 0.9372, 0.9243] +2022-05-14 19:52:10.769842: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 19:52:11.179790: lr: 0.000179 +2022-05-14 19:52:11.179986: saving scheduled checkpoint file... +2022-05-14 19:52:11.226236: saving checkpoint... +2022-05-14 19:52:12.048585: done, saving took 0.87 seconds +2022-05-14 19:52:12.069923: done +2022-05-14 19:52:29.828029: This epoch took 481.064193 s + +2022-05-14 19:52:29.828315: +epoch: 115 +2022-05-14 19:59:46.752476: train loss : 0.0406 +2022-05-14 20:00:11.782958: validation loss: 0.0519 +2022-05-14 20:00:11.783673: Average global foreground Dice: [0.9503, 0.9416, 0.9289] +2022-05-14 20:00:11.783762: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 20:00:12.193936: lr: 0.000177 +2022-05-14 20:00:12.194128: saving scheduled checkpoint file... +2022-05-14 20:00:12.240937: saving checkpoint... +2022-05-14 20:00:13.068425: done, saving took 0.87 seconds +2022-05-14 20:00:13.092079: done +2022-05-14 20:00:27.832633: This epoch took 478.004194 s + +2022-05-14 20:00:27.833002: +epoch: 116 +2022-05-14 20:07:44.562344: train loss : 0.0423 +2022-05-14 20:08:09.594671: validation loss: 0.0552 +2022-05-14 20:08:09.595296: Average global foreground Dice: [0.9501, 0.9388, 0.9273] +2022-05-14 20:08:09.595381: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 20:08:10.001683: lr: 0.000175 +2022-05-14 20:08:10.001880: saving scheduled checkpoint file... +2022-05-14 20:08:10.048097: saving checkpoint... +2022-05-14 20:08:10.843495: done, saving took 0.84 seconds +2022-05-14 20:08:10.861532: done +2022-05-14 20:08:28.851151: This epoch took 481.018032 s + +2022-05-14 20:08:28.851511: +epoch: 117 +2022-05-14 20:15:45.693311: train loss : 0.0387 +2022-05-14 20:16:10.695572: validation loss: 0.0528 +2022-05-14 20:16:10.696210: Average global foreground Dice: [0.951, 0.9481, 0.9278] +2022-05-14 20:16:10.696420: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 20:16:11.105464: lr: 0.000173 +2022-05-14 20:16:11.105653: saving scheduled checkpoint file... +2022-05-14 20:16:11.152308: saving checkpoint... +2022-05-14 20:16:11.910742: done, saving took 0.81 seconds +2022-05-14 20:16:11.929828: done +2022-05-14 20:16:31.876177: This epoch took 483.024541 s + +2022-05-14 20:16:31.876561: +epoch: 118 +2022-05-14 20:23:48.405250: train loss : 0.0456 +2022-05-14 20:24:13.392110: validation loss: 0.0587 +2022-05-14 20:24:13.392784: Average global foreground Dice: [0.9487, 0.9382, 0.9253] +2022-05-14 20:24:13.392871: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 20:24:13.803333: lr: 0.000172 +2022-05-14 20:24:13.803526: saving scheduled checkpoint file... +2022-05-14 20:24:13.850421: saving checkpoint... +2022-05-14 20:24:14.645051: done, saving took 0.84 seconds +2022-05-14 20:24:14.676280: done +2022-05-14 20:24:30.536577: This epoch took 478.659891 s + +2022-05-14 20:24:30.541395: +epoch: 119 +2022-05-14 20:31:47.500940: train loss : 0.0423 +2022-05-14 20:32:12.466476: validation loss: 0.0637 +2022-05-14 20:32:12.467103: Average global foreground Dice: [0.9494, 0.9354, 0.9234] +2022-05-14 20:32:12.467187: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 20:32:12.874422: lr: 0.00017 +2022-05-14 20:32:12.874615: saving scheduled checkpoint file... +2022-05-14 20:32:12.920689: saving checkpoint... +2022-05-14 20:32:13.707800: done, saving took 0.83 seconds +2022-05-14 20:32:13.726289: done +2022-05-14 20:32:33.169153: This epoch took 482.627640 s + +2022-05-14 20:32:33.169425: +epoch: 120 +2022-05-14 20:39:49.831056: train loss : 0.0437 +2022-05-14 20:40:14.812194: validation loss: 0.0523 +2022-05-14 20:40:14.812799: Average global foreground Dice: [0.9478, 0.9452, 0.9266] +2022-05-14 20:40:14.812886: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 20:40:15.222275: lr: 0.000168 +2022-05-14 20:40:15.222472: saving scheduled checkpoint file... +2022-05-14 20:40:15.270954: saving checkpoint... +2022-05-14 20:40:16.078714: done, saving took 0.86 seconds +2022-05-14 20:40:16.096906: done +2022-05-14 20:40:35.270330: This epoch took 482.100795 s + +2022-05-14 20:40:35.270617: +epoch: 121 +2022-05-14 20:47:51.403417: train loss : 0.0417 +2022-05-14 20:48:16.364813: validation loss: 0.0583 +2022-05-14 20:48:16.365466: Average global foreground Dice: [0.9499, 0.9435, 0.9302] +2022-05-14 20:48:16.365553: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 20:48:16.777759: lr: 0.000166 +2022-05-14 20:48:16.777966: saving scheduled checkpoint file... +2022-05-14 20:48:16.824347: saving checkpoint... +2022-05-14 20:48:17.617246: done, saving took 0.84 seconds +2022-05-14 20:48:17.635397: done +2022-05-14 20:48:40.268291: This epoch took 484.997569 s + +2022-05-14 20:48:40.268573: +epoch: 122 +2022-05-14 20:55:56.532562: train loss : 0.0428 +2022-05-14 20:56:21.529341: validation loss: 0.0540 +2022-05-14 20:56:21.529949: Average global foreground Dice: [0.95, 0.936, 0.9271] +2022-05-14 20:56:21.530036: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 20:56:21.940073: lr: 0.000164 +2022-05-14 20:56:21.940265: saving scheduled checkpoint file... +2022-05-14 20:56:21.986512: saving checkpoint... +2022-05-14 20:56:22.782165: done, saving took 0.84 seconds +2022-05-14 20:56:22.800258: done +2022-05-14 20:56:40.394922: This epoch took 480.126226 s + +2022-05-14 20:56:40.395307: +epoch: 123 +2022-05-14 21:03:57.017662: train loss : 0.0437 +2022-05-14 21:04:22.060166: validation loss: 0.0518 +2022-05-14 21:04:22.060830: Average global foreground Dice: [0.9495, 0.9401, 0.9278] +2022-05-14 21:04:22.060917: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 21:04:22.472365: lr: 0.000162 +2022-05-14 21:04:22.472558: saving scheduled checkpoint file... +2022-05-14 21:04:22.519205: saving checkpoint... +2022-05-14 21:04:23.323103: done, saving took 0.85 seconds +2022-05-14 21:04:23.341230: done +2022-05-14 21:04:39.395804: This epoch took 479.000384 s + +2022-05-14 21:04:39.396086: +epoch: 124 +2022-05-14 21:11:56.585202: train loss : 0.0425 +2022-05-14 21:12:21.644249: validation loss: 0.0491 +2022-05-14 21:12:21.644857: Average global foreground Dice: [0.9505, 0.9432, 0.9306] +2022-05-14 21:12:21.644953: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 21:12:22.058399: lr: 0.00016 +2022-05-14 21:12:22.058579: saving scheduled checkpoint file... +2022-05-14 21:12:22.105094: saving checkpoint... +2022-05-14 21:12:22.905853: done, saving took 0.85 seconds +2022-05-14 21:12:22.924052: done +2022-05-14 21:12:22.960379: saving checkpoint... +2022-05-14 21:12:23.731671: done, saving took 0.81 seconds +2022-05-14 21:12:43.188006: This epoch took 483.791626 s + +2022-05-14 21:12:43.188371: +epoch: 125 +2022-05-14 21:20:00.181332: train loss : 0.0411 +2022-05-14 21:20:25.139515: validation loss: 0.0493 +2022-05-14 21:20:25.140222: Average global foreground Dice: [0.9493, 0.9361, 0.9262] +2022-05-14 21:20:25.140306: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 21:20:25.550476: lr: 0.000158 +2022-05-14 21:20:25.550666: saving scheduled checkpoint file... +2022-05-14 21:20:25.597324: saving checkpoint... +2022-05-14 21:20:26.389254: done, saving took 0.84 seconds +2022-05-14 21:20:26.407857: done +2022-05-14 21:20:42.340123: This epoch took 479.151649 s + +2022-05-14 21:20:42.340414: +epoch: 126 +2022-05-14 21:27:59.397358: train loss : 0.0404 +2022-05-14 21:28:24.380793: validation loss: 0.0427 +2022-05-14 21:28:24.381478: Average global foreground Dice: [0.9497, 0.9478, 0.9285] +2022-05-14 21:28:24.381561: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 21:28:24.791302: lr: 0.000156 +2022-05-14 21:28:24.791489: saving scheduled checkpoint file... +2022-05-14 21:28:24.838094: saving checkpoint... +2022-05-14 21:28:25.637864: done, saving took 0.85 seconds +2022-05-14 21:28:25.656992: done +2022-05-14 21:28:25.695105: saving checkpoint... +2022-05-14 21:28:26.470631: done, saving took 0.81 seconds +2022-05-14 21:28:44.237581: This epoch took 481.897060 s + +2022-05-14 21:28:44.237920: +epoch: 127 +2022-05-14 21:36:00.895589: train loss : 0.0429 +2022-05-14 21:36:25.830930: validation loss: 0.0596 +2022-05-14 21:36:25.831562: Average global foreground Dice: [0.9507, 0.9438, 0.9247] +2022-05-14 21:36:25.831646: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 21:36:26.259074: lr: 0.000154 +2022-05-14 21:36:26.259260: saving scheduled checkpoint file... +2022-05-14 21:36:26.305522: saving checkpoint... +2022-05-14 21:36:26.996716: done, saving took 0.74 seconds +2022-05-14 21:36:27.015589: done +2022-05-14 21:36:27.051569: saving checkpoint... +2022-05-14 21:36:27.852092: done, saving took 0.84 seconds +2022-05-14 21:36:46.113260: This epoch took 481.875234 s + +2022-05-14 21:36:46.113554: +epoch: 128 +2022-05-14 21:44:03.024143: train loss : 0.0409 +2022-05-14 21:44:28.003436: validation loss: 0.0634 +2022-05-14 21:44:28.004123: Average global foreground Dice: [0.9488, 0.9453, 0.9208] +2022-05-14 21:44:28.004208: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 21:44:28.413130: lr: 0.000152 +2022-05-14 21:44:28.413330: saving scheduled checkpoint file... +2022-05-14 21:44:28.460274: saving checkpoint... +2022-05-14 21:44:29.118714: done, saving took 0.71 seconds +2022-05-14 21:44:29.136472: done +2022-05-14 21:44:51.096661: This epoch took 484.983005 s + +2022-05-14 21:44:51.096981: +epoch: 129 +2022-05-14 21:52:08.245734: train loss : 0.0424 +2022-05-14 21:52:33.349980: validation loss: 0.0601 +2022-05-14 21:52:33.350653: Average global foreground Dice: [0.9462, 0.9346, 0.9275] +2022-05-14 21:52:33.350738: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 21:52:33.766526: lr: 0.00015 +2022-05-14 21:52:33.766721: saving scheduled checkpoint file... +2022-05-14 21:52:33.813427: saving checkpoint... +2022-05-14 21:52:34.602521: done, saving took 0.84 seconds +2022-05-14 21:52:34.620407: done +2022-05-14 21:52:49.548318: This epoch took 478.451217 s + +2022-05-14 21:52:49.548689: +epoch: 130 +2022-05-14 22:00:06.250290: train loss : 0.0408 +2022-05-14 22:00:31.235599: validation loss: 0.0609 +2022-05-14 22:00:31.236300: Average global foreground Dice: [0.9498, 0.9485, 0.928] +2022-05-14 22:00:31.236383: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 22:00:31.648296: lr: 0.000148 +2022-05-14 22:00:31.648700: saving scheduled checkpoint file... +2022-05-14 22:00:31.695994: saving checkpoint... +2022-05-14 22:00:32.491848: done, saving took 0.84 seconds +2022-05-14 22:00:32.509700: done +2022-05-14 22:00:50.056668: This epoch took 480.507852 s + +2022-05-14 22:00:50.057084: +epoch: 131 +2022-05-14 22:08:06.694840: train loss : 0.0401 +2022-05-14 22:08:31.727852: validation loss: 0.0494 +2022-05-14 22:08:31.728504: Average global foreground Dice: [0.9498, 0.9405, 0.9322] +2022-05-14 22:08:31.728604: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 22:08:32.142253: lr: 0.000146 +2022-05-14 22:08:32.142439: saving scheduled checkpoint file... +2022-05-14 22:08:32.188918: saving checkpoint... +2022-05-14 22:08:32.999154: done, saving took 0.86 seconds +2022-05-14 22:08:33.017286: done +2022-05-14 22:08:33.053833: saving checkpoint... +2022-05-14 22:08:33.828842: done, saving took 0.81 seconds +2022-05-14 22:08:49.289996: This epoch took 479.232806 s + +2022-05-14 22:08:49.290275: +epoch: 132 +2022-05-14 22:16:05.842976: train loss : 0.0394 +2022-05-14 22:16:30.801248: validation loss: 0.0638 +2022-05-14 22:16:30.801879: Average global foreground Dice: [0.9487, 0.9381, 0.9286] +2022-05-14 22:16:30.802025: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 22:16:31.211324: lr: 0.000144 +2022-05-14 22:16:31.211519: saving scheduled checkpoint file... +2022-05-14 22:16:31.257805: saving checkpoint... +2022-05-14 22:16:32.047916: done, saving took 0.84 seconds +2022-05-14 22:16:32.066328: done +2022-05-14 22:16:50.595701: This epoch took 481.305325 s + +2022-05-14 22:16:50.596004: +epoch: 133 +2022-05-14 22:24:07.208312: train loss : 0.0399 +2022-05-14 22:24:32.222119: validation loss: 0.0535 +2022-05-14 22:24:32.222819: Average global foreground Dice: [0.9487, 0.9448, 0.9252] +2022-05-14 22:24:32.222906: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 22:24:32.634305: lr: 0.000142 +2022-05-14 22:24:32.634507: saving scheduled checkpoint file... +2022-05-14 22:24:32.683968: saving checkpoint... +2022-05-14 22:24:33.481743: done, saving took 0.85 seconds +2022-05-14 22:24:33.499857: done +2022-05-14 22:24:50.917980: This epoch took 480.321874 s + +2022-05-14 22:24:50.918284: +epoch: 134 +2022-05-14 22:32:07.570441: train loss : 0.0417 +2022-05-14 22:32:32.696746: validation loss: 0.0557 +2022-05-14 22:32:32.697359: Average global foreground Dice: [0.9481, 0.9426, 0.9279] +2022-05-14 22:32:32.697443: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 22:32:33.107758: lr: 0.00014 +2022-05-14 22:32:33.107961: saving scheduled checkpoint file... +2022-05-14 22:32:33.154538: saving checkpoint... +2022-05-14 22:32:33.948041: done, saving took 0.84 seconds +2022-05-14 22:32:33.966253: done +2022-05-14 22:32:50.234228: This epoch took 479.315848 s + +2022-05-14 22:32:50.234503: +epoch: 135 +2022-05-14 22:40:07.234844: train loss : 0.0411 +2022-05-14 22:40:32.330158: validation loss: 0.0529 +2022-05-14 22:40:32.330760: Average global foreground Dice: [0.9487, 0.9403, 0.9272] +2022-05-14 22:40:32.330856: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 22:40:32.740994: lr: 0.000138 +2022-05-14 22:40:32.741184: saving scheduled checkpoint file... +2022-05-14 22:40:32.787678: saving checkpoint... +2022-05-14 22:40:33.577563: done, saving took 0.84 seconds +2022-05-14 22:40:33.595708: done +2022-05-14 22:40:59.167616: This epoch took 488.933012 s + +2022-05-14 22:40:59.167928: +epoch: 136 +2022-05-14 22:48:15.948460: train loss : 0.0409 +2022-05-14 22:48:41.013007: validation loss: 0.0453 +2022-05-14 22:48:41.013663: Average global foreground Dice: [0.9497, 0.9382, 0.9267] +2022-05-14 22:48:41.013953: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 22:48:41.427213: lr: 0.000136 +2022-05-14 22:48:41.427407: saving scheduled checkpoint file... +2022-05-14 22:48:41.474089: saving checkpoint... +2022-05-14 22:48:42.271352: done, saving took 0.84 seconds +2022-05-14 22:48:42.289596: done +2022-05-14 22:49:03.417709: This epoch took 484.249681 s + +2022-05-14 22:49:03.418009: +epoch: 137 +2022-05-14 22:56:20.154006: train loss : 0.0426 +2022-05-14 22:56:45.147256: validation loss: 0.0453 +2022-05-14 22:56:45.147887: Average global foreground Dice: [0.9483, 0.9436, 0.9303] +2022-05-14 22:56:45.147973: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 22:56:45.564985: lr: 0.000134 +2022-05-14 22:56:45.565178: saving scheduled checkpoint file... +2022-05-14 22:56:45.611387: saving checkpoint... +2022-05-14 22:56:46.410307: done, saving took 0.85 seconds +2022-05-14 22:56:46.430829: done +2022-05-14 22:56:46.472196: saving checkpoint... +2022-05-14 22:56:47.253587: done, saving took 0.82 seconds +2022-05-14 22:57:07.071538: This epoch took 483.653412 s + +2022-05-14 22:57:07.071911: +epoch: 138 +2022-05-14 23:04:23.974287: train loss : 0.0390 +2022-05-14 23:04:49.021499: validation loss: 0.0489 +2022-05-14 23:04:49.022160: Average global foreground Dice: [0.9531, 0.9367, 0.9334] +2022-05-14 23:04:49.022247: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 23:04:49.432693: lr: 0.000132 +2022-05-14 23:04:49.432894: saving scheduled checkpoint file... +2022-05-14 23:04:49.479003: saving checkpoint... +2022-05-14 23:04:50.271634: done, saving took 0.84 seconds +2022-05-14 23:04:50.289660: done +2022-05-14 23:04:50.325856: saving checkpoint... +2022-05-14 23:04:51.104606: done, saving took 0.81 seconds +2022-05-14 23:05:16.893453: This epoch took 489.821432 s + +2022-05-14 23:05:16.893743: +epoch: 139 +2022-05-14 23:12:33.440416: train loss : 0.0409 +2022-05-14 23:12:58.488802: validation loss: 0.0513 +2022-05-14 23:12:58.489437: Average global foreground Dice: [0.948, 0.9439, 0.9269] +2022-05-14 23:12:58.489522: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 23:12:58.898513: lr: 0.00013 +2022-05-14 23:12:58.898699: saving scheduled checkpoint file... +2022-05-14 23:12:58.945340: saving checkpoint... +2022-05-14 23:12:59.737117: done, saving took 0.84 seconds +2022-05-14 23:12:59.757812: done +2022-05-14 23:12:59.799011: saving checkpoint... +2022-05-14 23:13:00.575469: done, saving took 0.82 seconds +2022-05-14 23:13:18.301832: This epoch took 481.407909 s + +2022-05-14 23:13:18.302205: +epoch: 140 +2022-05-14 23:20:34.714373: train loss : 0.0404 +2022-05-14 23:20:59.771980: validation loss: 0.0579 +2022-05-14 23:20:59.772601: Average global foreground Dice: [0.9493, 0.9382, 0.932] +2022-05-14 23:20:59.772685: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 23:21:00.181423: lr: 0.000128 +2022-05-14 23:21:00.181630: saving scheduled checkpoint file... +2022-05-14 23:21:00.228964: saving checkpoint... +2022-05-14 23:21:01.019815: done, saving took 0.84 seconds +2022-05-14 23:21:01.037632: done +2022-05-14 23:21:01.073708: saving checkpoint... +2022-05-14 23:21:01.839565: done, saving took 0.80 seconds +2022-05-14 23:21:20.205091: This epoch took 481.902785 s + +2022-05-14 23:21:20.205382: +epoch: 141 +2022-05-14 23:28:36.700713: train loss : 0.0382 +2022-05-14 23:29:01.698757: validation loss: 0.0737 +2022-05-14 23:29:01.699397: Average global foreground Dice: [0.95, 0.9406, 0.9268] +2022-05-14 23:29:01.699481: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 23:29:02.114071: lr: 0.000127 +2022-05-14 23:29:02.114353: saving scheduled checkpoint file... +2022-05-14 23:29:02.160597: saving checkpoint... +2022-05-14 23:29:02.945427: done, saving took 0.83 seconds +2022-05-14 23:29:02.963499: done +2022-05-14 23:29:19.389895: This epoch took 479.184399 s + +2022-05-14 23:29:19.390227: +epoch: 142 +2022-05-14 23:36:36.022132: train loss : 0.0399 +2022-05-14 23:37:01.006795: validation loss: 0.0491 +2022-05-14 23:37:01.007421: Average global foreground Dice: [0.9468, 0.9469, 0.923] +2022-05-14 23:37:01.007505: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 23:37:01.417676: lr: 0.000125 +2022-05-14 23:37:01.417877: saving scheduled checkpoint file... +2022-05-14 23:37:01.464440: saving checkpoint... +2022-05-14 23:37:02.257792: done, saving took 0.84 seconds +2022-05-14 23:37:02.278232: done +2022-05-14 23:37:19.051822: This epoch took 479.661493 s + +2022-05-14 23:37:19.052120: +epoch: 143 +2022-05-14 23:44:35.201427: train loss : 0.0431 +2022-05-14 23:45:00.192069: validation loss: 0.0518 +2022-05-14 23:45:00.192727: Average global foreground Dice: [0.9504, 0.9432, 0.9272] +2022-05-14 23:45:00.192828: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 23:45:00.603722: lr: 0.000123 +2022-05-14 23:45:00.603925: saving scheduled checkpoint file... +2022-05-14 23:45:00.653526: saving checkpoint... +2022-05-14 23:45:01.445275: done, saving took 0.84 seconds +2022-05-14 23:45:01.463170: done +2022-05-14 23:45:20.699544: This epoch took 481.647303 s + +2022-05-14 23:45:20.699924: +epoch: 144 +2022-05-14 23:52:37.252814: train loss : 0.0386 +2022-05-14 23:53:02.288641: validation loss: 0.0537 +2022-05-14 23:53:02.289322: Average global foreground Dice: [0.9515, 0.9369, 0.9298] +2022-05-14 23:53:02.289408: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 23:53:02.701139: lr: 0.000121 +2022-05-14 23:53:02.701334: saving scheduled checkpoint file... +2022-05-14 23:53:02.748405: saving checkpoint... +2022-05-14 23:53:03.550936: done, saving took 0.85 seconds +2022-05-14 23:53:03.569016: done +2022-05-14 23:53:20.790985: This epoch took 480.090959 s + +2022-05-14 23:53:20.791271: +epoch: 145 +2022-05-15 00:00:37.742970: train loss : 0.0382 +2022-05-15 00:01:02.779788: validation loss: 0.0521 +2022-05-15 00:01:02.780468: Average global foreground Dice: [0.949, 0.9434, 0.9285] +2022-05-15 00:01:02.780554: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 00:01:03.192530: lr: 0.000119 +2022-05-15 00:01:03.192724: saving scheduled checkpoint file... +2022-05-15 00:01:03.240011: saving checkpoint... +2022-05-15 00:01:04.043738: done, saving took 0.85 seconds +2022-05-15 00:01:04.061976: done +2022-05-15 00:01:04.098407: saving checkpoint... +2022-05-15 00:01:04.864972: done, saving took 0.80 seconds +2022-05-15 00:01:20.812715: This epoch took 480.021324 s + +2022-05-15 00:01:20.813128: +epoch: 146 +2022-05-15 00:08:37.492968: train loss : 0.0383 +2022-05-15 00:09:02.492023: validation loss: 0.0565 +2022-05-15 00:09:02.492711: Average global foreground Dice: [0.9504, 0.9457, 0.9287] +2022-05-15 00:09:02.492808: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 00:09:02.907168: lr: 0.000117 +2022-05-15 00:09:02.907383: saving scheduled checkpoint file... +2022-05-15 00:09:02.953830: saving checkpoint... +2022-05-15 00:09:03.742362: done, saving took 0.83 seconds +2022-05-15 00:09:03.760610: done +2022-05-15 00:09:03.796646: saving checkpoint... +2022-05-15 00:09:04.566633: done, saving took 0.81 seconds +2022-05-15 00:09:28.313158: This epoch took 487.499889 s + +2022-05-15 00:09:28.313613: +epoch: 147 +2022-05-15 00:16:44.906949: train loss : 0.0406 +2022-05-15 00:17:09.847784: validation loss: 0.0529 +2022-05-15 00:17:09.848446: Average global foreground Dice: [0.9501, 0.9359, 0.9321] +2022-05-15 00:17:09.848531: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 00:17:10.261969: lr: 0.000115 +2022-05-15 00:17:10.262196: saving scheduled checkpoint file... +2022-05-15 00:17:10.308681: saving checkpoint... +2022-05-15 00:17:11.060542: done, saving took 0.80 seconds +2022-05-15 00:17:11.077954: done +2022-05-15 00:17:31.562371: This epoch took 483.248637 s + +2022-05-15 00:17:31.562648: +epoch: 148 +2022-05-15 00:24:48.190991: train loss : 0.0390 +2022-05-15 00:25:13.179987: validation loss: 0.0588 +2022-05-15 00:25:13.180603: Average global foreground Dice: [0.9536, 0.9377, 0.9313] +2022-05-15 00:25:13.180690: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 00:25:13.602654: lr: 0.000113 +2022-05-15 00:25:13.602859: saving scheduled checkpoint file... +2022-05-15 00:25:13.649647: saving checkpoint... +2022-05-15 00:25:14.398691: done, saving took 0.80 seconds +2022-05-15 00:25:14.417215: done +2022-05-15 00:25:14.452063: saving checkpoint... +2022-05-15 00:25:15.152635: done, saving took 0.74 seconds +2022-05-15 00:25:36.944681: This epoch took 485.381923 s + +2022-05-15 00:25:36.944990: +epoch: 149 +2022-05-15 00:32:53.440605: train loss : 0.0383 +2022-05-15 00:33:18.450309: validation loss: 0.0514 +2022-05-15 00:33:18.450951: Average global foreground Dice: [0.9487, 0.9489, 0.9299] +2022-05-15 00:33:18.451036: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 00:33:18.862393: lr: 0.000111 +2022-05-15 00:33:18.862590: saving scheduled checkpoint file... +2022-05-15 00:33:18.909034: saving checkpoint... +2022-05-15 00:33:19.606757: done, saving took 0.74 seconds +2022-05-15 00:33:19.625178: done +2022-05-15 00:33:19.660313: saving checkpoint... +2022-05-15 00:33:20.305098: done, saving took 0.68 seconds +2022-05-15 00:33:39.160354: This epoch took 482.215245 s + +2022-05-15 00:33:39.160641: +epoch: 150 +2022-05-15 00:40:56.166823: train loss : 0.0388 +2022-05-15 00:41:21.137123: validation loss: 0.0565 +2022-05-15 00:41:21.137817: Average global foreground Dice: [0.9504, 0.9346, 0.9302] +2022-05-15 00:41:21.137904: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 00:41:21.549944: lr: 0.000109 +2022-05-15 00:41:21.550148: saving scheduled checkpoint file... +2022-05-15 00:41:21.596709: saving checkpoint... +2022-05-15 00:41:22.294546: done, saving took 0.74 seconds +2022-05-15 00:41:22.313020: done +2022-05-15 00:41:49.689607: This epoch took 490.528865 s + +2022-05-15 00:41:49.689954: +epoch: 151 +2022-05-15 00:49:06.405344: train loss : 0.0393 +2022-05-15 00:49:31.404157: validation loss: 0.0557 +2022-05-15 00:49:31.404800: Average global foreground Dice: [0.947, 0.9407, 0.9279] +2022-05-15 00:49:31.404886: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 00:49:31.821661: lr: 0.000107 +2022-05-15 00:49:31.821867: saving scheduled checkpoint file... +2022-05-15 00:49:31.867568: saving checkpoint... +2022-05-15 00:49:32.623124: done, saving took 0.80 seconds +2022-05-15 00:49:32.660931: done +2022-05-15 00:50:10.030459: This epoch took 500.340404 s + +2022-05-15 00:50:10.030799: +epoch: 152 +2022-05-15 00:57:26.476005: train loss : 0.0399 +2022-05-15 00:57:51.466884: validation loss: 0.0658 +2022-05-15 00:57:51.467566: Average global foreground Dice: [0.9497, 0.9361, 0.9229] +2022-05-15 00:57:51.467722: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 00:57:51.878034: lr: 0.000106 +2022-05-15 00:57:51.878226: saving scheduled checkpoint file... +2022-05-15 00:57:51.924411: saving checkpoint... +2022-05-15 00:57:52.713700: done, saving took 0.84 seconds +2022-05-15 00:57:52.731662: done +2022-05-15 00:58:11.037291: This epoch took 481.006388 s + +2022-05-15 00:58:11.037567: +epoch: 153 +2022-05-15 01:05:28.021906: train loss : 0.0378 +2022-05-15 01:05:53.021616: validation loss: 0.0459 +2022-05-15 01:05:53.022299: Average global foreground Dice: [0.9499, 0.9416, 0.9258] +2022-05-15 01:05:53.022382: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 01:05:53.446394: lr: 0.000104 +2022-05-15 01:05:53.446577: saving scheduled checkpoint file... +2022-05-15 01:05:53.493466: saving checkpoint... +2022-05-15 01:05:54.292416: done, saving took 0.85 seconds +2022-05-15 01:05:54.310720: done +2022-05-15 01:06:10.623490: This epoch took 479.585824 s + +2022-05-15 01:06:10.623849: +epoch: 154 +2022-05-15 01:13:27.390682: train loss : 0.0376 +2022-05-15 01:13:52.423111: validation loss: 0.0502 +2022-05-15 01:13:52.423723: Average global foreground Dice: [0.9522, 0.9453, 0.9329] +2022-05-15 01:13:52.423819: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 01:13:52.846045: lr: 0.000102 +2022-05-15 01:13:52.846253: saving scheduled checkpoint file... +2022-05-15 01:13:52.892736: saving checkpoint... +2022-05-15 01:13:53.696345: done, saving took 0.85 seconds +2022-05-15 01:13:53.716565: done +2022-05-15 01:14:14.025690: This epoch took 483.401738 s + +2022-05-15 01:14:14.025991: +epoch: 155 +2022-05-15 01:21:30.755478: train loss : 0.0407 +2022-05-15 01:21:55.758196: validation loss: 0.0558 +2022-05-15 01:21:55.758906: Average global foreground Dice: [0.9487, 0.9446, 0.9314] +2022-05-15 01:21:55.758993: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 01:21:56.175620: lr: 0.0001 +2022-05-15 01:21:56.175810: saving scheduled checkpoint file... +2022-05-15 01:21:56.223222: saving checkpoint... +2022-05-15 01:21:57.028908: done, saving took 0.85 seconds +2022-05-15 01:21:57.047159: done +2022-05-15 01:22:14.166190: This epoch took 480.140081 s + +2022-05-15 01:22:14.166558: +epoch: 156 +2022-05-15 01:29:30.877463: train loss : 0.0388 +2022-05-15 01:29:55.803922: validation loss: 0.0510 +2022-05-15 01:29:55.804693: Average global foreground Dice: [0.949, 0.9428, 0.9322] +2022-05-15 01:29:55.804786: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 01:29:56.224760: lr: 9.8e-05 +2022-05-15 01:29:56.224951: saving scheduled checkpoint file... +2022-05-15 01:29:56.272123: saving checkpoint... +2022-05-15 01:29:57.067552: done, saving took 0.84 seconds +2022-05-15 01:29:57.085625: done +2022-05-15 01:29:57.121799: saving checkpoint... +2022-05-15 01:29:57.896048: done, saving took 0.81 seconds +2022-05-15 01:30:18.328898: This epoch took 484.162239 s + +2022-05-15 01:30:18.329166: +epoch: 157 +2022-05-15 01:37:35.723782: train loss : 0.0402 +2022-05-15 01:38:00.761163: validation loss: 0.0623 +2022-05-15 01:38:00.761932: Average global foreground Dice: [0.9518, 0.9386, 0.9311] +2022-05-15 01:38:00.762026: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 01:38:01.203637: lr: 9.6e-05 +2022-05-15 01:38:01.203842: saving scheduled checkpoint file... +2022-05-15 01:38:01.251785: saving checkpoint... +2022-05-15 01:38:01.937228: done, saving took 0.73 seconds +2022-05-15 01:38:01.955258: done +2022-05-15 01:38:01.990880: saving checkpoint... +2022-05-15 01:38:02.675154: done, saving took 0.72 seconds +2022-05-15 01:38:23.315520: This epoch took 484.986254 s + +2022-05-15 01:38:23.315829: +epoch: 158 +2022-05-15 01:45:39.974735: train loss : 0.0405 +2022-05-15 01:46:04.963076: validation loss: 0.0503 +2022-05-15 01:46:04.963663: Average global foreground Dice: [0.9522, 0.9413, 0.9289] +2022-05-15 01:46:04.963746: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 01:46:05.377696: lr: 9.4e-05 +2022-05-15 01:46:05.377895: saving scheduled checkpoint file... +2022-05-15 01:46:05.424935: saving checkpoint... +2022-05-15 01:46:06.223866: done, saving took 0.85 seconds +2022-05-15 01:46:06.242100: done +2022-05-15 01:46:06.279099: saving checkpoint... +2022-05-15 01:46:07.077110: done, saving took 0.83 seconds +2022-05-15 01:46:25.378642: This epoch took 482.062706 s + +2022-05-15 01:46:25.379037: +epoch: 159 +2022-05-15 01:53:41.940816: train loss : 0.0381 +2022-05-15 01:54:06.932079: validation loss: 0.0548 +2022-05-15 01:54:06.932860: Average global foreground Dice: [0.9508, 0.9465, 0.9296] +2022-05-15 01:54:06.932957: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 01:54:07.352641: lr: 9.3e-05 +2022-05-15 01:54:07.352829: saving scheduled checkpoint file... +2022-05-15 01:54:07.399261: saving checkpoint... +2022-05-15 01:54:08.193818: done, saving took 0.84 seconds +2022-05-15 01:54:08.212215: done +2022-05-15 01:54:08.248537: saving checkpoint... +2022-05-15 01:54:09.026057: done, saving took 0.81 seconds +2022-05-15 01:54:29.545190: This epoch took 484.166051 s + +2022-05-15 01:54:29.545472: +epoch: 160 +2022-05-15 02:01:46.515945: train loss : 0.0380 +2022-05-15 02:02:11.516434: validation loss: 0.0442 +2022-05-15 02:02:11.517385: Average global foreground Dice: [0.9516, 0.9365, 0.9278] +2022-05-15 02:02:11.517485: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 02:02:11.929834: lr: 9.1e-05 +2022-05-15 02:02:11.930026: saving scheduled checkpoint file... +2022-05-15 02:02:11.977029: saving checkpoint... +2022-05-15 02:02:12.779015: done, saving took 0.85 seconds +2022-05-15 02:02:12.797211: done +2022-05-15 02:02:31.026645: This epoch took 481.481071 s + +2022-05-15 02:02:31.026942: +epoch: 161 +2022-05-15 02:09:47.011292: train loss : 0.0381 +2022-05-15 02:10:12.004288: validation loss: 0.0654 +2022-05-15 02:10:12.005058: Average global foreground Dice: [0.948, 0.9333, 0.9296] +2022-05-15 02:10:12.005145: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 02:10:12.418865: lr: 8.9e-05 +2022-05-15 02:10:12.419055: saving scheduled checkpoint file... +2022-05-15 02:10:12.466027: saving checkpoint... +2022-05-15 02:10:13.259907: done, saving took 0.84 seconds +2022-05-15 02:10:13.278194: done +2022-05-15 02:10:34.030461: This epoch took 483.003418 s + +2022-05-15 02:10:34.030823: +epoch: 162 +2022-05-15 02:17:51.226469: train loss : 0.0367 +2022-05-15 02:18:16.199948: validation loss: 0.0633 +2022-05-15 02:18:16.200654: Average global foreground Dice: [0.9498, 0.9336, 0.9271] +2022-05-15 02:18:16.200739: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 02:18:16.621487: lr: 8.7e-05 +2022-05-15 02:18:16.621683: saving scheduled checkpoint file... +2022-05-15 02:18:16.668824: saving checkpoint... +2022-05-15 02:18:17.465187: done, saving took 0.84 seconds +2022-05-15 02:18:17.483436: done +2022-05-15 02:18:46.052129: This epoch took 492.021201 s + +2022-05-15 02:18:46.052414: +epoch: 163 +2022-05-15 02:26:03.246043: train loss : 0.0390 +2022-05-15 02:26:28.280146: validation loss: 0.0444 +2022-05-15 02:26:28.280841: Average global foreground Dice: [0.9522, 0.9437, 0.9304] +2022-05-15 02:26:28.280930: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 02:26:28.696182: lr: 8.5e-05 +2022-05-15 02:26:28.696367: saving scheduled checkpoint file... +2022-05-15 02:26:28.743011: saving checkpoint... +2022-05-15 02:26:29.539883: done, saving took 0.84 seconds +2022-05-15 02:26:29.558028: done +2022-05-15 02:26:49.849693: This epoch took 483.797161 s + +2022-05-15 02:26:49.850052: +epoch: 164 +2022-05-15 02:34:06.990289: train loss : 0.0408 +2022-05-15 02:34:32.038537: validation loss: 0.0526 +2022-05-15 02:34:32.039219: Average global foreground Dice: [0.9517, 0.9361, 0.9247] +2022-05-15 02:34:32.039307: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 02:34:32.452987: lr: 8.4e-05 +2022-05-15 02:34:32.453178: saving scheduled checkpoint file... +2022-05-15 02:34:32.499522: saving checkpoint... +2022-05-15 02:34:33.286704: done, saving took 0.83 seconds +2022-05-15 02:34:33.305021: done +2022-05-15 02:34:59.079156: This epoch took 489.228981 s + +2022-05-15 02:34:59.079499: +epoch: 165 +2022-05-15 02:42:15.974362: train loss : 0.0360 +2022-05-15 02:42:40.977163: validation loss: 0.0501 +2022-05-15 02:42:40.977887: Average global foreground Dice: [0.9525, 0.9457, 0.9296] +2022-05-15 02:42:40.977974: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 02:42:41.401386: lr: 8.2e-05 +2022-05-15 02:42:41.401580: saving scheduled checkpoint file... +2022-05-15 02:42:41.448235: saving checkpoint... +2022-05-15 02:42:42.249038: done, saving took 0.85 seconds +2022-05-15 02:42:42.267479: done +2022-05-15 02:43:05.140830: This epoch took 486.061159 s + +2022-05-15 02:43:05.141197: +epoch: 166 +2022-05-15 02:50:22.386832: train loss : 0.0386 +2022-05-15 02:50:47.433255: validation loss: 0.0662 +2022-05-15 02:50:47.433871: Average global foreground Dice: [0.9497, 0.9341, 0.9267] +2022-05-15 02:50:47.433960: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 02:50:47.855453: lr: 8e-05 +2022-05-15 02:50:47.855636: saving scheduled checkpoint file... +2022-05-15 02:50:47.901923: saving checkpoint... +2022-05-15 02:50:48.695761: done, saving took 0.84 seconds +2022-05-15 02:50:48.713638: done +2022-05-15 02:51:12.840333: This epoch took 487.699030 s + +2022-05-15 02:51:12.840609: +epoch: 167 +2022-05-15 02:58:29.573302: train loss : 0.0392 +2022-05-15 02:58:54.552576: validation loss: 0.0600 +2022-05-15 02:58:54.553236: Average global foreground Dice: [0.9541, 0.9408, 0.9317] +2022-05-15 02:58:54.553322: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 02:58:54.969151: lr: 7.8e-05 +2022-05-15 02:58:54.969369: saving scheduled checkpoint file... +2022-05-15 02:58:55.016121: saving checkpoint... +2022-05-15 02:58:55.810992: done, saving took 0.84 seconds +2022-05-15 02:58:55.829237: done +2022-05-15 02:59:12.962678: This epoch took 480.121965 s + +2022-05-15 02:59:12.963020: +epoch: 168 +2022-05-15 03:06:30.009933: train loss : 0.0381 +2022-05-15 03:06:54.950900: validation loss: 0.0571 +2022-05-15 03:06:54.951675: Average global foreground Dice: [0.95, 0.9472, 0.9322] +2022-05-15 03:06:54.951763: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 03:06:55.364338: lr: 7.7e-05 +2022-05-15 03:06:55.364535: saving scheduled checkpoint file... +2022-05-15 03:06:55.411409: saving checkpoint... +2022-05-15 03:06:56.201017: done, saving took 0.84 seconds +2022-05-15 03:06:56.218750: done +2022-05-15 03:07:22.261228: This epoch took 489.298097 s + +2022-05-15 03:07:22.261552: +epoch: 169 +2022-05-15 03:14:39.069655: train loss : 0.0392 +2022-05-15 03:15:03.994974: validation loss: 0.0565 +2022-05-15 03:15:03.995607: Average global foreground Dice: [0.9519, 0.9387, 0.9314] +2022-05-15 03:15:03.995693: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 03:15:04.408220: lr: 7.5e-05 +2022-05-15 03:15:04.408418: saving scheduled checkpoint file... +2022-05-15 03:15:04.455091: saving checkpoint... +2022-05-15 03:15:05.247533: done, saving took 0.84 seconds +2022-05-15 03:15:05.265696: done +2022-05-15 03:15:27.377463: This epoch took 485.115803 s + +2022-05-15 03:15:27.377742: +epoch: 170 +2022-05-15 03:22:44.162539: train loss : 0.0384 +2022-05-15 03:23:09.160192: validation loss: 0.0604 +2022-05-15 03:23:09.160808: Average global foreground Dice: [0.9492, 0.9316, 0.9264] +2022-05-15 03:23:09.161016: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 03:23:09.574800: lr: 7.3e-05 +2022-05-15 03:23:09.574997: saving scheduled checkpoint file... +2022-05-15 03:23:09.622045: saving checkpoint... +2022-05-15 03:23:10.296323: done, saving took 0.72 seconds +2022-05-15 03:23:10.314128: done +2022-05-15 03:23:27.697972: This epoch took 480.320099 s + +2022-05-15 03:23:27.698251: +epoch: 171 +2022-05-15 03:30:44.427681: train loss : 0.0366 +2022-05-15 03:31:09.525051: validation loss: 0.0562 +2022-05-15 03:31:09.525840: Average global foreground Dice: [0.9487, 0.9359, 0.9327] +2022-05-15 03:31:09.525980: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 03:31:09.940037: lr: 7.2e-05 +2022-05-15 03:31:09.940238: saving scheduled checkpoint file... +2022-05-15 03:31:09.987586: saving checkpoint... +2022-05-15 03:31:10.716897: done, saving took 0.78 seconds +2022-05-15 03:31:10.734432: done +2022-05-15 03:31:49.769930: This epoch took 502.071562 s + +2022-05-15 03:31:49.770282: +epoch: 172 +2022-05-15 03:39:06.395995: train loss : 0.0378 +2022-05-15 03:39:31.348581: validation loss: 0.0548 +2022-05-15 03:39:31.349279: Average global foreground Dice: [0.9518, 0.945, 0.9325] +2022-05-15 03:39:31.349377: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 03:39:31.767359: lr: 7e-05 +2022-05-15 03:39:31.767548: saving scheduled checkpoint file... +2022-05-15 03:39:31.813800: saving checkpoint... +2022-05-15 03:39:32.543741: done, saving took 0.78 seconds +2022-05-15 03:39:32.561139: done +2022-05-15 03:39:53.514918: This epoch took 483.744512 s + +2022-05-15 03:39:53.515303: +epoch: 173 +2022-05-15 03:47:10.364202: train loss : 0.0368 +2022-05-15 03:47:35.348843: validation loss: 0.0568 +2022-05-15 03:47:35.349446: Average global foreground Dice: [0.9501, 0.9439, 0.9324] +2022-05-15 03:47:35.349532: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 03:47:35.762586: lr: 6.8e-05 +2022-05-15 03:47:35.762801: saving scheduled checkpoint file... +2022-05-15 03:47:35.809358: saving checkpoint... +2022-05-15 03:47:36.535650: done, saving took 0.77 seconds +2022-05-15 03:47:36.553340: done +2022-05-15 03:47:59.830525: This epoch took 486.315122 s + +2022-05-15 03:47:59.830850: +epoch: 174 +2022-05-15 03:55:16.748190: train loss : 0.0368 +2022-05-15 03:55:41.757848: validation loss: 0.0606 +2022-05-15 03:55:41.758520: Average global foreground Dice: [0.9502, 0.9446, 0.9315] +2022-05-15 03:55:41.758605: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 03:55:42.172487: lr: 6.7e-05 +2022-05-15 03:55:42.172675: saving scheduled checkpoint file... +2022-05-15 03:55:42.218895: saving checkpoint... +2022-05-15 03:55:43.007751: done, saving took 0.83 seconds +2022-05-15 03:55:43.025957: done +2022-05-15 03:55:43.062263: saving checkpoint... +2022-05-15 03:55:43.845157: done, saving took 0.82 seconds +2022-05-15 03:56:03.791822: This epoch took 483.960873 s + +2022-05-15 03:56:03.792107: +epoch: 175 +2022-05-15 04:03:20.483486: train loss : 0.0356 +2022-05-15 04:03:45.471906: validation loss: 0.0651 +2022-05-15 04:03:45.472530: Average global foreground Dice: [0.9517, 0.9454, 0.9303] +2022-05-15 04:03:45.472618: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 04:03:45.897032: lr: 6.5e-05 +2022-05-15 04:03:45.897429: saving scheduled checkpoint file... +2022-05-15 04:03:45.944233: saving checkpoint... +2022-05-15 04:03:46.724693: done, saving took 0.83 seconds +2022-05-15 04:03:46.743105: done +2022-05-15 04:03:46.782801: saving checkpoint... +2022-05-15 04:03:47.566887: done, saving took 0.82 seconds +2022-05-15 04:04:04.406678: This epoch took 480.614339 s + +2022-05-15 04:04:04.407043: +epoch: 176 +2022-05-15 04:11:21.173584: train loss : 0.0372 +2022-05-15 04:11:46.203305: validation loss: 0.0583 +2022-05-15 04:11:46.203956: Average global foreground Dice: [0.9522, 0.9476, 0.935] +2022-05-15 04:11:46.204042: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 04:11:46.619385: lr: 6.3e-05 +2022-05-15 04:11:46.619573: saving scheduled checkpoint file... +2022-05-15 04:11:46.665944: saving checkpoint... +2022-05-15 04:11:47.446427: done, saving took 0.83 seconds +2022-05-15 04:11:47.464524: done +2022-05-15 04:11:47.501013: saving checkpoint... +2022-05-15 04:11:48.272341: done, saving took 0.81 seconds +2022-05-15 04:12:12.514145: This epoch took 488.106999 s + +2022-05-15 04:12:12.514435: +epoch: 177 +2022-05-15 04:19:29.296315: train loss : 0.0381 +2022-05-15 04:19:54.311689: validation loss: 0.0538 +2022-05-15 04:19:54.312351: Average global foreground Dice: [0.9511, 0.9409, 0.9307] +2022-05-15 04:19:54.312438: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 04:19:54.725074: lr: 6.2e-05 +2022-05-15 04:19:54.725274: saving scheduled checkpoint file... +2022-05-15 04:19:54.772104: saving checkpoint... +2022-05-15 04:19:55.561934: done, saving took 0.84 seconds +2022-05-15 04:19:55.580244: done +2022-05-15 04:20:10.765280: This epoch took 478.250743 s + +2022-05-15 04:20:10.765557: +epoch: 178 +2022-05-15 04:27:27.478707: train loss : 0.0368 +2022-05-15 04:27:52.487425: validation loss: 0.0572 +2022-05-15 04:27:52.488096: Average global foreground Dice: [0.9494, 0.9448, 0.9311] +2022-05-15 04:27:52.488183: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 04:27:52.901194: lr: 6e-05 +2022-05-15 04:27:52.901392: saving scheduled checkpoint file... +2022-05-15 04:27:52.948248: saving checkpoint... +2022-05-15 04:27:53.741985: done, saving took 0.84 seconds +2022-05-15 04:27:53.760360: done +2022-05-15 04:27:53.796436: saving checkpoint... +2022-05-15 04:27:54.567280: done, saving took 0.81 seconds +2022-05-15 04:28:11.232828: This epoch took 480.467175 s + +2022-05-15 04:28:11.233110: +epoch: 179 +2022-05-15 04:35:27.318634: train loss : 0.0388 +2022-05-15 04:35:52.318754: validation loss: 0.0521 +2022-05-15 04:35:52.319422: Average global foreground Dice: [0.9508, 0.9468, 0.9305] +2022-05-15 04:35:52.319506: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 04:35:52.733853: lr: 5.9e-05 +2022-05-15 04:35:52.734030: saving scheduled checkpoint file... +2022-05-15 04:35:52.781061: saving checkpoint... +2022-05-15 04:35:53.587622: done, saving took 0.85 seconds +2022-05-15 04:35:53.605816: done +2022-05-15 04:35:53.641833: saving checkpoint... +2022-05-15 04:35:54.412240: done, saving took 0.81 seconds +2022-05-15 04:36:13.074889: This epoch took 481.841682 s + +2022-05-15 04:36:13.075167: +epoch: 180 +2022-05-15 04:43:29.263151: train loss : 0.0383 +2022-05-15 04:43:54.221374: validation loss: 0.0594 +2022-05-15 04:43:54.222042: Average global foreground Dice: [0.9496, 0.9313, 0.9279] +2022-05-15 04:43:54.222129: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 04:43:54.635664: lr: 5.7e-05 +2022-05-15 04:43:54.636060: saving scheduled checkpoint file... +2022-05-15 04:43:54.682822: saving checkpoint... +2022-05-15 04:43:55.469608: done, saving took 0.83 seconds +2022-05-15 04:43:55.487576: done +2022-05-15 04:44:16.007973: This epoch took 482.932709 s + +2022-05-15 04:44:16.008251: +epoch: 181 +2022-05-15 04:51:32.669114: train loss : 0.0385 +2022-05-15 04:51:57.623455: validation loss: 0.0564 +2022-05-15 04:51:57.624135: Average global foreground Dice: [0.9498, 0.9344, 0.9271] +2022-05-15 04:51:57.624220: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 04:51:58.038802: lr: 5.6e-05 +2022-05-15 04:51:58.038999: saving scheduled checkpoint file... +2022-05-15 04:51:58.085780: saving checkpoint... +2022-05-15 04:51:58.875923: done, saving took 0.84 seconds +2022-05-15 04:51:58.896170: done +2022-05-15 04:52:16.754457: This epoch took 480.746088 s + +2022-05-15 04:52:16.758944: +epoch: 182 +2022-05-15 04:59:32.931169: train loss : 0.0381 +2022-05-15 04:59:57.990371: validation loss: 0.0484 +2022-05-15 04:59:57.991064: Average global foreground Dice: [0.9494, 0.9464, 0.9322] +2022-05-15 04:59:57.991151: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 04:59:58.403550: lr: 5.4e-05 +2022-05-15 04:59:58.403739: saving scheduled checkpoint file... +2022-05-15 04:59:58.450509: saving checkpoint... +2022-05-15 04:59:59.241539: done, saving took 0.84 seconds +2022-05-15 04:59:59.259792: done +2022-05-15 05:00:18.259141: This epoch took 481.500019 s + +2022-05-15 05:00:18.259543: +epoch: 183 +2022-05-15 05:07:34.661481: train loss : 0.0370 +2022-05-15 05:07:59.648005: validation loss: 0.0591 +2022-05-15 05:07:59.648664: Average global foreground Dice: [0.9489, 0.9399, 0.9299] +2022-05-15 05:07:59.648747: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 05:08:00.062843: lr: 5.3e-05 +2022-05-15 05:08:00.063097: saving scheduled checkpoint file... +2022-05-15 05:08:00.109875: saving checkpoint... +2022-05-15 05:08:00.853331: done, saving took 0.79 seconds +2022-05-15 05:08:00.870649: done +2022-05-15 05:08:19.352439: This epoch took 481.092794 s + +2022-05-15 05:08:19.352740: +epoch: 184 +2022-05-15 05:15:35.678447: train loss : 0.0376 +2022-05-15 05:16:00.727453: validation loss: 0.0620 +2022-05-15 05:16:00.728125: Average global foreground Dice: [0.9471, 0.9397, 0.9298] +2022-05-15 05:16:00.728228: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 05:16:01.171443: lr: 5.1e-05 +2022-05-15 05:16:01.171632: saving scheduled checkpoint file... +2022-05-15 05:16:01.219676: saving checkpoint... +2022-05-15 05:16:02.029815: done, saving took 0.86 seconds +2022-05-15 05:16:02.048021: done +2022-05-15 05:16:23.515511: This epoch took 484.162621 s + +2022-05-15 05:16:23.515859: +epoch: 185 +2022-05-15 05:23:40.453725: train loss : 0.0345 +2022-05-15 05:24:05.452700: validation loss: 0.0596 +2022-05-15 05:24:05.453374: Average global foreground Dice: [0.9497, 0.9366, 0.9284] +2022-05-15 05:24:05.453460: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 05:24:05.865479: lr: 5e-05 +2022-05-15 05:24:05.865675: saving scheduled checkpoint file... +2022-05-15 05:24:05.915596: saving checkpoint... +2022-05-15 05:24:06.711784: done, saving took 0.85 seconds +2022-05-15 05:24:06.729970: done +2022-05-15 05:24:22.652481: This epoch took 479.136515 s + +2022-05-15 05:24:22.652815: +epoch: 186 +2022-05-15 05:31:39.140901: train loss : 0.0355 +2022-05-15 05:32:04.207733: validation loss: 0.0541 +2022-05-15 05:32:04.208459: Average global foreground Dice: [0.9516, 0.9419, 0.9315] +2022-05-15 05:32:04.208546: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 05:32:04.627375: lr: 4.8e-05 +2022-05-15 05:32:04.627581: saving scheduled checkpoint file... +2022-05-15 05:32:04.675510: saving checkpoint... +2022-05-15 05:32:05.464867: done, saving took 0.84 seconds +2022-05-15 05:32:05.483059: done +2022-05-15 05:32:31.679582: This epoch took 489.026664 s + +2022-05-15 05:32:31.679907: +epoch: 187 +2022-05-15 05:39:48.752783: train loss : 0.0379 +2022-05-15 05:40:13.808682: validation loss: 0.0514 +2022-05-15 05:40:13.809299: Average global foreground Dice: [0.9476, 0.9366, 0.9283] +2022-05-15 05:40:13.809386: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 05:40:14.230055: lr: 4.7e-05 +2022-05-15 05:40:14.230258: saving scheduled checkpoint file... +2022-05-15 05:40:14.277216: saving checkpoint... +2022-05-15 05:40:15.070387: done, saving took 0.84 seconds +2022-05-15 05:40:15.088159: done +2022-05-15 05:40:38.307622: This epoch took 486.627555 s + +2022-05-15 05:40:38.308023: +epoch: 188 +2022-05-15 05:47:54.497450: train loss : 0.0384 +2022-05-15 05:48:19.563839: validation loss: 0.0473 +2022-05-15 05:48:19.564491: Average global foreground Dice: [0.9518, 0.9358, 0.9297] +2022-05-15 05:48:19.564574: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 05:48:19.976202: lr: 4.5e-05 +2022-05-15 05:48:19.976397: saving scheduled checkpoint file... +2022-05-15 05:48:20.023420: saving checkpoint... +2022-05-15 05:48:20.806468: done, saving took 0.83 seconds +2022-05-15 05:48:20.824553: done +2022-05-15 05:48:40.308890: This epoch took 482.000764 s + +2022-05-15 05:48:40.309173: +epoch: 189 +2022-05-15 05:55:56.493809: train loss : 0.0358 +2022-05-15 05:56:21.589512: validation loss: 0.0623 +2022-05-15 05:56:21.590193: Average global foreground Dice: [0.9487, 0.9407, 0.9269] +2022-05-15 05:56:21.590278: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 05:56:22.008951: lr: 4.4e-05 +2022-05-15 05:56:22.009144: saving scheduled checkpoint file... +2022-05-15 05:56:22.056037: saving checkpoint... +2022-05-15 05:56:22.850565: done, saving took 0.84 seconds +2022-05-15 05:56:22.869272: done +2022-05-15 05:56:43.186238: This epoch took 482.876965 s + +2022-05-15 05:56:43.186530: +epoch: 190 +2022-05-15 06:04:00.104120: train loss : 0.0361 +2022-05-15 06:04:25.178760: validation loss: 0.0455 +2022-05-15 06:04:25.179458: Average global foreground Dice: [0.9497, 0.9382, 0.9283] +2022-05-15 06:04:25.179545: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 06:04:25.593322: lr: 4.3e-05 +2022-05-15 06:04:25.593520: saving scheduled checkpoint file... +2022-05-15 06:04:25.640587: saving checkpoint... +2022-05-15 06:04:26.431679: done, saving took 0.84 seconds +2022-05-15 06:04:26.449412: done +2022-05-15 06:04:44.707568: This epoch took 481.520915 s + +2022-05-15 06:04:44.707963: +epoch: 191 +2022-05-15 06:12:00.559644: train loss : 0.0374 +2022-05-15 06:12:25.635231: validation loss: 0.0541 +2022-05-15 06:12:25.635849: Average global foreground Dice: [0.9505, 0.9459, 0.9317] +2022-05-15 06:12:25.635934: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 06:12:26.051282: lr: 4.1e-05 +2022-05-15 06:12:26.051475: saving scheduled checkpoint file... +2022-05-15 06:12:26.097778: saving checkpoint... +2022-05-15 06:12:26.883861: done, saving took 0.83 seconds +2022-05-15 06:12:26.902194: done +2022-05-15 06:12:45.223192: This epoch took 480.515129 s + +2022-05-15 06:12:45.223483: +epoch: 192 +2022-05-15 06:20:02.164677: train loss : 0.0354 +2022-05-15 06:20:27.280555: validation loss: 0.0549 +2022-05-15 06:20:27.281205: Average global foreground Dice: [0.9496, 0.9365, 0.9286] +2022-05-15 06:20:27.281291: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 06:20:27.699999: lr: 4e-05 +2022-05-15 06:20:27.700200: saving scheduled checkpoint file... +2022-05-15 06:20:27.747625: saving checkpoint... +2022-05-15 06:20:28.540768: done, saving took 0.84 seconds +2022-05-15 06:20:28.559166: done +2022-05-15 06:21:05.047514: This epoch took 499.823923 s + +2022-05-15 06:21:05.047804: +epoch: 193 +2022-05-15 06:28:21.592418: train loss : 0.0373 +2022-05-15 06:28:46.652100: validation loss: 0.0612 +2022-05-15 06:28:46.652716: Average global foreground Dice: [0.9509, 0.9354, 0.9244] +2022-05-15 06:28:46.652812: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 06:28:47.066801: lr: 3.9e-05 +2022-05-15 06:28:47.066989: saving scheduled checkpoint file... +2022-05-15 06:28:47.113502: saving checkpoint... +2022-05-15 06:28:47.901717: done, saving took 0.83 seconds +2022-05-15 06:28:47.920092: done +2022-05-15 06:29:06.931061: This epoch took 481.883136 s + +2022-05-15 06:29:06.931427: +epoch: 194 +2022-05-15 06:36:23.064392: train loss : 0.0374 +2022-05-15 06:36:48.128646: validation loss: 0.0690 +2022-05-15 06:36:48.129352: Average global foreground Dice: [0.953, 0.9382, 0.9318] +2022-05-15 06:36:48.129439: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 06:36:48.543020: lr: 3.7e-05 +2022-05-15 06:36:48.543216: saving scheduled checkpoint file... +2022-05-15 06:36:48.589943: saving checkpoint... +2022-05-15 06:36:49.373115: done, saving took 0.83 seconds +2022-05-15 06:36:49.391031: done +2022-05-15 06:37:06.029625: This epoch took 479.098067 s + +2022-05-15 06:37:06.030019: +epoch: 195 +2022-05-15 06:44:22.617691: train loss : 0.0380 +2022-05-15 06:44:47.679055: validation loss: 0.0536 +2022-05-15 06:44:47.679715: Average global foreground Dice: [0.9506, 0.9344, 0.9271] +2022-05-15 06:44:47.679811: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 06:44:48.094753: lr: 3.6e-05 +2022-05-15 06:44:48.094957: saving scheduled checkpoint file... +2022-05-15 06:44:48.143384: saving checkpoint... +2022-05-15 06:44:48.940079: done, saving took 0.85 seconds +2022-05-15 06:44:48.962256: done +2022-05-15 06:45:08.309449: This epoch took 482.279321 s + +2022-05-15 06:45:08.309753: +epoch: 196 +2022-05-15 06:52:25.378469: train loss : 0.0366 +2022-05-15 06:52:50.393142: validation loss: 0.0700 +2022-05-15 06:52:50.393784: Average global foreground Dice: [0.9514, 0.9414, 0.93] +2022-05-15 06:52:50.393872: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 06:52:50.814906: lr: 3.5e-05 +2022-05-15 06:52:50.815101: saving scheduled checkpoint file... +2022-05-15 06:52:50.862159: saving checkpoint... +2022-05-15 06:52:51.645187: done, saving took 0.83 seconds +2022-05-15 06:52:51.663362: done +2022-05-15 06:53:09.589101: This epoch took 481.279229 s + +2022-05-15 06:53:09.589381: +epoch: 197 +2022-05-15 07:00:26.906713: train loss : 0.0379 +2022-05-15 07:00:52.065401: validation loss: 0.0513 +2022-05-15 07:00:52.066046: Average global foreground Dice: [0.9479, 0.9425, 0.9287] +2022-05-15 07:00:52.066134: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 07:00:52.493184: lr: 3.3e-05 +2022-05-15 07:00:52.493387: saving scheduled checkpoint file... +2022-05-15 07:00:52.541221: saving checkpoint... +2022-05-15 07:00:53.350907: done, saving took 0.86 seconds +2022-05-15 07:00:53.369401: done +2022-05-15 07:01:20.355821: This epoch took 490.766324 s + +2022-05-15 07:01:20.356130: +epoch: 198 +2022-05-15 07:08:36.883409: train loss : 0.0390 +2022-05-15 07:09:01.924998: validation loss: 0.0690 +2022-05-15 07:09:01.925666: Average global foreground Dice: [0.9511, 0.9374, 0.9311] +2022-05-15 07:09:01.925750: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 07:09:02.345290: lr: 3.2e-05 +2022-05-15 07:09:02.345488: saving scheduled checkpoint file... +2022-05-15 07:09:02.392205: saving checkpoint... +2022-05-15 07:09:03.181648: done, saving took 0.84 seconds +2022-05-15 07:09:03.202110: done +2022-05-15 07:09:33.185658: This epoch took 492.829429 s + +2022-05-15 07:09:33.185941: +epoch: 199 +2022-05-15 07:16:50.001328: train loss : 0.0377 +2022-05-15 07:17:15.015690: validation loss: 0.0555 +2022-05-15 07:17:15.016345: Average global foreground Dice: [0.9508, 0.9424, 0.9289] +2022-05-15 07:17:15.016429: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 07:17:15.432564: lr: 3.1e-05 +2022-05-15 07:17:15.432757: saving scheduled checkpoint file... +2022-05-15 07:17:15.481675: saving checkpoint... +2022-05-15 07:17:16.236111: done, saving took 0.80 seconds +2022-05-15 07:17:16.255075: done +2022-05-15 07:17:47.857340: This epoch took 494.671293 s + +2022-05-15 07:17:47.857649: +epoch: 200 +2022-05-15 07:25:04.623857: train loss : 0.0365 +2022-05-15 07:25:29.673550: validation loss: 0.0573 +2022-05-15 07:25:29.674264: Average global foreground Dice: [0.948, 0.9361, 0.9268] +2022-05-15 07:25:29.674350: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 07:25:30.089585: lr: 3e-05 +2022-05-15 07:25:30.089785: saving scheduled checkpoint file... +2022-05-15 07:25:30.136808: saving checkpoint... +2022-05-15 07:25:30.806631: done, saving took 0.72 seconds +2022-05-15 07:25:30.825062: done +2022-05-15 07:25:53.188282: This epoch took 485.330522 s + +2022-05-15 07:25:53.188598: +epoch: 201 +2022-05-15 07:33:09.417203: train loss : 0.0353 +2022-05-15 07:33:34.451126: validation loss: 0.0544 +2022-05-15 07:33:34.451886: Average global foreground Dice: [0.949, 0.9422, 0.9316] +2022-05-15 07:33:34.451973: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 07:33:34.878640: lr: 2.9e-05 +2022-05-15 07:33:34.878891: saving scheduled checkpoint file... +2022-05-15 07:33:34.926513: saving checkpoint... +2022-05-15 07:33:35.647847: done, saving took 0.77 seconds +2022-05-15 07:33:35.666151: done +2022-05-15 07:33:55.980317: This epoch took 482.791614 s + +2022-05-15 07:33:55.980609: +epoch: 202 +2022-05-15 07:41:12.779930: train loss : 0.0387 +2022-05-15 07:41:37.825313: validation loss: 0.0580 +2022-05-15 07:41:37.825963: Average global foreground Dice: [0.9511, 0.942, 0.9315] +2022-05-15 07:41:37.826049: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 07:41:38.242188: lr: 2.8e-05 +2022-05-15 07:41:38.242377: saving scheduled checkpoint file... +2022-05-15 07:41:38.289116: saving checkpoint... +2022-05-15 07:41:39.086599: done, saving took 0.84 seconds +2022-05-15 07:41:39.104712: done +2022-05-15 07:42:01.059632: This epoch took 485.078923 s + +2022-05-15 07:42:01.060004: +epoch: 203 +2022-05-15 07:49:18.223371: train loss : 0.0352 +2022-05-15 07:49:43.261824: validation loss: 0.0686 +2022-05-15 07:49:43.262417: Average global foreground Dice: [0.9463, 0.9416, 0.9255] +2022-05-15 07:49:43.262502: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 07:49:43.686027: lr: 2.6e-05 +2022-05-15 07:49:43.686220: saving scheduled checkpoint file... +2022-05-15 07:49:43.733942: saving checkpoint... +2022-05-15 07:49:44.564969: done, saving took 0.88 seconds +2022-05-15 07:49:44.585479: done +2022-05-15 07:50:03.766089: This epoch took 482.705958 s + +2022-05-15 07:50:03.766373: +epoch: 204 +2022-05-15 07:57:20.449440: train loss : 0.0350 +2022-05-15 07:57:45.471576: validation loss: 0.0460 +2022-05-15 07:57:45.472250: Average global foreground Dice: [0.9502, 0.9368, 0.9342] +2022-05-15 07:57:45.472344: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 07:57:45.888508: lr: 2.5e-05 +2022-05-15 07:57:45.888710: saving scheduled checkpoint file... +2022-05-15 07:57:45.935868: saving checkpoint... +2022-05-15 07:57:46.732497: done, saving took 0.84 seconds +2022-05-15 07:57:46.753060: done +2022-05-15 07:58:05.943812: This epoch took 482.177330 s + +2022-05-15 07:58:05.944113: +epoch: 205 +2022-05-15 08:05:22.352556: train loss : 0.0355 +2022-05-15 08:05:47.368978: validation loss: 0.0535 +2022-05-15 08:05:47.369611: Average global foreground Dice: [0.9517, 0.938, 0.929] +2022-05-15 08:05:47.369695: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 08:05:47.780387: lr: 2.4e-05 +2022-05-15 08:05:47.780584: saving scheduled checkpoint file... +2022-05-15 08:05:47.827589: saving checkpoint... +2022-05-15 08:05:48.634305: done, saving took 0.85 seconds +2022-05-15 08:05:48.652544: done +2022-05-15 08:06:07.732303: This epoch took 481.788090 s + +2022-05-15 08:06:07.732593: +epoch: 206 +2022-05-15 08:13:24.098077: train loss : 0.0347 +2022-05-15 08:13:49.153274: validation loss: 0.0527 +2022-05-15 08:13:49.153939: Average global foreground Dice: [0.9479, 0.9368, 0.9295] +2022-05-15 08:13:49.154026: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 08:13:49.561471: lr: 2.3e-05 +2022-05-15 08:13:49.561678: saving scheduled checkpoint file... +2022-05-15 08:13:49.608380: saving checkpoint... +2022-05-15 08:13:50.401644: done, saving took 0.84 seconds +2022-05-15 08:13:50.420013: done +2022-05-15 08:14:11.294346: This epoch took 483.561655 s + +2022-05-15 08:14:11.294649: +epoch: 207 +2022-05-15 08:21:27.937582: train loss : 0.0356 +2022-05-15 08:21:53.002533: validation loss: 0.0682 +2022-05-15 08:21:53.003379: Average global foreground Dice: [0.9493, 0.9377, 0.9277] +2022-05-15 08:21:53.003465: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 08:21:53.426435: lr: 2.2e-05 +2022-05-15 08:21:53.426633: saving scheduled checkpoint file... +2022-05-15 08:21:53.473579: saving checkpoint... +2022-05-15 08:21:54.265558: done, saving took 0.84 seconds +2022-05-15 08:21:54.283329: done +2022-05-15 08:22:15.671429: This epoch took 484.376680 s + +2022-05-15 08:22:15.671692: +epoch: 208 +2022-05-15 08:29:32.392127: train loss : 0.0354 +2022-05-15 08:29:57.493464: validation loss: 0.0551 +2022-05-15 08:29:57.494197: Average global foreground Dice: [0.9515, 0.941, 0.9314] +2022-05-15 08:29:57.494280: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 08:29:57.903213: lr: 2.1e-05 +2022-05-15 08:29:57.903503: saving scheduled checkpoint file... +2022-05-15 08:29:57.950478: saving checkpoint... +2022-05-15 08:29:58.740255: done, saving took 0.84 seconds +2022-05-15 08:29:58.758564: done +2022-05-15 08:30:28.797053: This epoch took 493.125261 s + +2022-05-15 08:30:28.797364: +epoch: 209 +2022-05-15 08:37:45.013705: train loss : 0.0356 +2022-05-15 08:38:10.085698: validation loss: 0.0503 +2022-05-15 08:38:10.086404: Average global foreground Dice: [0.9502, 0.9425, 0.9306] +2022-05-15 08:38:10.086503: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 08:38:10.494452: lr: 2e-05 +2022-05-15 08:38:10.494639: saving scheduled checkpoint file... +2022-05-15 08:38:10.540821: saving checkpoint... +2022-05-15 08:38:11.323673: done, saving took 0.83 seconds +2022-05-15 08:38:11.341891: done +2022-05-15 08:38:29.255023: This epoch took 480.457560 s + +2022-05-15 08:38:29.255311: +epoch: 210 +2022-05-15 08:45:45.602606: train loss : 0.0361 +2022-05-15 08:46:10.667334: validation loss: 0.0565 +2022-05-15 08:46:10.667992: Average global foreground Dice: [0.9538, 0.9429, 0.9293] +2022-05-15 08:46:10.668080: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 08:46:11.079633: lr: 1.9e-05 +2022-05-15 08:46:11.079824: saving scheduled checkpoint file... +2022-05-15 08:46:11.126870: saving checkpoint... +2022-05-15 08:46:11.930846: done, saving took 0.85 seconds +2022-05-15 08:46:11.949431: done +2022-05-15 08:46:28.105727: This epoch took 478.850314 s + +2022-05-15 08:46:28.106025: +epoch: 211 +2022-05-15 08:53:44.683482: train loss : 0.0369 +2022-05-15 08:54:09.772566: validation loss: 0.0488 +2022-05-15 08:54:09.773173: Average global foreground Dice: [0.9506, 0.9474, 0.9315] +2022-05-15 08:54:09.773258: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 08:54:10.184360: lr: 1.8e-05 +2022-05-15 08:54:10.184546: saving scheduled checkpoint file... +2022-05-15 08:54:10.231390: saving checkpoint... +2022-05-15 08:54:11.025731: done, saving took 0.84 seconds +2022-05-15 08:54:11.043683: done +2022-05-15 08:54:32.361690: This epoch took 484.255558 s + +2022-05-15 08:54:32.362044: +epoch: 212 +2022-05-15 09:01:48.658725: train loss : 0.0341 +2022-05-15 09:02:13.724348: validation loss: 0.0529 +2022-05-15 09:02:13.725077: Average global foreground Dice: [0.9507, 0.9373, 0.9299] +2022-05-15 09:02:13.725190: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 09:02:14.134785: lr: 1.7e-05 +2022-05-15 09:02:14.134974: saving scheduled checkpoint file... +2022-05-15 09:02:14.182050: saving checkpoint... +2022-05-15 09:02:14.972979: done, saving took 0.84 seconds +2022-05-15 09:02:14.990956: done +2022-05-15 09:02:31.089490: This epoch took 478.727231 s + +2022-05-15 09:02:31.089775: +epoch: 213 +2022-05-15 09:09:47.164367: train loss : 0.0378 +2022-05-15 09:10:12.191376: validation loss: 0.0592 +2022-05-15 09:10:12.192036: Average global foreground Dice: [0.9503, 0.942, 0.9308] +2022-05-15 09:10:12.192124: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 09:10:12.607548: lr: 1.6e-05 +2022-05-15 09:10:12.607740: saving scheduled checkpoint file... +2022-05-15 09:10:12.654908: saving checkpoint... +2022-05-15 09:10:13.391607: done, saving took 0.78 seconds +2022-05-15 09:10:13.410335: done +2022-05-15 09:10:34.165918: This epoch took 483.076038 s + +2022-05-15 09:10:34.166220: +epoch: 214 +2022-05-15 09:17:50.612534: train loss : 0.0350 +2022-05-15 09:18:15.667271: validation loss: 0.0530 +2022-05-15 09:18:15.667927: Average global foreground Dice: [0.9477, 0.9394, 0.9299] +2022-05-15 09:18:15.668013: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 09:18:16.085915: lr: 1.5e-05 +2022-05-15 09:18:16.086111: saving scheduled checkpoint file... +2022-05-15 09:18:16.133277: saving checkpoint... +2022-05-15 09:18:16.925702: done, saving took 0.84 seconds +2022-05-15 09:18:16.943971: done +2022-05-15 09:18:35.327778: This epoch took 481.161445 s + +2022-05-15 09:18:35.328048: +epoch: 215 +2022-05-15 09:25:51.451024: train loss : 0.0357 +2022-05-15 09:26:16.399152: validation loss: 0.0428 +2022-05-15 09:26:16.399784: Average global foreground Dice: [0.9505, 0.9514, 0.9341] +2022-05-15 09:26:16.399870: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 09:26:16.840156: lr: 1.5e-05 +2022-05-15 09:26:16.840342: saving scheduled checkpoint file... +2022-05-15 09:26:16.886228: saving checkpoint... +2022-05-15 09:26:17.683297: done, saving took 0.84 seconds +2022-05-15 09:26:17.702119: done +2022-05-15 09:26:47.119522: This epoch took 491.791376 s + +2022-05-15 09:26:47.119811: +epoch: 216 +2022-05-15 09:34:03.847133: train loss : 0.0367 +2022-05-15 09:34:28.879015: validation loss: 0.0597 +2022-05-15 09:34:28.879655: Average global foreground Dice: [0.9497, 0.9502, 0.9317] +2022-05-15 09:34:28.879740: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 09:34:29.290979: lr: 1.4e-05 +2022-05-15 09:34:29.291167: saving scheduled checkpoint file... +2022-05-15 09:34:29.337838: saving checkpoint... +2022-05-15 09:34:30.131534: done, saving took 0.84 seconds +2022-05-15 09:34:30.150269: done +2022-05-15 09:34:48.147624: This epoch took 481.027709 s + +2022-05-15 09:34:48.147941: +epoch: 217 +2022-05-15 09:42:04.562085: train loss : 0.0383 +2022-05-15 09:42:29.694092: validation loss: 0.0588 +2022-05-15 09:42:29.694687: Average global foreground Dice: [0.9507, 0.9384, 0.928] +2022-05-15 09:42:29.694784: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 09:42:30.104758: lr: 1.3e-05 +2022-05-15 09:42:30.104951: saving scheduled checkpoint file... +2022-05-15 09:42:30.152777: saving checkpoint... +2022-05-15 09:42:30.847728: done, saving took 0.74 seconds +2022-05-15 09:42:30.864714: done +2022-05-15 09:42:47.799082: This epoch took 479.651036 s + +2022-05-15 09:42:47.799342: +epoch: 218 +2022-05-15 09:50:04.556695: train loss : 0.0357 +2022-05-15 09:50:29.533455: validation loss: 0.0584 +2022-05-15 09:50:29.534126: Average global foreground Dice: [0.952, 0.9442, 0.9307] +2022-05-15 09:50:29.534213: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 09:50:29.942655: lr: 1.2e-05 +2022-05-15 09:50:29.942879: saving scheduled checkpoint file... +2022-05-15 09:50:29.989855: saving checkpoint... +2022-05-15 09:50:30.786961: done, saving took 0.84 seconds +2022-05-15 09:50:30.805372: done +2022-05-15 09:50:51.861170: This epoch took 484.061732 s + +2022-05-15 09:50:51.861435: +epoch: 219 +2022-05-15 09:58:08.895478: train loss : 0.0363 +2022-05-15 09:58:33.935189: validation loss: 0.0548 +2022-05-15 09:58:33.935827: Average global foreground Dice: [0.9534, 0.9472, 0.9305] +2022-05-15 09:58:33.935915: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 09:58:34.345487: lr: 1.1e-05 +2022-05-15 09:58:34.345692: saving scheduled checkpoint file... +2022-05-15 09:58:34.392567: saving checkpoint... +2022-05-15 09:58:35.185444: done, saving took 0.84 seconds +2022-05-15 09:58:35.203791: done +2022-05-15 09:58:52.499126: This epoch took 480.637570 s + +2022-05-15 09:58:52.499479: +epoch: 220 +2022-05-15 10:06:08.975894: train loss : 0.0372 +2022-05-15 10:06:33.995570: validation loss: 0.0474 +2022-05-15 10:06:33.996249: Average global foreground Dice: [0.9498, 0.9446, 0.9306] +2022-05-15 10:06:33.996340: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 10:06:34.411229: lr: 1.1e-05 +2022-05-15 10:06:34.411417: saving scheduled checkpoint file... +2022-05-15 10:06:34.458472: saving checkpoint... +2022-05-15 10:06:35.248888: done, saving took 0.84 seconds +2022-05-15 10:06:35.266776: done +2022-05-15 10:06:53.330915: This epoch took 480.831334 s + +2022-05-15 10:06:53.331219: +epoch: 221 +2022-05-15 10:14:09.827392: train loss : 0.0359 +2022-05-15 10:14:34.909974: validation loss: 0.0479 +2022-05-15 10:14:34.910678: Average global foreground Dice: [0.9501, 0.9391, 0.928] +2022-05-15 10:14:34.910773: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 10:14:35.322449: lr: 1e-05 +2022-05-15 10:14:35.322634: saving scheduled checkpoint file... +2022-05-15 10:14:35.369583: saving checkpoint... +2022-05-15 10:14:36.166536: done, saving took 0.84 seconds +2022-05-15 10:14:36.184369: done +2022-05-15 10:14:55.816447: This epoch took 482.485102 s + +2022-05-15 10:14:55.816837: +epoch: 222 +2022-05-15 10:22:11.954964: train loss : 0.0370 +2022-05-15 10:22:37.006546: validation loss: 0.0527 +2022-05-15 10:22:37.007188: Average global foreground Dice: [0.9515, 0.9414, 0.9262] +2022-05-15 10:22:37.007274: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 10:22:37.421170: lr: 9e-06 +2022-05-15 10:22:37.421368: saving scheduled checkpoint file... +2022-05-15 10:22:37.468995: saving checkpoint... +2022-05-15 10:22:38.388209: done, saving took 0.97 seconds +2022-05-15 10:22:38.406220: done +2022-05-15 10:22:57.106333: This epoch took 481.289392 s + +2022-05-15 10:22:57.106619: +epoch: 223 +2022-05-15 10:30:13.329265: train loss : 0.0352 +2022-05-15 10:30:38.359057: validation loss: 0.0600 +2022-05-15 10:30:38.359735: Average global foreground Dice: [0.9481, 0.9434, 0.9206] +2022-05-15 10:30:38.359838: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 10:30:38.769418: lr: 9e-06 +2022-05-15 10:30:38.769609: saving scheduled checkpoint file... +2022-05-15 10:30:38.816795: saving checkpoint... +2022-05-15 10:30:39.616200: done, saving took 0.85 seconds +2022-05-15 10:30:39.634240: done +2022-05-15 10:30:56.997312: This epoch took 479.890592 s + +2022-05-15 10:30:56.997588: +epoch: 224 +2022-05-15 10:38:13.146920: train loss : 0.0365 +2022-05-15 10:38:38.169647: validation loss: 0.0475 +2022-05-15 10:38:38.170420: Average global foreground Dice: [0.9489, 0.9406, 0.9275] +2022-05-15 10:38:38.170509: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 10:38:38.586655: lr: 8e-06 +2022-05-15 10:38:38.586865: saving scheduled checkpoint file... +2022-05-15 10:38:38.636226: saving checkpoint... +2022-05-15 10:38:39.465404: done, saving took 0.88 seconds +2022-05-15 10:38:39.486517: done +2022-05-15 10:38:56.438716: This epoch took 479.441002 s + +2022-05-15 10:38:56.439103: +epoch: 225 +2022-05-15 10:46:12.188656: train loss : 0.0355 +2022-05-15 10:46:37.243943: validation loss: 0.0490 +2022-05-15 10:46:37.244577: Average global foreground Dice: [0.9513, 0.9384, 0.9305] +2022-05-15 10:46:37.244663: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 10:46:37.656395: lr: 7e-06 +2022-05-15 10:46:37.656585: saving scheduled checkpoint file... +2022-05-15 10:46:37.703958: saving checkpoint... +2022-05-15 10:46:38.506580: done, saving took 0.85 seconds +2022-05-15 10:46:38.524599: done +2022-05-15 10:47:04.196985: This epoch took 487.757650 s + +2022-05-15 10:47:04.197317: +epoch: 226 +2022-05-15 10:54:20.282807: train loss : 0.0364 +2022-05-15 10:54:45.381594: validation loss: 0.0514 +2022-05-15 10:54:45.382242: Average global foreground Dice: [0.9484, 0.9429, 0.9295] +2022-05-15 10:54:45.382325: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 10:54:45.793927: lr: 7e-06 +2022-05-15 10:54:45.794119: saving scheduled checkpoint file... +2022-05-15 10:54:45.841645: saving checkpoint... +2022-05-15 10:54:46.658997: done, saving took 0.86 seconds +2022-05-15 10:54:46.680109: done +2022-05-15 10:55:05.496599: This epoch took 481.299167 s + +2022-05-15 10:55:05.496987: +epoch: 227 +2022-05-15 11:02:22.075240: train loss : 0.0356 +2022-05-15 11:02:47.157446: validation loss: 0.0426 +2022-05-15 11:02:47.158094: Average global foreground Dice: [0.9519, 0.9403, 0.9318] +2022-05-15 11:02:47.158181: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 11:02:47.574427: lr: 6e-06 +2022-05-15 11:02:47.574623: saving scheduled checkpoint file... +2022-05-15 11:02:47.620428: saving checkpoint... +2022-05-15 11:02:48.432800: done, saving took 0.86 seconds +2022-05-15 11:02:48.454313: done +2022-05-15 11:03:05.771488: This epoch took 480.274268 s + +2022-05-15 11:03:05.771920: +epoch: 228 +2022-05-15 11:10:23.013326: train loss : 0.0368 +2022-05-15 11:10:48.057128: validation loss: 0.0475 +2022-05-15 11:10:48.057803: Average global foreground Dice: [0.952, 0.9484, 0.9341] +2022-05-15 11:10:48.057891: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 11:10:48.473554: lr: 6e-06 +2022-05-15 11:10:48.473743: saving scheduled checkpoint file... +2022-05-15 11:10:48.520487: saving checkpoint... +2022-05-15 11:10:49.372106: done, saving took 0.90 seconds +2022-05-15 11:10:49.393308: done +2022-05-15 11:11:07.866802: This epoch took 482.094770 s + +2022-05-15 11:11:07.867074: +epoch: 229 +2022-05-15 11:18:24.355258: train loss : 0.0373 +2022-05-15 11:18:49.494519: validation loss: 0.0647 +2022-05-15 11:18:49.495147: Average global foreground Dice: [0.9515, 0.9453, 0.9301] +2022-05-15 11:18:49.495233: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 11:18:49.904379: lr: 5e-06 +2022-05-15 11:18:49.904563: saving scheduled checkpoint file... +2022-05-15 11:18:49.951284: saving checkpoint... +2022-05-15 11:18:50.794012: done, saving took 0.89 seconds +2022-05-15 11:18:50.815466: done +2022-05-15 11:19:19.093160: This epoch took 491.225988 s + +2022-05-15 11:19:19.093431: +epoch: 230 +2022-05-15 11:26:35.198373: train loss : 0.0359 +2022-05-15 11:27:00.267601: validation loss: 0.0638 +2022-05-15 11:27:00.268224: Average global foreground Dice: [0.948, 0.9365, 0.9306] +2022-05-15 11:27:00.268513: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 11:27:00.676125: lr: 5e-06 +2022-05-15 11:27:00.676313: saving scheduled checkpoint file... +2022-05-15 11:27:00.723008: saving checkpoint... +2022-05-15 11:27:01.521789: done, saving took 0.85 seconds +2022-05-15 11:27:01.542808: done +2022-05-15 11:27:19.079613: This epoch took 479.986051 s + +2022-05-15 11:27:19.079996: +epoch: 231 +2022-05-15 11:34:35.687178: train loss : 0.0350 +2022-05-15 11:35:00.761118: validation loss: 0.0484 +2022-05-15 11:35:00.761733: Average global foreground Dice: [0.9517, 0.9369, 0.9278] +2022-05-15 11:35:00.761830: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 11:35:01.172337: lr: 4e-06 +2022-05-15 11:35:01.172526: saving scheduled checkpoint file... +2022-05-15 11:35:01.218806: saving checkpoint... +2022-05-15 11:35:02.042608: done, saving took 0.87 seconds +2022-05-15 11:35:02.067501: done +2022-05-15 11:35:35.373867: This epoch took 496.293770 s + +2022-05-15 11:35:35.374134: +epoch: 232 +2022-05-15 11:42:51.957855: train loss : 0.0375 +2022-05-15 11:43:17.056653: validation loss: 0.0543 +2022-05-15 11:43:17.057298: Average global foreground Dice: [0.9503, 0.9391, 0.9306] +2022-05-15 11:43:17.057384: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 11:43:17.465323: lr: 4e-06 +2022-05-15 11:43:17.465518: saving scheduled checkpoint file... +2022-05-15 11:43:17.512287: saving checkpoint... +2022-05-15 11:43:18.268646: done, saving took 0.80 seconds +2022-05-15 11:43:18.289531: done +2022-05-15 11:43:49.086555: This epoch took 493.712314 s + +2022-05-15 11:43:49.086877: +epoch: 233 +2022-05-15 11:51:05.725490: train loss : 0.0347 +2022-05-15 11:51:30.764705: validation loss: 0.0601 +2022-05-15 11:51:30.765346: Average global foreground Dice: [0.9497, 0.9402, 0.9293] +2022-05-15 11:51:30.765433: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 11:51:31.179630: lr: 3e-06 +2022-05-15 11:51:31.179839: saving scheduled checkpoint file... +2022-05-15 11:51:31.226722: saving checkpoint... +2022-05-15 11:51:31.945496: done, saving took 0.77 seconds +2022-05-15 11:51:31.962883: done +2022-05-15 11:51:50.570177: This epoch took 481.483202 s + +2022-05-15 11:51:50.570467: +epoch: 234 +2022-05-15 11:59:06.410841: train loss : 0.0363 +2022-05-15 11:59:31.437429: validation loss: 0.0634 +2022-05-15 11:59:31.438091: Average global foreground Dice: [0.9504, 0.9462, 0.9297] +2022-05-15 11:59:31.438178: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 11:59:31.868901: lr: 3e-06 +2022-05-15 11:59:31.869098: saving scheduled checkpoint file... +2022-05-15 11:59:31.916160: saving checkpoint... +2022-05-15 11:59:32.649309: done, saving took 0.78 seconds +2022-05-15 11:59:32.666572: done +2022-05-15 11:59:52.555755: This epoch took 481.985193 s + +2022-05-15 11:59:52.556098: +epoch: 235 +2022-05-15 12:07:08.472756: train loss : 0.0371 +2022-05-15 12:07:33.534543: validation loss: 0.0702 +2022-05-15 12:07:33.535176: Average global foreground Dice: [0.9519, 0.9393, 0.9325] +2022-05-15 12:07:33.535259: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 12:07:33.944695: lr: 3e-06 +2022-05-15 12:07:33.944973: saving scheduled checkpoint file... +2022-05-15 12:07:33.994891: saving checkpoint... +2022-05-15 12:07:34.682889: done, saving took 0.74 seconds +2022-05-15 12:07:34.700891: done +2022-05-15 12:07:51.306510: This epoch took 478.750312 s + +2022-05-15 12:07:51.306797: +epoch: 236 +2022-05-15 12:15:07.830737: train loss : 0.0358 +2022-05-15 12:15:32.911986: validation loss: 0.0686 +2022-05-15 12:15:32.912656: Average global foreground Dice: [0.9505, 0.9402, 0.9273] +2022-05-15 12:15:32.912742: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 12:15:33.326995: lr: 2e-06 +2022-05-15 12:15:33.327199: saving scheduled checkpoint file... +2022-05-15 12:15:33.374777: saving checkpoint... +2022-05-15 12:15:34.100826: done, saving took 0.77 seconds +2022-05-15 12:15:34.119888: done +2022-05-15 12:15:51.583551: This epoch took 480.276655 s + +2022-05-15 12:15:51.583868: +epoch: 237 +2022-05-15 12:23:08.011205: train loss : 0.0355 +2022-05-15 12:23:33.062945: validation loss: 0.0723 +2022-05-15 12:23:33.063568: Average global foreground Dice: [0.9517, 0.9353, 0.931] +2022-05-15 12:23:33.063652: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 12:23:33.476362: lr: 2e-06 +2022-05-15 12:23:33.476550: saving scheduled checkpoint file... +2022-05-15 12:23:33.524485: saving checkpoint... +2022-05-15 12:23:34.229998: done, saving took 0.75 seconds +2022-05-15 12:23:34.247062: done +2022-05-15 12:23:57.756073: This epoch took 486.172107 s + +2022-05-15 12:23:57.756350: +epoch: 238 +2022-05-15 12:31:14.429707: train loss : 0.0344 +2022-05-15 12:31:39.447923: validation loss: 0.0535 +2022-05-15 12:31:39.448647: Average global foreground Dice: [0.9517, 0.9416, 0.9339] +2022-05-15 12:31:39.448732: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 12:31:39.862340: lr: 2e-06 +2022-05-15 12:31:39.862535: saving scheduled checkpoint file... +2022-05-15 12:31:39.911523: saving checkpoint... +2022-05-15 12:31:40.706607: done, saving took 0.84 seconds +2022-05-15 12:31:40.724601: done +2022-05-15 12:32:11.861950: This epoch took 494.105500 s + +2022-05-15 12:32:11.862246: +epoch: 239 +2022-05-15 12:39:28.539908: train loss : 0.0382 +2022-05-15 12:39:53.546256: validation loss: 0.0533 +2022-05-15 12:39:53.546890: Average global foreground Dice: [0.9539, 0.9412, 0.9292] +2022-05-15 12:39:53.546977: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 12:39:53.958026: lr: 1e-06 +2022-05-15 12:39:53.958235: saving scheduled checkpoint file... +2022-05-15 12:39:54.005035: saving checkpoint... +2022-05-15 12:39:54.804649: done, saving took 0.85 seconds +2022-05-15 12:39:54.822751: done +2022-05-15 12:40:12.539886: This epoch took 480.677510 s + +2022-05-15 12:40:12.540262: +epoch: 240 +2022-05-15 12:47:29.200259: train loss : 0.0358 +2022-05-15 12:47:54.176923: validation loss: 0.0641 +2022-05-15 12:47:54.177540: Average global foreground Dice: [0.9497, 0.9324, 0.9298] +2022-05-15 12:47:54.177675: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 12:47:54.591315: lr: 1e-06 +2022-05-15 12:47:54.591509: saving scheduled checkpoint file... +2022-05-15 12:47:54.638425: saving checkpoint... +2022-05-15 12:47:55.454665: done, saving took 0.86 seconds +2022-05-15 12:47:55.478839: done +2022-05-15 12:48:26.468259: This epoch took 493.927880 s + +2022-05-15 12:48:26.468652: +epoch: 241 +2022-05-15 12:55:43.410592: train loss : 0.0348 +2022-05-15 12:56:08.462093: validation loss: 0.0565 +2022-05-15 12:56:08.462743: Average global foreground Dice: [0.952, 0.9392, 0.9303] +2022-05-15 12:56:08.462841: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 12:56:08.878296: lr: 1e-06 +2022-05-15 12:56:08.878501: saving scheduled checkpoint file... +2022-05-15 12:56:08.925411: saving checkpoint... +2022-05-15 12:56:09.762654: done, saving took 0.88 seconds +2022-05-15 12:56:09.805789: done +2022-05-15 12:56:27.656587: This epoch took 481.187759 s + +2022-05-15 12:56:27.656887: +epoch: 242 +2022-05-15 13:03:45.091522: train loss : 0.0352 +2022-05-15 13:04:10.138222: validation loss: 0.0450 +2022-05-15 13:04:10.138906: Average global foreground Dice: [0.9462, 0.9379, 0.9278] +2022-05-15 13:04:10.138993: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 13:04:10.551841: lr: 1e-06 +2022-05-15 13:04:10.552026: saving scheduled checkpoint file... +2022-05-15 13:04:10.598786: saving checkpoint... +2022-05-15 13:04:11.419547: done, saving took 0.87 seconds +2022-05-15 13:04:11.455838: done +2022-05-15 13:04:27.872573: This epoch took 480.215588 s + +2022-05-15 13:04:27.872864: +epoch: 243 +2022-05-15 13:11:44.894445: train loss : 0.0358 +2022-05-15 13:12:09.974352: validation loss: 0.0582 +2022-05-15 13:12:09.975064: Average global foreground Dice: [0.9509, 0.9448, 0.9258] +2022-05-15 13:12:09.975152: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 13:12:10.395665: lr: 0.0 +2022-05-15 13:12:10.395860: saving scheduled checkpoint file... +2022-05-15 13:12:10.442697: saving checkpoint... +2022-05-15 13:12:11.251539: done, saving took 0.86 seconds +2022-05-15 13:12:11.269711: done +2022-05-15 13:12:34.087932: This epoch took 486.214972 s + +2022-05-15 13:12:34.088222: +epoch: 244 +2022-05-15 13:19:51.113577: train loss : 0.0375 +2022-05-15 13:20:16.190429: validation loss: 0.0581 +2022-05-15 13:20:16.191068: Average global foreground Dice: [0.9503, 0.9452, 0.935] +2022-05-15 13:20:16.191159: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 13:20:16.606323: lr: 0.0 +2022-05-15 13:20:16.606601: saving scheduled checkpoint file... +2022-05-15 13:20:16.653714: saving checkpoint... +2022-05-15 13:20:17.478326: done, saving took 0.87 seconds +2022-05-15 13:20:17.499371: done +2022-05-15 13:20:35.969367: This epoch took 481.881047 s + +2022-05-15 13:20:35.969663: +epoch: 245 +2022-05-15 13:27:52.532751: train loss : 0.0366 +2022-05-15 13:28:17.555146: validation loss: 0.0493 +2022-05-15 13:28:17.555772: Average global foreground Dice: [0.9546, 0.9449, 0.9308] +2022-05-15 13:28:17.555864: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 13:28:17.968218: lr: 0.0 +2022-05-15 13:28:17.968637: saving scheduled checkpoint file... +2022-05-15 13:28:18.015841: saving checkpoint... +2022-05-15 13:28:18.829876: done, saving took 0.86 seconds +2022-05-15 13:28:18.857482: done +2022-05-15 13:28:37.924272: This epoch took 481.954510 s + +2022-05-15 13:28:37.924557: +epoch: 246 +2022-05-15 13:35:54.825795: train loss : 0.0365 +2022-05-15 13:36:19.902726: validation loss: 0.0578 +2022-05-15 13:36:19.903350: Average global foreground Dice: [0.9513, 0.9379, 0.9299] +2022-05-15 13:36:19.903435: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 13:36:20.315878: lr: 0.0 +2022-05-15 13:36:20.316065: saving scheduled checkpoint file... +2022-05-15 13:36:20.367619: saving checkpoint... +2022-05-15 13:36:21.165187: done, saving took 0.85 seconds +2022-05-15 13:36:21.183493: done +2022-05-15 13:36:40.650695: This epoch took 482.726041 s + +2022-05-15 13:36:40.650996: +epoch: 247 +2022-05-15 13:43:57.094302: train loss : 0.0350 +2022-05-15 13:44:22.092733: validation loss: 0.0634 +2022-05-15 13:44:22.093472: Average global foreground Dice: [0.9512, 0.9467, 0.9322] +2022-05-15 13:44:22.093558: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 13:44:22.507324: lr: 0.0 +2022-05-15 13:44:22.507522: saving scheduled checkpoint file... +2022-05-15 13:44:22.555250: saving checkpoint... +2022-05-15 13:44:23.353757: done, saving took 0.85 seconds +2022-05-15 13:44:23.372736: done +2022-05-15 13:44:39.223735: This epoch took 478.572625 s + +2022-05-15 13:44:39.224083: +epoch: 248 +2022-05-15 13:51:56.005352: train loss : 0.0361 +2022-05-15 13:52:21.073042: validation loss: 0.0536 +2022-05-15 13:52:21.073681: Average global foreground Dice: [0.9504, 0.9419, 0.9308] +2022-05-15 13:52:21.073777: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 13:52:21.488823: lr: 0.0 +2022-05-15 13:52:21.489010: saving scheduled checkpoint file... +2022-05-15 13:52:21.537257: saving checkpoint... +2022-05-15 13:52:22.342726: done, saving took 0.85 seconds +2022-05-15 13:52:22.361502: done +2022-05-15 13:52:38.332743: This epoch took 479.108554 s + +2022-05-15 13:52:38.333041: +epoch: 249 +2022-05-15 13:59:55.136463: train loss : 0.0346 +2022-05-15 14:00:20.163216: validation loss: 0.0715 +2022-05-15 14:00:20.163825: Average global foreground Dice: [0.9513, 0.9326, 0.9305] +2022-05-15 14:00:20.163910: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 14:00:20.577992: lr: 0.0 +2022-05-15 14:00:20.578167: saving scheduled checkpoint file... +2022-05-15 14:00:20.625237: saving checkpoint... +2022-05-15 14:00:21.415169: done, saving took 0.84 seconds +2022-05-15 14:00:21.433365: done +2022-05-15 14:00:43.676719: This epoch took 485.343544 s + +2022-05-15 14:00:43.739480: saving checkpoint... +2022-05-15 14:00:44.386481: done, saving took 0.71 seconds diff --git a/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_1/debug.json b/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_1/debug.json new file mode 100644 index 0000000000000000000000000000000000000000..b99e0c2d0c93b2a34fa65d8a6f3bb1d7b1a1c3bb --- /dev/null +++ b/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_1/debug.json @@ -0,0 +1,107 @@ +{ + "all_tr_losses": "[]", + "all_val_eval_metrics": "[]", + "all_val_losses": "[]", + "all_val_losses_tr_mode": "[]", + "also_val_in_tr_mode": "False", + "amp_grad_scaler": "None", + "args": "Namespace(batch_size=2, config='./configs/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250.yaml', continue_training=False, crop_size=[128, 128, 128], dbs=False, deep_supervision_scales=[], deterministic=False, disable_decoder=False, disable_ds=True, disable_postprocessing_on_folds=False, disable_saving=False, drop_path=0.0, find_lr=False, find_zero_weight_decay=False, fix_ds_net_numpool=False, fold='1', fp32=False, hdfs_base='GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250', initial_lr=0.0003, is_sigmoid=False, is_spatial_aug_only=False, is_ssl=False, layer_decay=1.0, local_rank=0, loss_name='', lrschedule='warmup_cosine', mask_ratio=0.75, max_loss_cal='v1', max_num_epochs=250, merge_femur=False, min_lr=0, model='Generic_TransUNet_max_ppbp', model_params={'is_masked_attn': True, 'max_dec_layers': 3, 'is_max_bottleneck_transformer': False, 'max_msda': '', 'is_max_ms': True, 'max_ms_idxs': [-4, -3, -2], 'max_hidden_dim': 192, 'mw': 1.0, 'is_max_ds': True, 'is_masking': True, 'is_max_hungarian': True, 'num_queries': 20, 'is_max_cls': True, 'is_mhsa_float32': True}, n_class=17, network='3d_fullres', network_trainer='nnUNetTrainerV2_DDP', npz=False, opt_betas=None, opt_eps=1e-08, optim_name='adamw', p='nnUNetPlansv2.1', plan_update='', pretrained=False, pretrained_config=None, pretrained_weights=None, rank=0, reclip=None, resume='auto', skip_grad_nan=False, task='Task500_BraTS2021', task_pretrained='Task801_WORD', total_batch_size=None, use_compressed_data=False, val_final=False, val_folder='validation_raw', valbest=False, validation_only=False, vallatest=False, warmup_epochs=10, weight_decay=3e-05, world_size=8)", + "base_num_features": "32", + "basic_generator_patch_size": "[182 182 182]", + "batch_dice": "True", + "batch_size": "2", + "best_MA_tr_loss_for_patience": "None", + "best_epoch_based_on_MA_tr_loss": "None", + "best_val_eval_criterion_MA": "None", + "classes": "[1, 2, 3]", + "conv_per_stage": "2", + "data_aug_params": "{'selected_data_channels': None, 'selected_seg_channels': [0], 'do_elastic': True, 'elastic_deform_alpha': (0.0, 900.0), 'elastic_deform_sigma': (9.0, 13.0), 'p_eldef': 0.3, 'do_scaling': True, 'scale_range': (0.65, 1.6), 'independent_scale_factor_for_each_axis': True, 'p_independent_scale_per_axis': 0.3, 'p_scale': 0.3, 'do_rotation': True, 'rotation_x': (-1.5707963267948966, 1.5707963267948966), 'rotation_y': (-1.5707963267948966, 1.5707963267948966), 'rotation_z': (-1.5707963267948966, 1.5707963267948966), 'rotation_p_per_axis': 1, 'p_rot': 0.3, 'random_crop': False, 'random_crop_dist_to_border': None, 'do_gamma': True, 'gamma_retain_stats': True, 'gamma_range': (0.5, 1.6), 'p_gamma': 0.3, 'do_mirror': True, 'mirror_axes': (0, 1, 2), 'dummy_2D': False, 'mask_was_used_for_normalization': OrderedDict([(0, True), (1, True), (2, True), (3, True)]), 'border_mode_data': 'constant', 'all_segmentation_labels': None, 'move_last_seg_chanel_to_data': False, 'cascade_do_cascade_augmentations': False, 'cascade_random_binary_transform_p': 0.4, 'cascade_random_binary_transform_p_per_label': 1, 'cascade_random_binary_transform_size': (1, 8), 'cascade_remove_conn_comp_p': 0.2, 'cascade_remove_conn_comp_max_size_percent_threshold': 0.15, 'cascade_remove_conn_comp_fill_with_other_class_p': 0.0, 'do_additive_brightness': True, 'additive_brightness_p_per_sample': 0.3, 'additive_brightness_p_per_channel': 0.5, 'additive_brightness_mu': 0, 'additive_brightness_sigma': 0.2, 'num_threads': 12, 'num_cached_per_thread': 4, 'patch_size_for_spatialtransform': array([128, 128, 128]), 'eldef_deformation_scale': (0, 0.25)}", + "dataset_directory": "/opt/tiger/project/data/nnUNet_preprocessed/Task500_BraTS2021", + "deep_supervision_scales": "[[1, 1, 1], [0.5, 0.5, 0.5], [0.25, 0.25, 0.25], [0.125, 0.125, 0.125], [0.0625, 0.0625, 0.0625]]", + "deterministic": "False", + "disable_ds": "True", + "distribute_batch_size": "False", + "dl_tr": "", + "dl_val": "", + "do_dummy_2D_aug": "False", + "ds_loss_weights": "[1. 0. 0. 0. 0.]", + "epoch": "0", + "experiment_name": "nnUNetTrainerV2_DDP", + "find_zero_weight_decay": "False", + "fold": "1", + "folder_with_preprocessed_data": "/opt/tiger/project/data/nnUNet_preprocessed/Task500_BraTS2021/nnUNetData_plans_v2.1_stage0", + "fp16": "True", + "global_batch_size": "16", + "gt_niftis_folder": "/opt/tiger/project/data/nnUNet_preprocessed/Task500_BraTS2021/gt_segmentations", + "inference_pad_border_mode": "constant", + "inference_pad_kwargs": "{'constant_values': 0}", + "init_args": "('/opt/tiger/project/data/nnUNet_preprocessed/Task500_BraTS2021/nnUNetPlansv2.1_plans_3D.pkl', 1, 0, '/opt/tiger/project/data/nnUNet_trained_models/UNet_IN_NANFang/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250', '/opt/tiger/project/data/nnUNet_preprocessed/Task500_BraTS2021', True, 0, True, False, False, True)", + "initial_lr": "0.0003", + "input_size": "[128, 128, 128]", + "is_spatial_aug_only": "False", + "is_ssl": "False", + "layer_decay": "1.0", + "local_rank": "0", + "log_file": "/opt/tiger/project/data/nnUNet_trained_models/UNet_IN_NANFang/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_1/training_log_2022_5_14_04_20_21.txt", + "loss": "None", + "lr_scheduler": "", + "lr_scheduler_eps": "0.001", + "lr_scheduler_name": "warmup_cosine", + "lr_scheduler_patience": "30", + "lr_threshold": "1e-06", + "max_num_epochs": "250", + "min_lr": "0", + "min_region_size_per_class": "None", + "min_size_per_class": "None", + "model": "Generic_TransUNet_max_ppbp", + "model_params": "{'is_masked_attn': True, 'max_dec_layers': 3, 'is_max_bottleneck_transformer': False, 'max_msda': '', 'is_max_ms': True, 'max_ms_idxs': [-4, -3, -2], 'max_hidden_dim': 192, 'mw': 1.0, 'is_max_ds': True, 'is_masking': True, 'is_max_hungarian': True, 'num_queries': 20, 'is_max_cls': True, 'is_mhsa_float32': True}", + "net_conv_kernel_sizes": "[[3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3]]", + "net_num_pool_op_kernel_sizes": "[[2, 2, 2], [2, 2, 2], [2, 2, 2], [2, 2, 2], [2, 2, 2]]", + "net_pool_per_axis": "[5, 5, 5]", + "normalization_schemes": "OrderedDict([(0, 'nonCT'), (1, 'nonCT'), (2, 'nonCT'), (3, 'nonCT')])", + "num_batches_per_epoch": "250", + "num_classes": "3", + "num_input_channels": "4", + "num_val_batches_per_epoch": "50", + "online_eval_fn": "[]", + "online_eval_foreground_dc": "[]", + "online_eval_fp": "[]", + "online_eval_tp": "[]", + "only_keep_largest_connected_component": "None", + "optim_name": "adamw", + "optimizer": "AdamW (\nParameter Group 0\n amsgrad: False\n betas: (0.9, 0.999)\n eps: 1e-08\n initial_lr: 0.0003\n lr: 0.0\n weight_decay: 3e-05\n)", + "output_folder": "/opt/tiger/project/data/nnUNet_trained_models/UNet_IN_NANFang/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_1", + "output_folder_base": "/opt/tiger/project/data/nnUNet_trained_models/UNet_IN_NANFang/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250", + "oversample_foreground_percent": "0.0", + "pad_all_sides": "None", + "patch_size": "[128 128 128]", + "patience": "50", + "pin_memory": "True", + "plans_file": "/opt/tiger/project/data/nnUNet_preprocessed/Task500_BraTS2021/nnUNetPlansv2.1_plans_3D.pkl", + "reclip": "None", + "regions": "{'whole tumor': (1, 2, 3), 'tumor core': (2, 3), 'enhancing tumor': (3,)}", + "regions_class_order": "(1, 2, 3)", + "resume": "auto", + "save_best_checkpoint": "True", + "save_every": "1", + "save_final_checkpoint": "True", + "save_intermediate_checkpoints": "True", + "save_latest_only": "True", + "stage": "0", + "threeD": "True", + "tr_gen": "", + "train_loss_MA": "None", + "train_loss_MA_alpha": "0.93", + "train_loss_MA_eps": "0.0005", + "transpose_backward": "[0, 1, 2]", + "transpose_forward": "[0, 1, 2]", + "unpack_data": "True", + "use_mask_for_norm": "OrderedDict([(0, True), (1, True), (2, True), (3, True)])", + "use_progress_bar": "False", + "val_eval_criterion_MA": "None", + "val_eval_criterion_alpha": "0.9", + "val_gen": "", + "warmup_epochs": "10", + "was_initialized": "True", + "weight_decay": "3e-05" +} \ No newline at end of file diff --git a/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_1/model_best.model b/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_1/model_best.model new file mode 100644 index 0000000000000000000000000000000000000000..933d861f8904814ccc32bce241b920655340f95d --- /dev/null +++ b/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_1/model_best.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fcf7ae0945f3c309d9d8bcc2427057a6dbbdabd40b746ab8b83d8441704419b7 +size 409152161 diff --git a/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_1/model_best.model.pkl b/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_1/model_best.model.pkl new file mode 100644 index 0000000000000000000000000000000000000000..fc0245c85798b9d00b6efb30b41122957508d0e0 --- /dev/null +++ b/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_1/model_best.model.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bcbf0cad40c9f52aa6c69346ceac041cb927b505ed8738c52e53ea90fa0cac14 +size 393843 diff --git a/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_1/model_final_checkpoint.model b/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_1/model_final_checkpoint.model new file mode 100644 index 0000000000000000000000000000000000000000..8492fbb5f8a6d9a47d233136d1c57218c3e5fb2c --- /dev/null +++ b/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_1/model_final_checkpoint.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:93c6d518f69173328cb453c211bd7b07fc645f286435fe3220886c45bcaf2421 +size 409163936 diff --git a/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_1/model_final_checkpoint.model.pkl b/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_1/model_final_checkpoint.model.pkl new file mode 100644 index 0000000000000000000000000000000000000000..fc0245c85798b9d00b6efb30b41122957508d0e0 --- /dev/null +++ b/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_1/model_final_checkpoint.model.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bcbf0cad40c9f52aa6c69346ceac041cb927b505ed8738c52e53ea90fa0cac14 +size 393843 diff --git a/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_1/model_latest.model b/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_1/model_latest.model new file mode 100644 index 0000000000000000000000000000000000000000..00faaad3a04fa5b0048523faaae43d0e64e2e985 --- /dev/null +++ b/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_1/model_latest.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb2e12d5a0f298b356564a902607ec77683fedcf897bcc774318415c4999a507 +size 409163936 diff --git a/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_1/model_latest.model.pkl b/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_1/model_latest.model.pkl new file mode 100644 index 0000000000000000000000000000000000000000..fc0245c85798b9d00b6efb30b41122957508d0e0 --- /dev/null +++ b/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_1/model_latest.model.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bcbf0cad40c9f52aa6c69346ceac041cb927b505ed8738c52e53ea90fa0cac14 +size 393843 diff --git a/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_1/progress.png b/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_1/progress.png new file mode 100644 index 0000000000000000000000000000000000000000..12960e116de90596c9dd267518427a48438d2ca0 --- /dev/null +++ b/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_1/progress.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5eeacab1fd001aa3c1e3e828dd782071c6a6779f8977bf8ccb4558e86770bfcb +size 329117 diff --git a/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_1/training_log_2022_5_14_04_20_21.txt b/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_1/training_log_2022_5_14_04_20_21.txt new file mode 100644 index 0000000000000000000000000000000000000000..75c4555f52d34cea4c74d104877ce0348beaef37 --- /dev/null +++ b/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_1/training_log_2022_5_14_04_20_21.txt @@ -0,0 +1,3416 @@ +Starting... +2022-05-14 04:20:21.373828: Using splits from existing split file: /opt/tiger/project/data/nnUNet_preprocessed/Task500_BraTS2021/splits_final.pkl +2022-05-14 04:20:21.374275: The split file contains 5 splits. +2022-05-14 04:20:21.374327: Desired fold for training: 1 +2022-05-14 04:20:21.374366: This split has 1001 training and 250 validation cases. +2022-05-14 04:21:46.675149: TRAINING KEYS: + odict_keys(['BraTS2021_00000', 'BraTS2021_00002', 'BraTS2021_00003', 'BraTS2021_00008', 'BraTS2021_00009', 'BraTS2021_00014', 'BraTS2021_00016', 'BraTS2021_00017', 'BraTS2021_00018', 'BraTS2021_00019', 'BraTS2021_00022', 'BraTS2021_00024', 'BraTS2021_00025', 'BraTS2021_00026', 'BraTS2021_00028', 'BraTS2021_00030', 'BraTS2021_00031', 'BraTS2021_00032', 'BraTS2021_00033', 'BraTS2021_00035', 'BraTS2021_00043', 'BraTS2021_00044', 'BraTS2021_00045', 'BraTS2021_00046', 'BraTS2021_00048', 'BraTS2021_00049', 'BraTS2021_00051', 'BraTS2021_00052', 'BraTS2021_00054', 'BraTS2021_00056', 'BraTS2021_00058', 'BraTS2021_00059', 'BraTS2021_00060', 'BraTS2021_00062', 'BraTS2021_00063', 'BraTS2021_00064', 'BraTS2021_00066', 'BraTS2021_00068', 'BraTS2021_00070', 'BraTS2021_00071', 'BraTS2021_00072', 'BraTS2021_00074', 'BraTS2021_00077', 'BraTS2021_00078', 'BraTS2021_00085', 'BraTS2021_00087', 'BraTS2021_00088', 'BraTS2021_00089', 'BraTS2021_00094', 'BraTS2021_00096', 'BraTS2021_00098', 'BraTS2021_00099', 'BraTS2021_00100', 'BraTS2021_00102', 'BraTS2021_00103', 'BraTS2021_00104', 'BraTS2021_00105', 'BraTS2021_00106', 'BraTS2021_00107', 'BraTS2021_00108', 'BraTS2021_00110', 'BraTS2021_00111', 'BraTS2021_00112', 'BraTS2021_00113', 'BraTS2021_00115', 'BraTS2021_00116', 'BraTS2021_00117', 'BraTS2021_00118', 'BraTS2021_00120', 'BraTS2021_00121', 'BraTS2021_00122', 'BraTS2021_00123', 'BraTS2021_00124', 'BraTS2021_00126', 'BraTS2021_00127', 'BraTS2021_00128', 'BraTS2021_00130', 'BraTS2021_00131', 'BraTS2021_00132', 'BraTS2021_00134', 'BraTS2021_00136', 'BraTS2021_00137', 'BraTS2021_00138', 'BraTS2021_00139', 'BraTS2021_00140', 'BraTS2021_00142', 'BraTS2021_00143', 'BraTS2021_00144', 'BraTS2021_00146', 'BraTS2021_00148', 'BraTS2021_00149', 'BraTS2021_00150', 'BraTS2021_00151', 'BraTS2021_00152', 'BraTS2021_00154', 'BraTS2021_00155', 'BraTS2021_00156', 'BraTS2021_00158', 'BraTS2021_00160', 'BraTS2021_00165', 'BraTS2021_00166', 'BraTS2021_00167', 'BraTS2021_00172', 'BraTS2021_00176', 'BraTS2021_00183', 'BraTS2021_00185', 'BraTS2021_00186', 'BraTS2021_00187', 'BraTS2021_00192', 'BraTS2021_00193', 'BraTS2021_00194', 'BraTS2021_00195', 'BraTS2021_00196', 'BraTS2021_00199', 'BraTS2021_00201', 'BraTS2021_00203', 'BraTS2021_00206', 'BraTS2021_00209', 'BraTS2021_00211', 'BraTS2021_00212', 'BraTS2021_00214', 'BraTS2021_00216', 'BraTS2021_00217', 'BraTS2021_00218', 'BraTS2021_00219', 'BraTS2021_00221', 'BraTS2021_00227', 'BraTS2021_00228', 'BraTS2021_00230', 'BraTS2021_00231', 'BraTS2021_00235', 'BraTS2021_00236', 'BraTS2021_00238', 'BraTS2021_00239', 'BraTS2021_00240', 'BraTS2021_00241', 'BraTS2021_00242', 'BraTS2021_00243', 'BraTS2021_00246', 'BraTS2021_00249', 'BraTS2021_00250', 'BraTS2021_00251', 'BraTS2021_00253', 'BraTS2021_00254', 'BraTS2021_00259', 'BraTS2021_00261', 'BraTS2021_00262', 'BraTS2021_00266', 'BraTS2021_00269', 'BraTS2021_00270', 'BraTS2021_00271', 'BraTS2021_00273', 'BraTS2021_00274', 'BraTS2021_00275', 'BraTS2021_00280', 'BraTS2021_00283', 'BraTS2021_00284', 'BraTS2021_00285', 'BraTS2021_00286', 'BraTS2021_00288', 'BraTS2021_00291', 'BraTS2021_00292', 'BraTS2021_00294', 'BraTS2021_00296', 'BraTS2021_00297', 'BraTS2021_00298', 'BraTS2021_00299', 'BraTS2021_00300', 'BraTS2021_00301', 'BraTS2021_00304', 'BraTS2021_00305', 'BraTS2021_00306', 'BraTS2021_00309', 'BraTS2021_00311', 'BraTS2021_00313', 'BraTS2021_00314', 'BraTS2021_00316', 'BraTS2021_00317', 'BraTS2021_00318', 'BraTS2021_00320', 'BraTS2021_00321', 'BraTS2021_00322', 'BraTS2021_00324', 'BraTS2021_00325', 'BraTS2021_00327', 'BraTS2021_00328', 'BraTS2021_00329', 'BraTS2021_00332', 'BraTS2021_00334', 'BraTS2021_00336', 'BraTS2021_00338', 'BraTS2021_00340', 'BraTS2021_00346', 'BraTS2021_00347', 'BraTS2021_00348', 'BraTS2021_00349', 'BraTS2021_00350', 'BraTS2021_00351', 'BraTS2021_00352', 'BraTS2021_00353', 'BraTS2021_00356', 'BraTS2021_00359', 'BraTS2021_00364', 'BraTS2021_00366', 'BraTS2021_00367', 'BraTS2021_00369', 'BraTS2021_00370', 'BraTS2021_00371', 'BraTS2021_00373', 'BraTS2021_00375', 'BraTS2021_00376', 'BraTS2021_00378', 'BraTS2021_00379', 'BraTS2021_00380', 'BraTS2021_00382', 'BraTS2021_00383', 'BraTS2021_00386', 'BraTS2021_00387', 'BraTS2021_00388', 'BraTS2021_00389', 'BraTS2021_00390', 'BraTS2021_00391', 'BraTS2021_00392', 'BraTS2021_00395', 'BraTS2021_00399', 'BraTS2021_00400', 'BraTS2021_00401', 'BraTS2021_00402', 'BraTS2021_00403', 'BraTS2021_00404', 'BraTS2021_00405', 'BraTS2021_00406', 'BraTS2021_00407', 'BraTS2021_00409', 'BraTS2021_00410', 'BraTS2021_00412', 'BraTS2021_00413', 'BraTS2021_00414', 'BraTS2021_00416', 'BraTS2021_00418', 'BraTS2021_00419', 'BraTS2021_00421', 'BraTS2021_00423', 'BraTS2021_00425', 'BraTS2021_00426', 'BraTS2021_00429', 'BraTS2021_00430', 'BraTS2021_00431', 'BraTS2021_00432', 'BraTS2021_00433', 'BraTS2021_00436', 'BraTS2021_00441', 'BraTS2021_00442', 'BraTS2021_00443', 'BraTS2021_00444', 'BraTS2021_00446', 'BraTS2021_00448', 'BraTS2021_00449', 'BraTS2021_00451', 'BraTS2021_00452', 'BraTS2021_00454', 'BraTS2021_00456', 'BraTS2021_00457', 'BraTS2021_00459', 'BraTS2021_00464', 'BraTS2021_00469', 'BraTS2021_00470', 'BraTS2021_00472', 'BraTS2021_00478', 'BraTS2021_00479', 'BraTS2021_00480', 'BraTS2021_00481', 'BraTS2021_00483', 'BraTS2021_00485', 'BraTS2021_00491', 'BraTS2021_00493', 'BraTS2021_00494', 'BraTS2021_00495', 'BraTS2021_00496', 'BraTS2021_00498', 'BraTS2021_00499', 'BraTS2021_00500', 'BraTS2021_00501', 'BraTS2021_00502', 'BraTS2021_00504', 'BraTS2021_00505', 'BraTS2021_00507', 'BraTS2021_00510', 'BraTS2021_00511', 'BraTS2021_00512', 'BraTS2021_00513', 'BraTS2021_00514', 'BraTS2021_00516', 'BraTS2021_00517', 'BraTS2021_00519', 'BraTS2021_00523', 'BraTS2021_00524', 'BraTS2021_00526', 'BraTS2021_00528', 'BraTS2021_00530', 'BraTS2021_00532', 'BraTS2021_00533', 'BraTS2021_00537', 'BraTS2021_00538', 'BraTS2021_00539', 'BraTS2021_00540', 'BraTS2021_00542', 'BraTS2021_00543', 'BraTS2021_00544', 'BraTS2021_00548', 'BraTS2021_00549', 'BraTS2021_00550', 'BraTS2021_00551', 'BraTS2021_00554', 'BraTS2021_00556', 'BraTS2021_00558', 'BraTS2021_00561', 'BraTS2021_00563', 'BraTS2021_00565', 'BraTS2021_00567', 'BraTS2021_00568', 'BraTS2021_00569', 'BraTS2021_00572', 'BraTS2021_00574', 'BraTS2021_00575', 'BraTS2021_00577', 'BraTS2021_00578', 'BraTS2021_00581', 'BraTS2021_00582', 'BraTS2021_00583', 'BraTS2021_00584', 'BraTS2021_00586', 'BraTS2021_00587', 'BraTS2021_00588', 'BraTS2021_00589', 'BraTS2021_00590', 'BraTS2021_00591', 'BraTS2021_00596', 'BraTS2021_00597', 'BraTS2021_00601', 'BraTS2021_00602', 'BraTS2021_00604', 'BraTS2021_00605', 'BraTS2021_00606', 'BraTS2021_00607', 'BraTS2021_00608', 'BraTS2021_00611', 'BraTS2021_00613', 'BraTS2021_00618', 'BraTS2021_00619', 'BraTS2021_00620', 'BraTS2021_00621', 'BraTS2021_00622', 'BraTS2021_00623', 'BraTS2021_00624', 'BraTS2021_00625', 'BraTS2021_00626', 'BraTS2021_00630', 'BraTS2021_00636', 'BraTS2021_00638', 'BraTS2021_00639', 'BraTS2021_00641', 'BraTS2021_00646', 'BraTS2021_00649', 'BraTS2021_00650', 'BraTS2021_00651', 'BraTS2021_00652', 'BraTS2021_00654', 'BraTS2021_00655', 'BraTS2021_00656', 'BraTS2021_00659', 'BraTS2021_00667', 'BraTS2021_00668', 'BraTS2021_00674', 'BraTS2021_00676', 'BraTS2021_00677', 'BraTS2021_00679', 'BraTS2021_00680', 'BraTS2021_00682', 'BraTS2021_00683', 'BraTS2021_00684', 'BraTS2021_00685', 'BraTS2021_00686', 'BraTS2021_00687', 'BraTS2021_00689', 'BraTS2021_00690', 'BraTS2021_00691', 'BraTS2021_00693', 'BraTS2021_00694', 'BraTS2021_00697', 'BraTS2021_00698', 'BraTS2021_00704', 'BraTS2021_00705', 'BraTS2021_00706', 'BraTS2021_00707', 'BraTS2021_00708', 'BraTS2021_00709', 'BraTS2021_00714', 'BraTS2021_00715', 'BraTS2021_00716', 'BraTS2021_00723', 'BraTS2021_00724', 'BraTS2021_00725', 'BraTS2021_00727', 'BraTS2021_00728', 'BraTS2021_00729', 'BraTS2021_00732', 'BraTS2021_00733', 'BraTS2021_00734', 'BraTS2021_00735', 'BraTS2021_00737', 'BraTS2021_00739', 'BraTS2021_00740', 'BraTS2021_00742', 'BraTS2021_00747', 'BraTS2021_00750', 'BraTS2021_00751', 'BraTS2021_00753', 'BraTS2021_00756', 'BraTS2021_00757', 'BraTS2021_00758', 'BraTS2021_00759', 'BraTS2021_00760', 'BraTS2021_00764', 'BraTS2021_00765', 'BraTS2021_00767', 'BraTS2021_00768', 'BraTS2021_00772', 'BraTS2021_00773', 'BraTS2021_00774', 'BraTS2021_00775', 'BraTS2021_00777', 'BraTS2021_00780', 'BraTS2021_00781', 'BraTS2021_00782', 'BraTS2021_00787', 'BraTS2021_00789', 'BraTS2021_00791', 'BraTS2021_00792', 'BraTS2021_00793', 'BraTS2021_00796', 'BraTS2021_00797', 'BraTS2021_00799', 'BraTS2021_00800', 'BraTS2021_00801', 'BraTS2021_00802', 'BraTS2021_00803', 'BraTS2021_00804', 'BraTS2021_00805', 'BraTS2021_00806', 'BraTS2021_00807', 'BraTS2021_00808', 'BraTS2021_00810', 'BraTS2021_00816', 'BraTS2021_00818', 'BraTS2021_00820', 'BraTS2021_00823', 'BraTS2021_00824', 'BraTS2021_00828', 'BraTS2021_00830', 'BraTS2021_00831', 'BraTS2021_00834', 'BraTS2021_00836', 'BraTS2021_00838', 'BraTS2021_00839', 'BraTS2021_00840', 'BraTS2021_00999', 'BraTS2021_01000', 'BraTS2021_01001', 'BraTS2021_01004', 'BraTS2021_01005', 'BraTS2021_01007', 'BraTS2021_01008', 'BraTS2021_01010', 'BraTS2021_01011', 'BraTS2021_01012', 'BraTS2021_01013', 'BraTS2021_01014', 'BraTS2021_01015', 'BraTS2021_01016', 'BraTS2021_01017', 'BraTS2021_01018', 'BraTS2021_01021', 'BraTS2021_01022', 'BraTS2021_01023', 'BraTS2021_01024', 'BraTS2021_01025', 'BraTS2021_01026', 'BraTS2021_01029', 'BraTS2021_01030', 'BraTS2021_01031', 'BraTS2021_01032', 'BraTS2021_01033', 'BraTS2021_01035', 'BraTS2021_01036', 'BraTS2021_01037', 'BraTS2021_01038', 'BraTS2021_01039', 'BraTS2021_01042', 'BraTS2021_01043', 'BraTS2021_01044', 'BraTS2021_01046', 'BraTS2021_01047', 'BraTS2021_01048', 'BraTS2021_01049', 'BraTS2021_01050', 'BraTS2021_01051', 'BraTS2021_01052', 'BraTS2021_01055', 'BraTS2021_01056', 'BraTS2021_01057', 'BraTS2021_01058', 'BraTS2021_01059', 'BraTS2021_01060', 'BraTS2021_01062', 'BraTS2021_01063', 'BraTS2021_01064', 'BraTS2021_01065', 'BraTS2021_01066', 'BraTS2021_01069', 'BraTS2021_01070', 'BraTS2021_01071', 'BraTS2021_01072', 'BraTS2021_01073', 'BraTS2021_01074', 'BraTS2021_01075', 'BraTS2021_01076', 'BraTS2021_01077', 'BraTS2021_01078', 'BraTS2021_01079', 'BraTS2021_01080', 'BraTS2021_01081', 'BraTS2021_01082', 'BraTS2021_01083', 'BraTS2021_01085', 'BraTS2021_01087', 'BraTS2021_01088', 'BraTS2021_01089', 'BraTS2021_01090', 'BraTS2021_01091', 'BraTS2021_01093', 'BraTS2021_01094', 'BraTS2021_01095', 'BraTS2021_01096', 'BraTS2021_01097', 'BraTS2021_01099', 'BraTS2021_01100', 'BraTS2021_01101', 'BraTS2021_01103', 'BraTS2021_01104', 'BraTS2021_01105', 'BraTS2021_01106', 'BraTS2021_01107', 'BraTS2021_01108', 'BraTS2021_01109', 'BraTS2021_01110', 'BraTS2021_01111', 'BraTS2021_01113', 'BraTS2021_01115', 'BraTS2021_01116', 'BraTS2021_01117', 'BraTS2021_01118', 'BraTS2021_01120', 'BraTS2021_01121', 'BraTS2021_01122', 'BraTS2021_01123', 'BraTS2021_01124', 'BraTS2021_01126', 'BraTS2021_01127', 'BraTS2021_01128', 'BraTS2021_01129', 'BraTS2021_01130', 'BraTS2021_01131', 'BraTS2021_01132', 'BraTS2021_01133', 'BraTS2021_01134', 'BraTS2021_01135', 'BraTS2021_01136', 'BraTS2021_01137', 'BraTS2021_01138', 'BraTS2021_01139', 'BraTS2021_01141', 'BraTS2021_01142', 'BraTS2021_01143', 'BraTS2021_01145', 'BraTS2021_01146', 'BraTS2021_01147', 'BraTS2021_01149', 'BraTS2021_01150', 'BraTS2021_01151', 'BraTS2021_01152', 'BraTS2021_01153', 'BraTS2021_01154', 'BraTS2021_01156', 'BraTS2021_01157', 'BraTS2021_01158', 'BraTS2021_01159', 'BraTS2021_01160', 'BraTS2021_01161', 'BraTS2021_01162', 'BraTS2021_01163', 'BraTS2021_01164', 'BraTS2021_01167', 'BraTS2021_01168', 'BraTS2021_01169', 'BraTS2021_01170', 'BraTS2021_01171', 'BraTS2021_01173', 'BraTS2021_01174', 'BraTS2021_01175', 'BraTS2021_01176', 'BraTS2021_01177', 'BraTS2021_01178', 'BraTS2021_01179', 'BraTS2021_01180', 'BraTS2021_01181', 'BraTS2021_01182', 'BraTS2021_01183', 'BraTS2021_01185', 'BraTS2021_01187', 'BraTS2021_01189', 'BraTS2021_01190', 'BraTS2021_01191', 'BraTS2021_01192', 'BraTS2021_01195', 'BraTS2021_01196', 'BraTS2021_01197', 'BraTS2021_01198', 'BraTS2021_01201', 'BraTS2021_01202', 'BraTS2021_01203', 'BraTS2021_01204', 'BraTS2021_01205', 'BraTS2021_01206', 'BraTS2021_01207', 'BraTS2021_01208', 'BraTS2021_01209', 'BraTS2021_01210', 'BraTS2021_01211', 'BraTS2021_01212', 'BraTS2021_01213', 'BraTS2021_01214', 'BraTS2021_01216', 'BraTS2021_01217', 'BraTS2021_01218', 'BraTS2021_01221', 'BraTS2021_01222', 'BraTS2021_01223', 'BraTS2021_01225', 'BraTS2021_01226', 'BraTS2021_01227', 'BraTS2021_01230', 'BraTS2021_01231', 'BraTS2021_01235', 'BraTS2021_01236', 'BraTS2021_01237', 'BraTS2021_01238', 'BraTS2021_01239', 'BraTS2021_01240', 'BraTS2021_01241', 'BraTS2021_01242', 'BraTS2021_01243', 'BraTS2021_01244', 'BraTS2021_01247', 'BraTS2021_01248', 'BraTS2021_01250', 'BraTS2021_01251', 'BraTS2021_01253', 'BraTS2021_01255', 'BraTS2021_01256', 'BraTS2021_01260', 'BraTS2021_01261', 'BraTS2021_01262', 'BraTS2021_01263', 'BraTS2021_01265', 'BraTS2021_01266', 'BraTS2021_01267', 'BraTS2021_01268', 'BraTS2021_01270', 'BraTS2021_01271', 'BraTS2021_01273', 'BraTS2021_01274', 'BraTS2021_01275', 'BraTS2021_01276', 'BraTS2021_01277', 'BraTS2021_01278', 'BraTS2021_01280', 'BraTS2021_01282', 'BraTS2021_01283', 'BraTS2021_01284', 'BraTS2021_01285', 'BraTS2021_01286', 'BraTS2021_01289', 'BraTS2021_01290', 'BraTS2021_01291', 'BraTS2021_01292', 'BraTS2021_01293', 'BraTS2021_01294', 'BraTS2021_01296', 'BraTS2021_01298', 'BraTS2021_01299', 'BraTS2021_01300', 'BraTS2021_01301', 'BraTS2021_01302', 'BraTS2021_01304', 'BraTS2021_01305', 'BraTS2021_01306', 'BraTS2021_01307', 'BraTS2021_01308', 'BraTS2021_01309', 'BraTS2021_01310', 'BraTS2021_01312', 'BraTS2021_01313', 'BraTS2021_01314', 'BraTS2021_01315', 'BraTS2021_01316', 'BraTS2021_01317', 'BraTS2021_01318', 'BraTS2021_01319', 'BraTS2021_01320', 'BraTS2021_01321', 'BraTS2021_01322', 'BraTS2021_01324', 'BraTS2021_01325', 'BraTS2021_01326', 'BraTS2021_01328', 'BraTS2021_01329', 'BraTS2021_01330', 'BraTS2021_01331', 'BraTS2021_01332', 'BraTS2021_01333', 'BraTS2021_01334', 'BraTS2021_01335', 'BraTS2021_01336', 'BraTS2021_01339', 'BraTS2021_01340', 'BraTS2021_01342', 'BraTS2021_01345', 'BraTS2021_01347', 'BraTS2021_01348', 'BraTS2021_01349', 'BraTS2021_01350', 'BraTS2021_01351', 'BraTS2021_01353', 'BraTS2021_01354', 'BraTS2021_01356', 'BraTS2021_01357', 'BraTS2021_01358', 'BraTS2021_01359', 'BraTS2021_01360', 'BraTS2021_01361', 'BraTS2021_01363', 'BraTS2021_01365', 'BraTS2021_01366', 'BraTS2021_01367', 'BraTS2021_01368', 'BraTS2021_01369', 'BraTS2021_01370', 'BraTS2021_01371', 'BraTS2021_01372', 'BraTS2021_01373', 'BraTS2021_01374', 'BraTS2021_01375', 'BraTS2021_01376', 'BraTS2021_01378', 'BraTS2021_01379', 'BraTS2021_01380', 'BraTS2021_01383', 'BraTS2021_01384', 'BraTS2021_01385', 'BraTS2021_01387', 'BraTS2021_01388', 'BraTS2021_01389', 'BraTS2021_01390', 'BraTS2021_01391', 'BraTS2021_01392', 'BraTS2021_01393', 'BraTS2021_01394', 'BraTS2021_01395', 'BraTS2021_01396', 'BraTS2021_01397', 'BraTS2021_01398', 'BraTS2021_01399', 'BraTS2021_01401', 'BraTS2021_01402', 'BraTS2021_01403', 'BraTS2021_01404', 'BraTS2021_01405', 'BraTS2021_01406', 'BraTS2021_01407', 'BraTS2021_01409', 'BraTS2021_01410', 'BraTS2021_01412', 'BraTS2021_01413', 'BraTS2021_01414', 'BraTS2021_01415', 'BraTS2021_01416', 'BraTS2021_01417', 'BraTS2021_01418', 'BraTS2021_01419', 'BraTS2021_01420', 'BraTS2021_01421', 'BraTS2021_01423', 'BraTS2021_01424', 'BraTS2021_01425', 'BraTS2021_01426', 'BraTS2021_01428', 'BraTS2021_01429', 'BraTS2021_01430', 'BraTS2021_01431', 'BraTS2021_01435', 'BraTS2021_01436', 'BraTS2021_01437', 'BraTS2021_01440', 'BraTS2021_01441', 'BraTS2021_01442', 'BraTS2021_01443', 'BraTS2021_01444', 'BraTS2021_01445', 'BraTS2021_01447', 'BraTS2021_01448', 'BraTS2021_01449', 'BraTS2021_01450', 'BraTS2021_01452', 'BraTS2021_01453', 'BraTS2021_01454', 'BraTS2021_01455', 'BraTS2021_01456', 'BraTS2021_01457', 'BraTS2021_01458', 'BraTS2021_01459', 'BraTS2021_01461', 'BraTS2021_01463', 'BraTS2021_01464', 'BraTS2021_01465', 'BraTS2021_01466', 'BraTS2021_01467', 'BraTS2021_01468', 'BraTS2021_01469', 'BraTS2021_01470', 'BraTS2021_01472', 'BraTS2021_01473', 'BraTS2021_01475', 'BraTS2021_01476', 'BraTS2021_01477', 'BraTS2021_01478', 'BraTS2021_01479', 'BraTS2021_01480', 'BraTS2021_01481', 'BraTS2021_01482', 'BraTS2021_01483', 'BraTS2021_01484', 'BraTS2021_01488', 'BraTS2021_01489', 'BraTS2021_01490', 'BraTS2021_01491', 'BraTS2021_01492', 'BraTS2021_01493', 'BraTS2021_01494', 'BraTS2021_01495', 'BraTS2021_01496', 'BraTS2021_01498', 'BraTS2021_01499', 'BraTS2021_01500', 'BraTS2021_01501', 'BraTS2021_01502', 'BraTS2021_01503', 'BraTS2021_01504', 'BraTS2021_01507', 'BraTS2021_01508', 'BraTS2021_01509', 'BraTS2021_01511', 'BraTS2021_01512', 'BraTS2021_01513', 'BraTS2021_01514', 'BraTS2021_01515', 'BraTS2021_01516', 'BraTS2021_01517', 'BraTS2021_01518', 'BraTS2021_01519', 'BraTS2021_01521', 'BraTS2021_01522', 'BraTS2021_01523', 'BraTS2021_01524', 'BraTS2021_01526', 'BraTS2021_01527', 'BraTS2021_01529', 'BraTS2021_01530', 'BraTS2021_01531', 'BraTS2021_01532', 'BraTS2021_01533', 'BraTS2021_01534', 'BraTS2021_01535', 'BraTS2021_01536', 'BraTS2021_01537', 'BraTS2021_01538', 'BraTS2021_01540', 'BraTS2021_01541', 'BraTS2021_01542', 'BraTS2021_01543', 'BraTS2021_01544', 'BraTS2021_01545', 'BraTS2021_01546', 'BraTS2021_01547', 'BraTS2021_01548', 'BraTS2021_01549', 'BraTS2021_01552', 'BraTS2021_01553', 'BraTS2021_01554', 'BraTS2021_01556', 'BraTS2021_01558', 'BraTS2021_01559', 'BraTS2021_01560', 'BraTS2021_01561', 'BraTS2021_01562', 'BraTS2021_01564', 'BraTS2021_01565', 'BraTS2021_01566', 'BraTS2021_01567', 'BraTS2021_01568', 'BraTS2021_01570', 'BraTS2021_01571', 'BraTS2021_01573', 'BraTS2021_01574', 'BraTS2021_01575', 'BraTS2021_01577', 'BraTS2021_01578', 'BraTS2021_01580', 'BraTS2021_01582', 'BraTS2021_01583', 'BraTS2021_01584', 'BraTS2021_01585', 'BraTS2021_01586', 'BraTS2021_01587', 'BraTS2021_01588', 'BraTS2021_01589', 'BraTS2021_01590', 'BraTS2021_01591', 'BraTS2021_01592', 'BraTS2021_01593', 'BraTS2021_01594', 'BraTS2021_01596', 'BraTS2021_01597', 'BraTS2021_01598', 'BraTS2021_01599', 'BraTS2021_01600', 'BraTS2021_01601', 'BraTS2021_01602', 'BraTS2021_01603', 'BraTS2021_01606', 'BraTS2021_01607', 'BraTS2021_01608', 'BraTS2021_01609', 'BraTS2021_01611', 'BraTS2021_01612', 'BraTS2021_01613', 'BraTS2021_01614', 'BraTS2021_01615', 'BraTS2021_01616', 'BraTS2021_01617', 'BraTS2021_01618', 'BraTS2021_01619', 'BraTS2021_01620', 'BraTS2021_01621', 'BraTS2021_01622', 'BraTS2021_01623', 'BraTS2021_01625', 'BraTS2021_01626', 'BraTS2021_01628', 'BraTS2021_01629', 'BraTS2021_01630', 'BraTS2021_01631', 'BraTS2021_01634', 'BraTS2021_01635', 'BraTS2021_01636', 'BraTS2021_01638', 'BraTS2021_01639', 'BraTS2021_01640', 'BraTS2021_01641', 'BraTS2021_01643', 'BraTS2021_01644', 'BraTS2021_01645', 'BraTS2021_01646', 'BraTS2021_01648', 'BraTS2021_01649', 'BraTS2021_01650', 'BraTS2021_01651', 'BraTS2021_01652', 'BraTS2021_01653', 'BraTS2021_01654', 'BraTS2021_01655', 'BraTS2021_01656', 'BraTS2021_01657', 'BraTS2021_01659', 'BraTS2021_01660', 'BraTS2021_01661', 'BraTS2021_01662', 'BraTS2021_01663', 'BraTS2021_01664', 'BraTS2021_01665', 'BraTS2021_01666']) +2022-05-14 04:21:46.675368: VALIDATION KEYS: + odict_keys(['BraTS2021_00005', 'BraTS2021_00006', 'BraTS2021_00011', 'BraTS2021_00012', 'BraTS2021_00020', 'BraTS2021_00021', 'BraTS2021_00036', 'BraTS2021_00053', 'BraTS2021_00061', 'BraTS2021_00081', 'BraTS2021_00084', 'BraTS2021_00090', 'BraTS2021_00095', 'BraTS2021_00097', 'BraTS2021_00101', 'BraTS2021_00109', 'BraTS2021_00133', 'BraTS2021_00147', 'BraTS2021_00157', 'BraTS2021_00159', 'BraTS2021_00162', 'BraTS2021_00170', 'BraTS2021_00171', 'BraTS2021_00177', 'BraTS2021_00178', 'BraTS2021_00184', 'BraTS2021_00188', 'BraTS2021_00191', 'BraTS2021_00204', 'BraTS2021_00207', 'BraTS2021_00210', 'BraTS2021_00220', 'BraTS2021_00222', 'BraTS2021_00233', 'BraTS2021_00234', 'BraTS2021_00237', 'BraTS2021_00247', 'BraTS2021_00258', 'BraTS2021_00260', 'BraTS2021_00263', 'BraTS2021_00267', 'BraTS2021_00281', 'BraTS2021_00282', 'BraTS2021_00289', 'BraTS2021_00290', 'BraTS2021_00293', 'BraTS2021_00303', 'BraTS2021_00310', 'BraTS2021_00312', 'BraTS2021_00331', 'BraTS2021_00339', 'BraTS2021_00341', 'BraTS2021_00343', 'BraTS2021_00344', 'BraTS2021_00360', 'BraTS2021_00377', 'BraTS2021_00397', 'BraTS2021_00417', 'BraTS2021_00440', 'BraTS2021_00445', 'BraTS2021_00453', 'BraTS2021_00455', 'BraTS2021_00466', 'BraTS2021_00468', 'BraTS2021_00477', 'BraTS2021_00488', 'BraTS2021_00506', 'BraTS2021_00518', 'BraTS2021_00520', 'BraTS2021_00525', 'BraTS2021_00529', 'BraTS2021_00545', 'BraTS2021_00547', 'BraTS2021_00552', 'BraTS2021_00555', 'BraTS2021_00557', 'BraTS2021_00559', 'BraTS2021_00570', 'BraTS2021_00571', 'BraTS2021_00576', 'BraTS2021_00579', 'BraTS2021_00580', 'BraTS2021_00593', 'BraTS2021_00594', 'BraTS2021_00598', 'BraTS2021_00599', 'BraTS2021_00610', 'BraTS2021_00612', 'BraTS2021_00615', 'BraTS2021_00616', 'BraTS2021_00628', 'BraTS2021_00631', 'BraTS2021_00640', 'BraTS2021_00642', 'BraTS2021_00645', 'BraTS2021_00657', 'BraTS2021_00658', 'BraTS2021_00661', 'BraTS2021_00663', 'BraTS2021_00675', 'BraTS2021_00688', 'BraTS2021_00692', 'BraTS2021_00703', 'BraTS2021_00718', 'BraTS2021_00730', 'BraTS2021_00731', 'BraTS2021_00736', 'BraTS2021_00744', 'BraTS2021_00746', 'BraTS2021_00778', 'BraTS2021_00784', 'BraTS2021_00788', 'BraTS2021_00795', 'BraTS2021_00809', 'BraTS2021_00811', 'BraTS2021_00814', 'BraTS2021_00819', 'BraTS2021_00837', 'BraTS2021_01002', 'BraTS2021_01003', 'BraTS2021_01009', 'BraTS2021_01019', 'BraTS2021_01020', 'BraTS2021_01027', 'BraTS2021_01028', 'BraTS2021_01034', 'BraTS2021_01040', 'BraTS2021_01041', 'BraTS2021_01045', 'BraTS2021_01053', 'BraTS2021_01054', 'BraTS2021_01061', 'BraTS2021_01067', 'BraTS2021_01068', 'BraTS2021_01084', 'BraTS2021_01086', 'BraTS2021_01092', 'BraTS2021_01098', 'BraTS2021_01102', 'BraTS2021_01112', 'BraTS2021_01114', 'BraTS2021_01119', 'BraTS2021_01125', 'BraTS2021_01140', 'BraTS2021_01144', 'BraTS2021_01148', 'BraTS2021_01155', 'BraTS2021_01165', 'BraTS2021_01166', 'BraTS2021_01172', 'BraTS2021_01184', 'BraTS2021_01186', 'BraTS2021_01188', 'BraTS2021_01193', 'BraTS2021_01194', 'BraTS2021_01199', 'BraTS2021_01200', 'BraTS2021_01215', 'BraTS2021_01219', 'BraTS2021_01220', 'BraTS2021_01224', 'BraTS2021_01228', 'BraTS2021_01229', 'BraTS2021_01232', 'BraTS2021_01233', 'BraTS2021_01234', 'BraTS2021_01245', 'BraTS2021_01246', 'BraTS2021_01249', 'BraTS2021_01252', 'BraTS2021_01254', 'BraTS2021_01257', 'BraTS2021_01258', 'BraTS2021_01259', 'BraTS2021_01264', 'BraTS2021_01269', 'BraTS2021_01272', 'BraTS2021_01279', 'BraTS2021_01281', 'BraTS2021_01287', 'BraTS2021_01288', 'BraTS2021_01295', 'BraTS2021_01297', 'BraTS2021_01303', 'BraTS2021_01311', 'BraTS2021_01323', 'BraTS2021_01327', 'BraTS2021_01337', 'BraTS2021_01338', 'BraTS2021_01341', 'BraTS2021_01343', 'BraTS2021_01344', 'BraTS2021_01346', 'BraTS2021_01352', 'BraTS2021_01355', 'BraTS2021_01362', 'BraTS2021_01364', 'BraTS2021_01377', 'BraTS2021_01381', 'BraTS2021_01382', 'BraTS2021_01386', 'BraTS2021_01400', 'BraTS2021_01408', 'BraTS2021_01411', 'BraTS2021_01422', 'BraTS2021_01427', 'BraTS2021_01432', 'BraTS2021_01433', 'BraTS2021_01434', 'BraTS2021_01438', 'BraTS2021_01439', 'BraTS2021_01446', 'BraTS2021_01451', 'BraTS2021_01460', 'BraTS2021_01462', 'BraTS2021_01471', 'BraTS2021_01474', 'BraTS2021_01485', 'BraTS2021_01486', 'BraTS2021_01487', 'BraTS2021_01497', 'BraTS2021_01505', 'BraTS2021_01506', 'BraTS2021_01510', 'BraTS2021_01520', 'BraTS2021_01525', 'BraTS2021_01528', 'BraTS2021_01539', 'BraTS2021_01550', 'BraTS2021_01551', 'BraTS2021_01555', 'BraTS2021_01557', 'BraTS2021_01563', 'BraTS2021_01569', 'BraTS2021_01572', 'BraTS2021_01576', 'BraTS2021_01579', 'BraTS2021_01581', 'BraTS2021_01595', 'BraTS2021_01604', 'BraTS2021_01605', 'BraTS2021_01610', 'BraTS2021_01624', 'BraTS2021_01627', 'BraTS2021_01632', 'BraTS2021_01633', 'BraTS2021_01637', 'BraTS2021_01642', 'BraTS2021_01647', 'BraTS2021_01658']) +2022-05-14 04:21:50.332643: lr: 3.3e-05 +2022-05-14 04:22:01.960656: +epoch: 0 +2022-05-14 04:29:44.143657: train loss : 0.3123 +2022-05-14 04:30:09.161000: validation loss: 0.1619 +2022-05-14 04:30:09.161655: Average global foreground Dice: [0.8951, 0.8008, 0.8217] +2022-05-14 04:30:09.161750: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 04:30:09.490682: lr: 3.3e-05 +2022-05-14 04:30:09.490887: saving scheduled checkpoint file... +2022-05-14 04:30:09.546413: saving checkpoint... +2022-05-14 04:30:10.165593: done, saving took 0.67 seconds +2022-05-14 04:30:10.181887: done +2022-05-14 04:30:20.061452: This epoch took 498.099998 s + +2022-05-14 04:30:20.061765: +epoch: 1 +2022-05-14 04:37:34.696601: train loss : 0.1520 +2022-05-14 04:37:59.840569: validation loss: 0.1333 +2022-05-14 04:37:59.841266: Average global foreground Dice: [0.9174, 0.8372, 0.8505] +2022-05-14 04:37:59.841351: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 04:38:00.241261: lr: 6.7e-05 +2022-05-14 04:38:00.241453: saving scheduled checkpoint file... +2022-05-14 04:38:00.295589: saving checkpoint... +2022-05-14 04:38:01.108798: done, saving took 0.87 seconds +2022-05-14 04:38:01.127145: done +2022-05-14 04:38:01.163545: saving checkpoint... +2022-05-14 04:38:01.718764: done, saving took 0.59 seconds +2022-05-14 04:38:23.455044: This epoch took 483.393174 s + +2022-05-14 04:38:23.455354: +epoch: 2 +2022-05-14 04:45:38.584424: train loss : 0.1147 +2022-05-14 04:46:03.688575: validation loss: 0.1181 +2022-05-14 04:46:03.689243: Average global foreground Dice: [0.9233, 0.8731, 0.8605] +2022-05-14 04:46:03.689330: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 04:46:04.092124: lr: 0.0001 +2022-05-14 04:46:04.092335: saving scheduled checkpoint file... +2022-05-14 04:46:04.144568: saving checkpoint... +2022-05-14 04:46:04.942667: done, saving took 0.85 seconds +2022-05-14 04:46:04.958410: done +2022-05-14 04:46:04.995112: saving checkpoint... +2022-05-14 04:46:05.769594: done, saving took 0.81 seconds +2022-05-14 04:46:21.911979: This epoch took 478.456503 s + +2022-05-14 04:46:21.912373: +epoch: 3 +2022-05-14 04:53:37.462041: train loss : 0.0924 +2022-05-14 04:54:02.593008: validation loss: 0.0745 +2022-05-14 04:54:02.593645: Average global foreground Dice: [0.9303, 0.9003, 0.8783] +2022-05-14 04:54:02.593733: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 04:54:02.993437: lr: 0.000133 +2022-05-14 04:54:02.993642: saving scheduled checkpoint file... +2022-05-14 04:54:03.043010: saving checkpoint... +2022-05-14 04:54:03.892686: done, saving took 0.90 seconds +2022-05-14 04:54:03.909658: done +2022-05-14 04:54:03.939717: saving checkpoint... +2022-05-14 04:54:04.716281: done, saving took 0.81 seconds +2022-05-14 04:54:23.482972: This epoch took 481.570268 s + +2022-05-14 04:54:23.483246: +epoch: 4 +2022-05-14 05:01:38.877088: train loss : 0.0957 +2022-05-14 05:02:03.941275: validation loss: 0.0818 +2022-05-14 05:02:03.941927: Average global foreground Dice: [0.9272, 0.8844, 0.8618] +2022-05-14 05:02:03.942014: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 05:02:04.361482: lr: 0.000167 +2022-05-14 05:02:04.361674: saving scheduled checkpoint file... +2022-05-14 05:02:04.415404: saving checkpoint... +2022-05-14 05:02:05.219560: done, saving took 0.86 seconds +2022-05-14 05:02:05.237648: done +2022-05-14 05:02:05.279850: saving checkpoint... +2022-05-14 05:02:06.019174: done, saving took 0.78 seconds +2022-05-14 05:02:24.389162: This epoch took 480.905811 s + +2022-05-14 05:02:24.389423: +epoch: 5 +2022-05-14 05:09:40.198127: train loss : 0.0804 +2022-05-14 05:10:05.314664: validation loss: 0.0848 +2022-05-14 05:10:05.315422: Average global foreground Dice: [0.9319, 0.9031, 0.887] +2022-05-14 05:10:05.315536: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 05:10:05.765206: lr: 0.0002 +2022-05-14 05:10:05.765401: saving scheduled checkpoint file... +2022-05-14 05:10:05.817005: saving checkpoint... +2022-05-14 05:10:06.685178: done, saving took 0.92 seconds +2022-05-14 05:10:06.701546: done +2022-05-14 05:10:06.738196: saving checkpoint... +2022-05-14 05:10:07.464796: done, saving took 0.76 seconds +2022-05-14 05:10:24.316704: This epoch took 479.927176 s + +2022-05-14 05:10:24.317002: +epoch: 6 +2022-05-14 05:17:40.540337: train loss : 0.0814 +2022-05-14 05:18:05.678595: validation loss: 0.0628 +2022-05-14 05:18:05.679252: Average global foreground Dice: [0.9356, 0.9053, 0.8892] +2022-05-14 05:18:05.679336: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 05:18:06.077618: lr: 0.000233 +2022-05-14 05:18:06.077817: saving scheduled checkpoint file... +2022-05-14 05:18:06.132149: saving checkpoint... +2022-05-14 05:18:06.897704: done, saving took 0.82 seconds +2022-05-14 05:18:06.914181: done +2022-05-14 05:18:06.949802: saving checkpoint... +2022-05-14 05:18:07.649335: done, saving took 0.74 seconds +2022-05-14 05:18:31.103860: This epoch took 486.786667 s + +2022-05-14 05:18:31.104206: +epoch: 7 +2022-05-14 05:25:47.333308: train loss : 0.0718 +2022-05-14 05:26:12.411634: validation loss: 0.0719 +2022-05-14 05:26:12.412298: Average global foreground Dice: [0.9336, 0.9212, 0.9005] +2022-05-14 05:26:12.412385: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 05:26:12.878417: lr: 0.000267 +2022-05-14 05:26:12.878620: saving scheduled checkpoint file... +2022-05-14 05:26:12.932101: saving checkpoint... +2022-05-14 05:26:13.810634: done, saving took 0.93 seconds +2022-05-14 05:26:13.829754: done +2022-05-14 05:26:13.867835: saving checkpoint... +2022-05-14 05:26:14.728672: done, saving took 0.90 seconds +2022-05-14 05:26:32.755889: This epoch took 481.651577 s + +2022-05-14 05:26:32.756171: +epoch: 8 +2022-05-14 05:33:48.030341: train loss : 0.0726 +2022-05-14 05:34:13.147613: validation loss: 0.0578 +2022-05-14 05:34:13.148268: Average global foreground Dice: [0.9307, 0.9079, 0.8957] +2022-05-14 05:34:13.148365: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 05:34:13.639956: lr: 0.0003 +2022-05-14 05:34:13.640141: saving scheduled checkpoint file... +2022-05-14 05:34:13.693891: saving checkpoint... +2022-05-14 05:34:14.498442: done, saving took 0.86 seconds +2022-05-14 05:34:14.516726: done +2022-05-14 05:34:14.558123: saving checkpoint... +2022-05-14 05:34:15.364540: done, saving took 0.85 seconds +2022-05-14 05:34:32.246437: This epoch took 479.490165 s + +2022-05-14 05:34:32.246740: +epoch: 9 +2022-05-14 05:41:48.356678: train loss : 0.0736 +2022-05-14 05:42:13.466914: validation loss: 0.0570 +2022-05-14 05:42:13.467534: Average global foreground Dice: [0.9383, 0.9126, 0.9023] +2022-05-14 05:42:13.467635: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 05:42:13.872619: lr: 0.0003 +2022-05-14 05:42:13.872828: saving scheduled checkpoint file... +2022-05-14 05:42:13.921847: saving checkpoint... +2022-05-14 05:42:14.752723: done, saving took 0.88 seconds +2022-05-14 05:42:14.770821: done +2022-05-14 05:42:14.803890: saving checkpoint... +2022-05-14 05:42:15.586396: done, saving took 0.82 seconds +2022-05-14 05:42:32.941870: This epoch took 480.695025 s + +2022-05-14 05:42:32.942143: +epoch: 10 +2022-05-14 05:49:48.375852: train loss : 0.0710 +2022-05-14 05:50:13.433817: validation loss: 0.0732 +2022-05-14 05:50:13.434489: Average global foreground Dice: [0.9104, 0.9009, 0.8984] +2022-05-14 05:50:13.434591: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 05:50:13.843531: lr: 0.0003 +2022-05-14 05:50:13.843733: saving scheduled checkpoint file... +2022-05-14 05:50:13.890780: saving checkpoint... +2022-05-14 05:50:14.700491: done, saving took 0.86 seconds +2022-05-14 05:50:14.718579: done +2022-05-14 05:50:14.759916: saving checkpoint... +2022-05-14 05:50:15.544864: done, saving took 0.83 seconds +2022-05-14 05:50:38.245967: This epoch took 485.303724 s + +2022-05-14 05:50:38.246234: +epoch: 11 +2022-05-14 05:57:53.736954: train loss : 0.0683 +2022-05-14 05:58:18.844402: validation loss: 0.0551 +2022-05-14 05:58:18.845054: Average global foreground Dice: [0.9386, 0.9121, 0.9027] +2022-05-14 05:58:18.845153: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 05:58:19.253145: lr: 0.0003 +2022-05-14 05:58:19.253356: saving scheduled checkpoint file... +2022-05-14 05:58:19.301507: saving checkpoint... +2022-05-14 05:58:20.128147: done, saving took 0.87 seconds +2022-05-14 05:58:20.144224: done +2022-05-14 05:58:20.173953: saving checkpoint... +2022-05-14 05:58:20.955735: done, saving took 0.81 seconds +2022-05-14 05:58:41.256338: This epoch took 483.010006 s + +2022-05-14 05:58:41.256804: +epoch: 12 +2022-05-14 06:05:56.438266: train loss : 0.0699 +2022-05-14 06:06:21.542688: validation loss: 0.0555 +2022-05-14 06:06:21.543411: Average global foreground Dice: [0.9375, 0.9253, 0.9076] +2022-05-14 06:06:21.543495: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 06:06:21.946259: lr: 0.0003 +2022-05-14 06:06:21.946459: saving scheduled checkpoint file... +2022-05-14 06:06:21.994568: saving checkpoint... +2022-05-14 06:06:22.803960: done, saving took 0.86 seconds +2022-05-14 06:06:22.820895: done +2022-05-14 06:06:22.857176: saving checkpoint... +2022-05-14 06:06:23.654475: done, saving took 0.83 seconds +2022-05-14 06:06:41.276805: This epoch took 480.019894 s + +2022-05-14 06:06:41.277136: +epoch: 13 +2022-05-14 06:13:56.279581: train loss : 0.0653 +2022-05-14 06:14:21.428440: validation loss: 0.0710 +2022-05-14 06:14:21.429070: Average global foreground Dice: [0.9414, 0.9252, 0.9032] +2022-05-14 06:14:21.429153: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 06:14:21.828367: lr: 0.0003 +2022-05-14 06:14:21.828561: saving scheduled checkpoint file... +2022-05-14 06:14:21.876334: saving checkpoint... +2022-05-14 06:14:22.685912: done, saving took 0.86 seconds +2022-05-14 06:14:22.702038: done +2022-05-14 06:14:22.738184: saving checkpoint... +2022-05-14 06:14:23.532604: done, saving took 0.83 seconds +2022-05-14 06:14:54.584419: This epoch took 493.307175 s + +2022-05-14 06:14:54.584773: +epoch: 14 +2022-05-14 06:22:09.778492: train loss : 0.0634 +2022-05-14 06:22:34.900151: validation loss: 0.0722 +2022-05-14 06:22:34.900837: Average global foreground Dice: [0.9407, 0.9033, 0.904] +2022-05-14 06:22:34.900924: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 06:22:35.306123: lr: 0.0003 +2022-05-14 06:22:35.306328: saving scheduled checkpoint file... +2022-05-14 06:22:35.360540: saving checkpoint... +2022-05-14 06:22:36.172759: done, saving took 0.87 seconds +2022-05-14 06:22:36.192517: done +2022-05-14 06:22:36.238048: saving checkpoint... +2022-05-14 06:22:37.039303: done, saving took 0.85 seconds +2022-05-14 06:22:56.525278: This epoch took 481.940404 s + +2022-05-14 06:22:56.525552: +epoch: 15 +2022-05-14 06:30:11.821295: train loss : 0.0602 +2022-05-14 06:30:36.906627: validation loss: 0.0542 +2022-05-14 06:30:36.907236: Average global foreground Dice: [0.9358, 0.9281, 0.9034] +2022-05-14 06:30:36.907320: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 06:30:37.320991: lr: 0.0003 +2022-05-14 06:30:37.321176: saving scheduled checkpoint file... +2022-05-14 06:30:37.369637: saving checkpoint... +2022-05-14 06:30:38.172882: done, saving took 0.85 seconds +2022-05-14 06:30:38.190927: done +2022-05-14 06:30:38.231095: saving checkpoint... +2022-05-14 06:30:39.028488: done, saving took 0.84 seconds +2022-05-14 06:30:56.529195: This epoch took 480.003449 s + +2022-05-14 06:30:56.529472: +epoch: 16 +2022-05-14 06:38:12.712112: train loss : 0.0626 +2022-05-14 06:38:37.774272: validation loss: 0.0557 +2022-05-14 06:38:37.774906: Average global foreground Dice: [0.9365, 0.9203, 0.9101] +2022-05-14 06:38:37.774994: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 06:38:38.257023: lr: 0.000299 +2022-05-14 06:38:38.257209: saving scheduled checkpoint file... +2022-05-14 06:38:38.306271: saving checkpoint... +2022-05-14 06:38:39.126914: done, saving took 0.87 seconds +2022-05-14 06:38:39.143639: done +2022-05-14 06:38:39.184675: saving checkpoint... +2022-05-14 06:38:40.007339: done, saving took 0.86 seconds +2022-05-14 06:39:24.147715: This epoch took 507.618128 s + +2022-05-14 06:39:24.148024: +epoch: 17 +2022-05-14 06:46:39.755962: train loss : 0.0563 +2022-05-14 06:47:04.878022: validation loss: 0.0573 +2022-05-14 06:47:04.878686: Average global foreground Dice: [0.9407, 0.9379, 0.9133] +2022-05-14 06:47:04.878773: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 06:47:05.280083: lr: 0.000299 +2022-05-14 06:47:05.280269: saving scheduled checkpoint file... +2022-05-14 06:47:05.329253: saving checkpoint... +2022-05-14 06:47:06.169514: done, saving took 0.89 seconds +2022-05-14 06:47:06.186546: done +2022-05-14 06:47:06.220468: saving checkpoint... +2022-05-14 06:47:07.063332: done, saving took 0.88 seconds +2022-05-14 06:47:27.065362: This epoch took 482.917225 s + +2022-05-14 06:47:27.065715: +epoch: 18 +2022-05-14 06:54:42.496078: train loss : 0.0606 +2022-05-14 06:55:07.594636: validation loss: 0.0580 +2022-05-14 06:55:07.595360: Average global foreground Dice: [0.9435, 0.9269, 0.9123] +2022-05-14 06:55:07.595443: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 06:55:07.997416: lr: 0.000299 +2022-05-14 06:55:07.997609: saving scheduled checkpoint file... +2022-05-14 06:55:08.044919: saving checkpoint... +2022-05-14 06:55:08.858684: done, saving took 0.86 seconds +2022-05-14 06:55:08.874980: done +2022-05-14 06:55:08.908350: saving checkpoint... +2022-05-14 06:55:09.730928: done, saving took 0.86 seconds +2022-05-14 06:55:30.838106: This epoch took 483.772270 s + +2022-05-14 06:55:30.838446: +epoch: 19 +2022-05-14 07:02:46.130294: train loss : 0.0626 +2022-05-14 07:03:11.246331: validation loss: 0.0468 +2022-05-14 07:03:11.247177: Average global foreground Dice: [0.9433, 0.9114, 0.8985] +2022-05-14 07:03:11.247262: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 07:03:11.653249: lr: 0.000299 +2022-05-14 07:03:11.653450: saving scheduled checkpoint file... +2022-05-14 07:03:11.703123: saving checkpoint... +2022-05-14 07:03:12.459256: done, saving took 0.81 seconds +2022-05-14 07:03:12.474792: done +2022-05-14 07:03:12.504494: saving checkpoint... +2022-05-14 07:03:13.241434: done, saving took 0.77 seconds +2022-05-14 07:03:42.660804: This epoch took 491.822254 s + +2022-05-14 07:03:42.661087: +epoch: 20 +2022-05-14 07:10:58.585268: train loss : 0.0593 +2022-05-14 07:11:23.730987: validation loss: 0.0585 +2022-05-14 07:11:23.731904: Average global foreground Dice: [0.9467, 0.933, 0.9073] +2022-05-14 07:11:23.731992: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 07:11:24.141223: lr: 0.000298 +2022-05-14 07:11:24.141422: saving scheduled checkpoint file... +2022-05-14 07:11:24.189355: saving checkpoint... +2022-05-14 07:11:24.920735: done, saving took 0.78 seconds +2022-05-14 07:11:24.936420: done +2022-05-14 07:11:24.971751: saving checkpoint... +2022-05-14 07:11:25.747325: done, saving took 0.81 seconds +2022-05-14 07:11:43.550112: This epoch took 480.888908 s + +2022-05-14 07:11:43.550524: +epoch: 21 +2022-05-14 07:18:59.373440: train loss : 0.0600 +2022-05-14 07:19:24.517640: validation loss: 0.0566 +2022-05-14 07:19:24.518272: Average global foreground Dice: [0.9402, 0.9346, 0.9119] +2022-05-14 07:19:24.518460: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 07:19:24.915919: lr: 0.000298 +2022-05-14 07:19:24.916120: saving scheduled checkpoint file... +2022-05-14 07:19:24.964481: saving checkpoint... +2022-05-14 07:19:25.690034: done, saving took 0.77 seconds +2022-05-14 07:19:25.706527: done +2022-05-14 07:19:25.742198: saving checkpoint... +2022-05-14 07:19:26.483730: done, saving took 0.78 seconds +2022-05-14 07:19:46.825345: This epoch took 483.274684 s + +2022-05-14 07:19:46.825722: +epoch: 22 +2022-05-14 07:27:02.896137: train loss : 0.0572 +2022-05-14 07:27:28.097313: validation loss: 0.0551 +2022-05-14 07:27:28.098022: Average global foreground Dice: [0.9439, 0.9331, 0.9148] +2022-05-14 07:27:28.098106: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 07:27:28.496650: lr: 0.000298 +2022-05-14 07:27:28.496839: saving scheduled checkpoint file... +2022-05-14 07:27:28.545225: saving checkpoint... +2022-05-14 07:27:29.262428: done, saving took 0.77 seconds +2022-05-14 07:27:29.279141: done +2022-05-14 07:27:29.315148: saving checkpoint... +2022-05-14 07:27:30.063202: done, saving took 0.78 seconds +2022-05-14 07:27:53.902069: This epoch took 487.076243 s + +2022-05-14 07:27:53.902355: +epoch: 23 +2022-05-14 07:35:09.966392: train loss : 0.0588 +2022-05-14 07:35:35.097645: validation loss: 0.0472 +2022-05-14 07:35:35.098298: Average global foreground Dice: [0.9465, 0.931, 0.9058] +2022-05-14 07:35:35.098384: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 07:35:35.499657: lr: 0.000297 +2022-05-14 07:35:35.499861: saving scheduled checkpoint file... +2022-05-14 07:35:35.548481: saving checkpoint... +2022-05-14 07:35:36.288849: done, saving took 0.79 seconds +2022-05-14 07:35:36.299401: done +2022-05-14 07:35:36.330616: saving checkpoint... +2022-05-14 07:35:37.080151: done, saving took 0.78 seconds +2022-05-14 07:36:00.920714: This epoch took 487.012146 s + +2022-05-14 07:36:00.921038: +epoch: 24 +2022-05-14 07:43:16.626617: train loss : 0.0540 +2022-05-14 07:43:41.784875: validation loss: 0.0504 +2022-05-14 07:43:41.785539: Average global foreground Dice: [0.944, 0.9398, 0.9159] +2022-05-14 07:43:41.785639: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 07:43:42.183294: lr: 0.000297 +2022-05-14 07:43:42.183484: saving scheduled checkpoint file... +2022-05-14 07:43:42.236505: saving checkpoint... +2022-05-14 07:43:43.070036: done, saving took 0.89 seconds +2022-05-14 07:43:43.088047: done +2022-05-14 07:43:43.121374: saving checkpoint... +2022-05-14 07:43:43.959772: done, saving took 0.87 seconds +2022-05-14 07:44:02.044607: This epoch took 481.123455 s + +2022-05-14 07:44:02.044885: +epoch: 25 +2022-05-14 07:51:17.923748: train loss : 0.0625 +2022-05-14 07:51:43.023587: validation loss: 0.0499 +2022-05-14 07:51:43.024288: Average global foreground Dice: [0.9418, 0.9398, 0.9171] +2022-05-14 07:51:43.024374: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 07:51:43.430251: lr: 0.000297 +2022-05-14 07:51:43.430449: saving scheduled checkpoint file... +2022-05-14 07:51:43.483032: saving checkpoint... +2022-05-14 07:51:44.312262: done, saving took 0.88 seconds +2022-05-14 07:51:44.333545: done +2022-05-14 07:51:44.367060: saving checkpoint... +2022-05-14 07:51:45.197257: done, saving took 0.86 seconds +2022-05-14 07:52:03.321228: This epoch took 481.276242 s + +2022-05-14 07:52:03.321496: +epoch: 26 +2022-05-14 07:59:19.412701: train loss : 0.0570 +2022-05-14 07:59:44.499267: validation loss: 0.0521 +2022-05-14 07:59:44.499953: Average global foreground Dice: [0.9446, 0.9317, 0.9097] +2022-05-14 07:59:44.500040: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 07:59:44.900133: lr: 0.000296 +2022-05-14 07:59:44.900331: saving scheduled checkpoint file... +2022-05-14 07:59:44.954001: saving checkpoint... +2022-05-14 07:59:45.790180: done, saving took 0.89 seconds +2022-05-14 07:59:45.808299: done +2022-05-14 07:59:45.846759: saving checkpoint... +2022-05-14 07:59:46.989419: done, saving took 1.18 seconds +2022-05-14 08:00:03.061449: This epoch took 479.739842 s + +2022-05-14 08:00:03.061794: +epoch: 27 +2022-05-14 08:07:19.485027: train loss : 0.0600 +2022-05-14 08:07:44.615680: validation loss: 0.0510 +2022-05-14 08:07:44.616364: Average global foreground Dice: [0.946, 0.9384, 0.9115] +2022-05-14 08:07:44.616464: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 08:07:45.015832: lr: 0.000296 +2022-05-14 08:07:45.016026: saving scheduled checkpoint file... +2022-05-14 08:07:45.065723: saving checkpoint... +2022-05-14 08:07:45.883843: done, saving took 0.87 seconds +2022-05-14 08:07:45.978147: done +2022-05-14 08:07:46.007280: saving checkpoint... +2022-05-14 08:07:46.795160: done, saving took 0.82 seconds +2022-05-14 08:08:12.801766: This epoch took 489.739871 s + +2022-05-14 08:08:12.802052: +epoch: 28 +2022-05-14 08:15:28.278110: train loss : 0.0627 +2022-05-14 08:15:53.369350: validation loss: 0.0571 +2022-05-14 08:15:53.370043: Average global foreground Dice: [0.9428, 0.9338, 0.9115] +2022-05-14 08:15:53.370128: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 08:15:53.776888: lr: 0.000295 +2022-05-14 08:15:53.777081: saving scheduled checkpoint file... +2022-05-14 08:15:53.829785: saving checkpoint... +2022-05-14 08:15:54.659890: done, saving took 0.88 seconds +2022-05-14 08:15:54.677741: done +2022-05-14 08:15:54.711201: saving checkpoint... +2022-05-14 08:15:55.550323: done, saving took 0.87 seconds +2022-05-14 08:16:21.074645: This epoch took 488.272408 s + +2022-05-14 08:16:21.074927: +epoch: 29 +2022-05-14 08:23:35.770446: train loss : 0.0551 +2022-05-14 08:24:00.897026: validation loss: 0.0621 +2022-05-14 08:24:00.897641: Average global foreground Dice: [0.9424, 0.9297, 0.9108] +2022-05-14 08:24:00.897730: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 08:24:01.300196: lr: 0.000295 +2022-05-14 08:24:01.300385: saving scheduled checkpoint file... +2022-05-14 08:24:01.353098: saving checkpoint... +2022-05-14 08:24:02.177901: done, saving took 0.88 seconds +2022-05-14 08:24:02.195959: done +2022-05-14 08:24:02.236470: saving checkpoint... +2022-05-14 08:24:03.065414: done, saving took 0.87 seconds +2022-05-14 08:24:20.617849: This epoch took 479.542824 s + +2022-05-14 08:24:20.618117: +epoch: 30 +2022-05-14 08:31:36.534873: train loss : 0.0580 +2022-05-14 08:32:01.624653: validation loss: 0.0537 +2022-05-14 08:32:01.625361: Average global foreground Dice: [0.943, 0.9205, 0.92] +2022-05-14 08:32:01.625444: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 08:32:02.026384: lr: 0.000294 +2022-05-14 08:32:02.026592: saving scheduled checkpoint file... +2022-05-14 08:32:02.076393: saving checkpoint... +2022-05-14 08:32:02.893331: done, saving took 0.87 seconds +2022-05-14 08:32:02.911690: done +2022-05-14 08:32:02.952569: saving checkpoint... +2022-05-14 08:32:03.794629: done, saving took 0.88 seconds +2022-05-14 08:32:21.679047: This epoch took 481.060821 s + +2022-05-14 08:32:21.679354: +epoch: 31 +2022-05-14 08:39:38.130245: train loss : 0.0579 +2022-05-14 08:40:03.225180: validation loss: 0.0648 +2022-05-14 08:40:03.225821: Average global foreground Dice: [0.9425, 0.931, 0.9111] +2022-05-14 08:40:03.225908: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 08:40:03.627303: lr: 0.000294 +2022-05-14 08:40:03.627482: saving scheduled checkpoint file... +2022-05-14 08:40:03.675440: saving checkpoint... +2022-05-14 08:40:04.495247: done, saving took 0.87 seconds +2022-05-14 08:40:04.513465: done +2022-05-14 08:40:04.554369: saving checkpoint... +2022-05-14 08:40:05.297807: done, saving took 0.78 seconds +2022-05-14 08:40:27.285328: This epoch took 485.605866 s + +2022-05-14 08:40:27.285633: +epoch: 32 +2022-05-14 08:47:42.523265: train loss : 0.0526 +2022-05-14 08:48:07.616103: validation loss: 0.0630 +2022-05-14 08:48:07.616760: Average global foreground Dice: [0.9454, 0.9259, 0.9147] +2022-05-14 08:48:07.616846: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 08:48:08.015460: lr: 0.000293 +2022-05-14 08:48:08.015658: saving scheduled checkpoint file... +2022-05-14 08:48:08.063183: saving checkpoint... +2022-05-14 08:48:08.779046: done, saving took 0.76 seconds +2022-05-14 08:48:08.795636: done +2022-05-14 08:48:08.830698: saving checkpoint... +2022-05-14 08:48:09.620103: done, saving took 0.82 seconds +2022-05-14 08:48:27.668289: This epoch took 480.382536 s + +2022-05-14 08:48:27.668709: +epoch: 33 +2022-05-14 08:55:43.631244: train loss : 0.0552 +2022-05-14 08:56:08.742412: validation loss: 0.0478 +2022-05-14 08:56:08.743058: Average global foreground Dice: [0.9444, 0.9299, 0.9179] +2022-05-14 08:56:08.743141: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 08:56:09.146919: lr: 0.000293 +2022-05-14 08:56:09.147124: saving scheduled checkpoint file... +2022-05-14 08:56:09.195302: saving checkpoint... +2022-05-14 08:56:10.013899: done, saving took 0.87 seconds +2022-05-14 08:56:10.030046: done +2022-05-14 08:56:10.066492: saving checkpoint... +2022-05-14 08:56:10.894014: done, saving took 0.86 seconds +2022-05-14 08:56:27.779492: This epoch took 480.110677 s + +2022-05-14 08:56:27.779786: +epoch: 34 +2022-05-14 09:03:43.876119: train loss : 0.0567 +2022-05-14 09:04:09.018415: validation loss: 0.0534 +2022-05-14 09:04:09.019063: Average global foreground Dice: [0.944, 0.924, 0.9141] +2022-05-14 09:04:09.019148: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 09:04:09.502951: lr: 0.000292 +2022-05-14 09:04:09.503153: saving scheduled checkpoint file... +2022-05-14 09:04:09.551304: saving checkpoint... +2022-05-14 09:04:10.329052: done, saving took 0.83 seconds +2022-05-14 09:04:10.344759: done +2022-05-14 09:04:10.382431: saving checkpoint... +2022-05-14 09:04:11.176524: done, saving took 0.83 seconds +2022-05-14 09:04:29.527843: This epoch took 481.747955 s + +2022-05-14 09:04:29.528127: +epoch: 35 +2022-05-14 09:11:45.545434: train loss : 0.0577 +2022-05-14 09:12:10.599627: validation loss: 0.0599 +2022-05-14 09:12:10.600383: Average global foreground Dice: [0.9459, 0.9217, 0.916] +2022-05-14 09:12:10.600469: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 09:12:11.004470: lr: 0.000291 +2022-05-14 09:12:11.004676: saving scheduled checkpoint file... +2022-05-14 09:12:11.054106: saving checkpoint... +2022-05-14 09:12:11.876516: done, saving took 0.87 seconds +2022-05-14 09:12:11.894862: done +2022-05-14 09:12:11.936110: saving checkpoint... +2022-05-14 09:12:12.764287: done, saving took 0.87 seconds +2022-05-14 09:12:30.709776: This epoch took 481.181545 s + +2022-05-14 09:12:30.710072: +epoch: 36 +2022-05-14 09:19:46.902086: train loss : 0.0543 +2022-05-14 09:20:11.974886: validation loss: 0.0563 +2022-05-14 09:20:11.975522: Average global foreground Dice: [0.9463, 0.9361, 0.9124] +2022-05-14 09:20:11.975617: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 09:20:12.377019: lr: 0.000291 +2022-05-14 09:20:12.377204: saving scheduled checkpoint file... +2022-05-14 09:20:12.424795: saving checkpoint... +2022-05-14 09:20:13.253284: done, saving took 0.88 seconds +2022-05-14 09:20:13.269355: done +2022-05-14 09:20:13.305281: saving checkpoint... +2022-05-14 09:20:14.133693: done, saving took 0.86 seconds +2022-05-14 09:20:32.547311: This epoch took 481.837139 s + +2022-05-14 09:20:32.547596: +epoch: 37 +2022-05-14 09:27:48.569778: train loss : 0.0532 +2022-05-14 09:28:13.673846: validation loss: 0.0623 +2022-05-14 09:28:13.674492: Average global foreground Dice: [0.9439, 0.9368, 0.9179] +2022-05-14 09:28:13.674589: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 09:28:14.078483: lr: 0.00029 +2022-05-14 09:28:14.078682: saving scheduled checkpoint file... +2022-05-14 09:28:14.126449: saving checkpoint... +2022-05-14 09:28:14.944713: done, saving took 0.87 seconds +2022-05-14 09:28:14.960656: done +2022-05-14 09:28:14.996583: saving checkpoint... +2022-05-14 09:28:15.825197: done, saving took 0.86 seconds +2022-05-14 09:28:33.834340: This epoch took 481.286619 s + +2022-05-14 09:28:33.838939: +epoch: 38 +2022-05-14 09:35:49.497065: train loss : 0.0583 +2022-05-14 09:36:14.590244: validation loss: 0.0453 +2022-05-14 09:36:14.590921: Average global foreground Dice: [0.9475, 0.9438, 0.9181] +2022-05-14 09:36:14.591005: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 09:36:15.005121: lr: 0.000289 +2022-05-14 09:36:15.005316: saving scheduled checkpoint file... +2022-05-14 09:36:15.053306: saving checkpoint... +2022-05-14 09:36:15.886270: done, saving took 0.88 seconds +2022-05-14 09:36:15.904276: done +2022-05-14 09:36:15.938176: saving checkpoint... +2022-05-14 09:36:16.762635: done, saving took 0.86 seconds +2022-05-14 09:36:33.955291: This epoch took 480.116220 s + +2022-05-14 09:36:33.955595: +epoch: 39 +2022-05-14 09:43:49.798409: train loss : 0.0546 +2022-05-14 09:44:14.907861: validation loss: 0.0548 +2022-05-14 09:44:14.908543: Average global foreground Dice: [0.9451, 0.9292, 0.915] +2022-05-14 09:44:14.908642: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 09:44:15.323361: lr: 0.000289 +2022-05-14 09:44:15.323551: saving scheduled checkpoint file... +2022-05-14 09:44:15.377278: saving checkpoint... +2022-05-14 09:44:16.190033: done, saving took 0.87 seconds +2022-05-14 09:44:16.206430: done +2022-05-14 09:44:16.242995: saving checkpoint... +2022-05-14 09:44:17.050172: done, saving took 0.84 seconds +2022-05-14 09:44:41.239444: This epoch took 487.283746 s + +2022-05-14 09:44:41.239727: +epoch: 40 +2022-05-14 09:51:56.744728: train loss : 0.0546 +2022-05-14 09:52:21.785333: validation loss: 0.0625 +2022-05-14 09:52:21.785977: Average global foreground Dice: [0.9471, 0.928, 0.9157] +2022-05-14 09:52:21.786061: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 09:52:22.194439: lr: 0.000288 +2022-05-14 09:52:22.194633: saving scheduled checkpoint file... +2022-05-14 09:52:22.241963: saving checkpoint... +2022-05-14 09:52:23.049563: done, saving took 0.85 seconds +2022-05-14 09:52:23.067992: done +2022-05-14 09:52:23.109210: saving checkpoint... +2022-05-14 09:52:23.899475: done, saving took 0.83 seconds +2022-05-14 09:52:50.421903: This epoch took 489.182073 s + +2022-05-14 09:52:50.422189: +epoch: 41 +2022-05-14 10:00:06.196724: train loss : 0.0535 +2022-05-14 10:00:31.371096: validation loss: 0.0546 +2022-05-14 10:00:31.371811: Average global foreground Dice: [0.9504, 0.9219, 0.9178] +2022-05-14 10:00:31.372093: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 10:00:31.772631: lr: 0.000287 +2022-05-14 10:00:31.772825: saving scheduled checkpoint file... +2022-05-14 10:00:31.822504: saving checkpoint... +2022-05-14 10:00:32.658444: done, saving took 0.89 seconds +2022-05-14 10:00:32.670165: done +2022-05-14 10:00:32.704062: saving checkpoint... +2022-05-14 10:00:33.422795: done, saving took 0.75 seconds +2022-05-14 10:00:54.795081: This epoch took 484.372793 s + +2022-05-14 10:00:54.795355: +epoch: 42 +2022-05-14 10:08:11.007767: train loss : 0.0527 +2022-05-14 10:08:36.170759: validation loss: 0.0496 +2022-05-14 10:08:36.171467: Average global foreground Dice: [0.9463, 0.9368, 0.9145] +2022-05-14 10:08:36.171553: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 10:08:36.582561: lr: 0.000286 +2022-05-14 10:08:36.582753: saving scheduled checkpoint file... +2022-05-14 10:08:36.635489: saving checkpoint... +2022-05-14 10:08:37.453893: done, saving took 0.87 seconds +2022-05-14 10:08:37.470071: done +2022-05-14 10:08:37.505978: saving checkpoint... +2022-05-14 10:08:38.309338: done, saving took 0.84 seconds +2022-05-14 10:08:52.570099: This epoch took 477.774642 s + +2022-05-14 10:08:52.570387: +epoch: 43 +2022-05-14 10:16:08.829611: train loss : 0.0542 +2022-05-14 10:16:33.889781: validation loss: 0.0609 +2022-05-14 10:16:33.890441: Average global foreground Dice: [0.9412, 0.9351, 0.9169] +2022-05-14 10:16:33.890542: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 10:16:34.291706: lr: 0.000285 +2022-05-14 10:16:34.291903: saving scheduled checkpoint file... +2022-05-14 10:16:34.339675: saving checkpoint... +2022-05-14 10:16:35.187342: done, saving took 0.90 seconds +2022-05-14 10:16:35.203584: done +2022-05-14 10:16:35.233738: saving checkpoint... +2022-05-14 10:16:36.037114: done, saving took 0.83 seconds +2022-05-14 10:16:55.100743: This epoch took 482.530253 s + +2022-05-14 10:16:55.101028: +epoch: 44 +2022-05-14 10:24:10.840733: train loss : 0.0531 +2022-05-14 10:24:35.901691: validation loss: 0.0541 +2022-05-14 10:24:35.902491: Average global foreground Dice: [0.9423, 0.9358, 0.9208] +2022-05-14 10:24:35.902587: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 10:24:36.304554: lr: 0.000285 +2022-05-14 10:24:36.304775: saving scheduled checkpoint file... +2022-05-14 10:24:36.358132: saving checkpoint... +2022-05-14 10:24:37.190366: done, saving took 0.89 seconds +2022-05-14 10:24:37.208802: done +2022-05-14 10:24:37.242571: saving checkpoint... +2022-05-14 10:24:38.049749: done, saving took 0.84 seconds +2022-05-14 10:24:53.848746: This epoch took 478.747618 s + +2022-05-14 10:24:53.849020: +epoch: 45 +2022-05-14 10:32:09.764944: train loss : 0.0530 +2022-05-14 10:32:34.852003: validation loss: 0.0474 +2022-05-14 10:32:34.852634: Average global foreground Dice: [0.9482, 0.9316, 0.9185] +2022-05-14 10:32:34.852722: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 10:32:35.250700: lr: 0.000284 +2022-05-14 10:32:35.250891: saving scheduled checkpoint file... +2022-05-14 10:32:35.298588: saving checkpoint... +2022-05-14 10:32:36.110681: done, saving took 0.86 seconds +2022-05-14 10:32:36.128731: done +2022-05-14 10:32:36.169702: saving checkpoint... +2022-05-14 10:32:36.971958: done, saving took 0.84 seconds +2022-05-14 10:32:55.839514: This epoch took 481.990391 s + +2022-05-14 10:32:55.839816: +epoch: 46 +2022-05-14 10:40:11.662048: train loss : 0.0502 +2022-05-14 10:40:36.735227: validation loss: 0.0630 +2022-05-14 10:40:36.735873: Average global foreground Dice: [0.9481, 0.9242, 0.9154] +2022-05-14 10:40:36.735971: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 10:40:37.137879: lr: 0.000283 +2022-05-14 10:40:37.138068: saving scheduled checkpoint file... +2022-05-14 10:40:37.186910: saving checkpoint... +2022-05-14 10:40:38.032786: done, saving took 0.89 seconds +2022-05-14 10:40:38.048955: done +2022-05-14 10:41:22.193665: This epoch took 506.353732 s + +2022-05-14 10:41:22.194009: +epoch: 47 +2022-05-14 10:48:37.478770: train loss : 0.0552 +2022-05-14 10:49:02.627280: validation loss: 0.0512 +2022-05-14 10:49:02.628250: Average global foreground Dice: [0.9409, 0.9304, 0.916] +2022-05-14 10:49:02.628340: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 10:49:03.028575: lr: 0.000282 +2022-05-14 10:49:03.028765: saving scheduled checkpoint file... +2022-05-14 10:49:03.078842: saving checkpoint... +2022-05-14 10:49:03.901092: done, saving took 0.87 seconds +2022-05-14 10:49:03.917496: done +2022-05-14 10:49:23.594959: This epoch took 481.400851 s + +2022-05-14 10:49:23.595217: +epoch: 48 +2022-05-14 10:56:39.177216: train loss : 0.0527 +2022-05-14 10:57:04.232867: validation loss: 0.0575 +2022-05-14 10:57:04.233497: Average global foreground Dice: [0.946, 0.9441, 0.9183] +2022-05-14 10:57:04.233598: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 10:57:04.635081: lr: 0.000281 +2022-05-14 10:57:04.635270: saving scheduled checkpoint file... +2022-05-14 10:57:04.684407: saving checkpoint... +2022-05-14 10:57:05.500733: done, saving took 0.87 seconds +2022-05-14 10:57:05.519391: done +2022-05-14 10:57:05.561342: saving checkpoint... +2022-05-14 10:57:06.366806: done, saving took 0.85 seconds +2022-05-14 10:57:47.826063: This epoch took 504.230738 s + +2022-05-14 10:57:47.826360: +epoch: 49 +2022-05-14 11:05:03.727930: train loss : 0.0520 +2022-05-14 11:05:28.839865: validation loss: 0.0532 +2022-05-14 11:05:28.840509: Average global foreground Dice: [0.9493, 0.9335, 0.9187] +2022-05-14 11:05:28.840606: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 11:05:29.246027: lr: 0.00028 +2022-05-14 11:05:29.246220: saving scheduled checkpoint file... +2022-05-14 11:05:29.295383: saving checkpoint... +2022-05-14 11:05:30.056000: done, saving took 0.81 seconds +2022-05-14 11:05:30.065945: done +2022-05-14 11:05:30.095086: saving checkpoint... +2022-05-14 11:05:30.768612: done, saving took 0.70 seconds +2022-05-14 11:05:48.615204: This epoch took 480.788739 s + +2022-05-14 11:05:48.615458: +epoch: 50 +2022-05-14 11:13:04.227938: train loss : 0.0493 +2022-05-14 11:13:29.298988: validation loss: 0.0600 +2022-05-14 11:13:29.299703: Average global foreground Dice: [0.9465, 0.9288, 0.9185] +2022-05-14 11:13:29.299789: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 11:13:29.719730: lr: 0.000279 +2022-05-14 11:13:29.719959: saving scheduled checkpoint file... +2022-05-14 11:13:29.774188: saving checkpoint... +2022-05-14 11:13:30.486000: done, saving took 0.77 seconds +2022-05-14 11:13:30.501129: done +2022-05-14 11:13:30.536218: saving checkpoint... +2022-05-14 11:13:31.271106: done, saving took 0.77 seconds +2022-05-14 11:13:49.365297: This epoch took 480.749738 s + +2022-05-14 11:13:49.365591: +epoch: 51 +2022-05-14 11:21:05.732470: train loss : 0.0549 +2022-05-14 11:21:30.815653: validation loss: 0.0529 +2022-05-14 11:21:30.816603: Average global foreground Dice: [0.943, 0.9204, 0.9127] +2022-05-14 11:21:30.816712: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 11:21:31.284189: lr: 0.000278 +2022-05-14 11:21:31.284381: saving scheduled checkpoint file... +2022-05-14 11:21:31.336229: saving checkpoint... +2022-05-14 11:21:32.155338: done, saving took 0.87 seconds +2022-05-14 11:21:32.173736: done +2022-05-14 11:21:51.373163: This epoch took 482.007469 s + +2022-05-14 11:21:51.373450: +epoch: 52 +2022-05-14 11:29:07.118413: train loss : 0.0494 +2022-05-14 11:29:32.213391: validation loss: 0.0571 +2022-05-14 11:29:32.214087: Average global foreground Dice: [0.9397, 0.9397, 0.9128] +2022-05-14 11:29:32.214172: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 11:29:32.620225: lr: 0.000277 +2022-05-14 11:29:32.620415: saving scheduled checkpoint file... +2022-05-14 11:29:32.670987: saving checkpoint... +2022-05-14 11:29:33.436723: done, saving took 0.82 seconds +2022-05-14 11:29:33.452259: done +2022-05-14 11:29:50.880097: This epoch took 479.506547 s + +2022-05-14 11:29:50.880382: +epoch: 53 +2022-05-14 11:37:06.329877: train loss : 0.0498 +2022-05-14 11:37:31.448315: validation loss: 0.0598 +2022-05-14 11:37:31.448988: Average global foreground Dice: [0.9393, 0.9242, 0.9155] +2022-05-14 11:37:31.449074: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 11:37:31.855312: lr: 0.000276 +2022-05-14 11:37:31.855510: saving scheduled checkpoint file... +2022-05-14 11:37:31.905035: saving checkpoint... +2022-05-14 11:37:32.670100: done, saving took 0.81 seconds +2022-05-14 11:37:32.686081: done +2022-05-14 11:37:50.481186: This epoch took 479.600698 s + +2022-05-14 11:37:50.481466: +epoch: 54 +2022-05-14 11:45:06.273216: train loss : 0.0487 +2022-05-14 11:45:31.315060: validation loss: 0.0525 +2022-05-14 11:45:31.315748: Average global foreground Dice: [0.9439, 0.9356, 0.9193] +2022-05-14 11:45:31.315834: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 11:45:31.719921: lr: 0.000275 +2022-05-14 11:45:31.720125: saving scheduled checkpoint file... +2022-05-14 11:45:31.769522: saving checkpoint... +2022-05-14 11:45:32.495808: done, saving took 0.78 seconds +2022-05-14 11:45:32.514467: done +2022-05-14 11:45:50.698776: This epoch took 480.217210 s + +2022-05-14 11:45:50.699053: +epoch: 55 +2022-05-14 11:53:06.338727: train loss : 0.0515 +2022-05-14 11:53:31.418519: validation loss: 0.0582 +2022-05-14 11:53:31.419214: Average global foreground Dice: [0.9479, 0.9328, 0.9158] +2022-05-14 11:53:31.419300: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 11:53:31.895233: lr: 0.000274 +2022-05-14 11:53:31.895426: saving scheduled checkpoint file... +2022-05-14 11:53:31.948313: saving checkpoint... +2022-05-14 11:53:32.766047: done, saving took 0.87 seconds +2022-05-14 11:53:32.784482: done +2022-05-14 11:53:56.905860: This epoch took 486.206702 s + +2022-05-14 11:53:56.906223: +epoch: 56 +2022-05-14 12:01:12.032746: train loss : 0.0481 +2022-05-14 12:01:37.139187: validation loss: 0.0550 +2022-05-14 12:01:37.139838: Average global foreground Dice: [0.9463, 0.9366, 0.9197] +2022-05-14 12:01:37.139924: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 12:01:37.546425: lr: 0.000272 +2022-05-14 12:01:37.546632: saving scheduled checkpoint file... +2022-05-14 12:01:37.597810: saving checkpoint... +2022-05-14 12:01:38.363747: done, saving took 0.82 seconds +2022-05-14 12:01:38.379780: done +2022-05-14 12:01:38.414695: saving checkpoint... +2022-05-14 12:01:39.160733: done, saving took 0.78 seconds +2022-05-14 12:01:55.136892: This epoch took 478.230433 s + +2022-05-14 12:01:55.137246: +epoch: 57 +2022-05-14 12:09:10.894910: train loss : 0.0492 +2022-05-14 12:09:35.961807: validation loss: 0.0574 +2022-05-14 12:09:35.962424: Average global foreground Dice: [0.9466, 0.9319, 0.9144] +2022-05-14 12:09:35.962669: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 12:09:36.377097: lr: 0.000271 +2022-05-14 12:09:36.377304: saving scheduled checkpoint file... +2022-05-14 12:09:36.425709: saving checkpoint... +2022-05-14 12:09:37.187387: done, saving took 0.81 seconds +2022-05-14 12:09:37.196887: done +2022-05-14 12:09:37.226005: saving checkpoint... +2022-05-14 12:09:37.934234: done, saving took 0.74 seconds +2022-05-14 12:09:54.858684: This epoch took 479.721333 s + +2022-05-14 12:09:54.858965: +epoch: 58 +2022-05-14 12:17:10.968792: train loss : 0.0519 +2022-05-14 12:17:36.038600: validation loss: 0.0502 +2022-05-14 12:17:36.039438: Average global foreground Dice: [0.9491, 0.9361, 0.9176] +2022-05-14 12:17:36.039524: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 12:17:36.452980: lr: 0.00027 +2022-05-14 12:17:36.453176: saving scheduled checkpoint file... +2022-05-14 12:17:36.503147: saving checkpoint... +2022-05-14 12:17:37.271455: done, saving took 0.82 seconds +2022-05-14 12:17:37.289144: done +2022-05-14 12:17:37.324744: saving checkpoint... +2022-05-14 12:17:38.034707: done, saving took 0.75 seconds +2022-05-14 12:18:07.614426: This epoch took 492.755353 s + +2022-05-14 12:18:07.614742: +epoch: 59 +2022-05-14 12:25:22.963156: train loss : 0.0519 +2022-05-14 12:25:48.021912: validation loss: 0.0561 +2022-05-14 12:25:48.022594: Average global foreground Dice: [0.9496, 0.9339, 0.9164] +2022-05-14 12:25:48.022682: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 12:25:48.440449: lr: 0.000269 +2022-05-14 12:25:48.440651: saving scheduled checkpoint file... +2022-05-14 12:25:48.487324: saving checkpoint... +2022-05-14 12:25:49.307440: done, saving took 0.87 seconds +2022-05-14 12:25:49.326615: done +2022-05-14 12:25:49.367435: saving checkpoint... +2022-05-14 12:25:50.173121: done, saving took 0.85 seconds +2022-05-14 12:26:11.090496: This epoch took 483.475646 s + +2022-05-14 12:26:11.090849: +epoch: 60 +2022-05-14 12:33:26.760693: train loss : 0.0507 +2022-05-14 12:33:51.843664: validation loss: 0.0500 +2022-05-14 12:33:51.844305: Average global foreground Dice: [0.9427, 0.9356, 0.9187] +2022-05-14 12:33:51.844404: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 12:33:52.255011: lr: 0.000268 +2022-05-14 12:33:52.255210: saving scheduled checkpoint file... +2022-05-14 12:33:52.304140: saving checkpoint... +2022-05-14 12:33:53.115944: done, saving took 0.86 seconds +2022-05-14 12:33:53.133734: done +2022-05-14 12:33:53.175117: saving checkpoint... +2022-05-14 12:33:53.937201: done, saving took 0.80 seconds +2022-05-14 12:34:14.872354: This epoch took 483.781405 s + +2022-05-14 12:34:14.872643: +epoch: 61 +2022-05-14 12:41:31.034999: train loss : 0.0510 +2022-05-14 12:41:56.074751: validation loss: 0.0603 +2022-05-14 12:41:56.075381: Average global foreground Dice: [0.9432, 0.9365, 0.9211] +2022-05-14 12:41:56.075463: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 12:41:56.481468: lr: 0.000267 +2022-05-14 12:41:56.481664: saving scheduled checkpoint file... +2022-05-14 12:41:56.530279: saving checkpoint... +2022-05-14 12:41:57.362446: done, saving took 0.88 seconds +2022-05-14 12:41:57.373739: done +2022-05-14 12:41:57.403572: saving checkpoint... +2022-05-14 12:41:58.194037: done, saving took 0.82 seconds +2022-05-14 12:42:21.560551: This epoch took 486.687810 s + +2022-05-14 12:42:21.560889: +epoch: 62 +2022-05-14 12:49:37.899249: train loss : 0.0494 +2022-05-14 12:50:02.955160: validation loss: 0.0557 +2022-05-14 12:50:02.955790: Average global foreground Dice: [0.9449, 0.9353, 0.9214] +2022-05-14 12:50:02.955994: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 12:50:03.363441: lr: 0.000265 +2022-05-14 12:50:03.363645: saving scheduled checkpoint file... +2022-05-14 12:50:03.417016: saving checkpoint... +2022-05-14 12:50:04.145254: done, saving took 0.78 seconds +2022-05-14 12:50:04.161797: done +2022-05-14 12:50:04.198955: saving checkpoint... +2022-05-14 12:50:04.930435: done, saving took 0.77 seconds +2022-05-14 12:50:20.179221: This epoch took 478.618227 s + +2022-05-14 12:50:20.179499: +epoch: 63 +2022-05-14 12:57:36.213533: train loss : 0.0486 +2022-05-14 12:58:01.217229: validation loss: 0.0481 +2022-05-14 12:58:01.217868: Average global foreground Dice: [0.9465, 0.9337, 0.9198] +2022-05-14 12:58:01.217952: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 12:58:01.633623: lr: 0.000264 +2022-05-14 12:58:01.633823: saving scheduled checkpoint file... +2022-05-14 12:58:01.681514: saving checkpoint... +2022-05-14 12:58:02.501638: done, saving took 0.87 seconds +2022-05-14 12:58:02.517539: done +2022-05-14 12:58:02.553334: saving checkpoint... +2022-05-14 12:58:03.352008: done, saving took 0.83 seconds +2022-05-14 12:58:22.401591: This epoch took 482.221983 s + +2022-05-14 12:58:22.401865: +epoch: 64 +2022-05-14 13:05:38.296693: train loss : 0.0487 +2022-05-14 13:06:03.304019: validation loss: 0.0601 +2022-05-14 13:06:03.304832: Average global foreground Dice: [0.9423, 0.9335, 0.9184] +2022-05-14 13:06:03.304928: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 13:06:03.719200: lr: 0.000263 +2022-05-14 13:06:03.719380: saving scheduled checkpoint file... +2022-05-14 13:06:03.768230: saving checkpoint... +2022-05-14 13:06:04.597132: done, saving took 0.88 seconds +2022-05-14 13:06:04.614898: done +2022-05-14 13:06:22.760691: This epoch took 480.358727 s + +2022-05-14 13:06:22.760997: +epoch: 65 +2022-05-14 13:13:38.903421: train loss : 0.0506 +2022-05-14 13:14:03.923624: validation loss: 0.0574 +2022-05-14 13:14:03.924292: Average global foreground Dice: [0.9499, 0.9317, 0.9191] +2022-05-14 13:14:03.924375: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 13:14:04.332010: lr: 0.000261 +2022-05-14 13:14:04.332199: saving scheduled checkpoint file... +2022-05-14 13:14:04.380931: saving checkpoint... +2022-05-14 13:14:05.183349: done, saving took 0.85 seconds +2022-05-14 13:14:05.193209: done +2022-05-14 13:14:05.224144: saving checkpoint... +2022-05-14 13:14:05.919795: done, saving took 0.73 seconds +2022-05-14 13:14:21.895953: This epoch took 479.134852 s + +2022-05-14 13:14:21.896224: +epoch: 66 +2022-05-14 13:21:38.024529: train loss : 0.0478 +2022-05-14 13:22:03.053507: validation loss: 0.0547 +2022-05-14 13:22:03.054167: Average global foreground Dice: [0.9476, 0.9355, 0.9175] +2022-05-14 13:22:03.054252: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 13:22:03.461617: lr: 0.00026 +2022-05-14 13:22:03.461815: saving scheduled checkpoint file... +2022-05-14 13:22:03.514811: saving checkpoint... +2022-05-14 13:22:04.341796: done, saving took 0.88 seconds +2022-05-14 13:22:04.357965: done +2022-05-14 13:22:04.394172: saving checkpoint... +2022-05-14 13:22:05.193274: done, saving took 0.84 seconds +2022-05-14 13:22:23.618749: This epoch took 481.722415 s + +2022-05-14 13:22:23.619031: +epoch: 67 +2022-05-14 13:29:39.853613: train loss : 0.0505 +2022-05-14 13:30:04.865600: validation loss: 0.0464 +2022-05-14 13:30:04.866311: Average global foreground Dice: [0.9497, 0.9392, 0.9212] +2022-05-14 13:30:04.866395: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 13:30:05.272656: lr: 0.000259 +2022-05-14 13:30:05.272850: saving scheduled checkpoint file... +2022-05-14 13:30:05.322009: saving checkpoint... +2022-05-14 13:30:06.038018: done, saving took 0.77 seconds +2022-05-14 13:30:06.053599: done +2022-05-14 13:30:06.088164: saving checkpoint... +2022-05-14 13:30:06.798669: done, saving took 0.74 seconds +2022-05-14 13:30:34.864775: This epoch took 491.245642 s + +2022-05-14 13:30:34.865058: +epoch: 68 +2022-05-14 13:37:51.161195: train loss : 0.0459 +2022-05-14 13:38:16.192312: validation loss: 0.0526 +2022-05-14 13:38:16.192979: Average global foreground Dice: [0.9456, 0.9279, 0.918] +2022-05-14 13:38:16.193062: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 13:38:16.600037: lr: 0.000257 +2022-05-14 13:38:16.600227: saving scheduled checkpoint file... +2022-05-14 13:38:16.649075: saving checkpoint... +2022-05-14 13:38:17.470062: done, saving took 0.87 seconds +2022-05-14 13:38:17.481897: done +2022-05-14 13:38:37.755698: This epoch took 482.890538 s + +2022-05-14 13:38:37.755991: +epoch: 69 +2022-05-14 13:45:54.101089: train loss : 0.0519 +2022-05-14 13:46:19.085113: validation loss: 0.0676 +2022-05-14 13:46:19.085829: Average global foreground Dice: [0.9476, 0.929, 0.9161] +2022-05-14 13:46:19.085952: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 13:46:19.500187: lr: 0.000256 +2022-05-14 13:46:19.500386: saving scheduled checkpoint file... +2022-05-14 13:46:19.553800: saving checkpoint... +2022-05-14 13:46:20.377777: done, saving took 0.88 seconds +2022-05-14 13:46:20.393907: done +2022-05-14 13:46:39.295713: This epoch took 481.539621 s + +2022-05-14 13:46:39.295990: +epoch: 70 +2022-05-14 13:53:55.541899: train loss : 0.0483 +2022-05-14 13:54:20.630864: validation loss: 0.0482 +2022-05-14 13:54:20.631487: Average global foreground Dice: [0.9477, 0.9383, 0.9183] +2022-05-14 13:54:20.631583: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 13:54:21.039707: lr: 0.000255 +2022-05-14 13:54:21.039909: saving scheduled checkpoint file... +2022-05-14 13:54:21.091213: saving checkpoint... +2022-05-14 13:54:21.906958: done, saving took 0.87 seconds +2022-05-14 13:54:21.923459: done +2022-05-14 13:54:49.131530: This epoch took 489.835439 s + +2022-05-14 13:54:49.131804: +epoch: 71 +2022-05-14 14:02:04.763535: train loss : 0.0483 +2022-05-14 14:02:29.785786: validation loss: 0.0527 +2022-05-14 14:02:29.786466: Average global foreground Dice: [0.9473, 0.9358, 0.9178] +2022-05-14 14:02:29.786565: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 14:02:30.199324: lr: 0.000253 +2022-05-14 14:02:30.199531: saving scheduled checkpoint file... +2022-05-14 14:02:30.249021: saving checkpoint... +2022-05-14 14:02:31.065063: done, saving took 0.87 seconds +2022-05-14 14:02:31.076840: done +2022-05-14 14:02:57.207752: This epoch took 488.075844 s + +2022-05-14 14:02:57.208030: +epoch: 72 +2022-05-14 14:10:12.463399: train loss : 0.0469 +2022-05-14 14:10:37.631416: validation loss: 0.0523 +2022-05-14 14:10:37.632250: Average global foreground Dice: [0.95, 0.9341, 0.9187] +2022-05-14 14:10:37.632335: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 14:10:38.044865: lr: 0.000252 +2022-05-14 14:10:38.045053: saving scheduled checkpoint file... +2022-05-14 14:10:38.094170: saving checkpoint... +2022-05-14 14:10:38.908530: done, saving took 0.86 seconds +2022-05-14 14:10:38.924838: done +2022-05-14 14:10:38.960852: saving checkpoint... +2022-05-14 14:10:39.766911: done, saving took 0.84 seconds +2022-05-14 14:10:56.893947: This epoch took 479.685818 s + +2022-05-14 14:10:56.894251: +epoch: 73 +2022-05-14 14:18:12.362890: train loss : 0.0486 +2022-05-14 14:18:37.433852: validation loss: 0.0393 +2022-05-14 14:18:37.434489: Average global foreground Dice: [0.9509, 0.9348, 0.918] +2022-05-14 14:18:37.434589: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 14:18:37.853780: lr: 0.00025 +2022-05-14 14:18:37.853976: saving scheduled checkpoint file... +2022-05-14 14:18:37.903292: saving checkpoint... +2022-05-14 14:18:38.715099: done, saving took 0.86 seconds +2022-05-14 14:18:38.732807: done +2022-05-14 14:18:38.773628: saving checkpoint... +2022-05-14 14:18:39.627682: done, saving took 0.89 seconds +2022-05-14 14:18:57.452053: This epoch took 480.557699 s + +2022-05-14 14:18:57.452317: +epoch: 74 +2022-05-14 14:26:13.101753: train loss : 0.0465 +2022-05-14 14:26:38.145099: validation loss: 0.0498 +2022-05-14 14:26:38.145775: Average global foreground Dice: [0.9496, 0.941, 0.9246] +2022-05-14 14:26:38.145862: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 14:26:38.560427: lr: 0.000249 +2022-05-14 14:26:38.560637: saving scheduled checkpoint file... +2022-05-14 14:26:38.608173: saving checkpoint... +2022-05-14 14:26:39.429211: done, saving took 0.87 seconds +2022-05-14 14:26:39.445239: done +2022-05-14 14:26:39.481245: saving checkpoint... +2022-05-14 14:26:40.293418: done, saving took 0.85 seconds +2022-05-14 14:27:02.030002: This epoch took 484.577587 s + +2022-05-14 14:27:02.030261: +epoch: 75 +2022-05-14 14:34:17.260000: train loss : 0.0484 +2022-05-14 14:34:42.372145: validation loss: 0.0512 +2022-05-14 14:34:42.372979: Average global foreground Dice: [0.9479, 0.9382, 0.9149] +2022-05-14 14:34:42.373063: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 14:34:42.782805: lr: 0.000247 +2022-05-14 14:34:42.782995: saving scheduled checkpoint file... +2022-05-14 14:34:42.832173: saving checkpoint... +2022-05-14 14:34:43.651248: done, saving took 0.87 seconds +2022-05-14 14:34:43.669526: done +2022-05-14 14:34:43.710535: saving checkpoint... +2022-05-14 14:34:44.513585: done, saving took 0.84 seconds +2022-05-14 14:35:00.431293: This epoch took 478.400907 s + +2022-05-14 14:35:00.435889: +epoch: 76 +2022-05-14 14:42:16.368805: train loss : 0.0475 +2022-05-14 14:42:41.395893: validation loss: 0.0537 +2022-05-14 14:42:41.396676: Average global foreground Dice: [0.9505, 0.931, 0.9165] +2022-05-14 14:42:41.396761: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 14:42:41.806377: lr: 0.000246 +2022-05-14 14:42:41.806576: saving scheduled checkpoint file... +2022-05-14 14:42:41.855021: saving checkpoint... +2022-05-14 14:42:42.681327: done, saving took 0.87 seconds +2022-05-14 14:42:42.692015: done +2022-05-14 14:42:59.479343: This epoch took 479.043330 s + +2022-05-14 14:42:59.479630: +epoch: 77 +2022-05-14 14:50:15.560486: train loss : 0.0453 +2022-05-14 14:50:40.544399: validation loss: 0.0587 +2022-05-14 14:50:40.545076: Average global foreground Dice: [0.9503, 0.9385, 0.918] +2022-05-14 14:50:40.545161: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 14:50:40.955481: lr: 0.000244 +2022-05-14 14:50:40.955683: saving scheduled checkpoint file... +2022-05-14 14:50:41.008860: saving checkpoint... +2022-05-14 14:50:41.825910: done, saving took 0.87 seconds +2022-05-14 14:50:41.843984: done +2022-05-14 14:50:41.885903: saving checkpoint... +2022-05-14 14:50:42.703230: done, saving took 0.86 seconds +2022-05-14 14:51:02.056805: This epoch took 482.577074 s + +2022-05-14 14:51:02.057134: +epoch: 78 +2022-05-14 14:58:17.345772: train loss : 0.0489 +2022-05-14 14:58:42.408642: validation loss: 0.0550 +2022-05-14 14:58:42.409267: Average global foreground Dice: [0.9479, 0.9313, 0.9246] +2022-05-14 14:58:42.409354: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 14:58:42.828310: lr: 0.000243 +2022-05-14 14:58:42.828512: saving scheduled checkpoint file... +2022-05-14 14:58:42.877950: saving checkpoint... +2022-05-14 14:58:43.696024: done, saving took 0.87 seconds +2022-05-14 14:58:43.714358: done +2022-05-14 14:58:43.755277: saving checkpoint... +2022-05-14 14:58:44.509405: done, saving took 0.79 seconds +2022-05-14 14:59:02.605366: This epoch took 480.548129 s + +2022-05-14 14:59:02.605695: +epoch: 79 +2022-05-14 15:06:18.540177: train loss : 0.0499 +2022-05-14 15:06:43.573042: validation loss: 0.0550 +2022-05-14 15:06:43.573705: Average global foreground Dice: [0.9501, 0.9343, 0.9216] +2022-05-14 15:06:43.573794: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 15:06:43.988331: lr: 0.000241 +2022-05-14 15:06:43.988514: saving scheduled checkpoint file... +2022-05-14 15:06:44.036786: saving checkpoint... +2022-05-14 15:06:44.870086: done, saving took 0.88 seconds +2022-05-14 15:06:44.888374: done +2022-05-14 15:06:44.929496: saving checkpoint... +2022-05-14 15:06:45.731596: done, saving took 0.84 seconds +2022-05-14 15:07:03.161234: This epoch took 480.555434 s + +2022-05-14 15:07:03.161523: +epoch: 80 +2022-05-14 15:14:18.904386: train loss : 0.0490 +2022-05-14 15:14:43.946985: validation loss: 0.0535 +2022-05-14 15:14:43.947632: Average global foreground Dice: [0.9489, 0.9361, 0.9176] +2022-05-14 15:14:43.947719: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 15:14:44.373635: lr: 0.00024 +2022-05-14 15:14:44.373839: saving scheduled checkpoint file... +2022-05-14 15:14:44.421949: saving checkpoint... +2022-05-14 15:14:45.247159: done, saving took 0.87 seconds +2022-05-14 15:14:45.265359: done +2022-05-14 15:14:45.308188: saving checkpoint... +2022-05-14 15:14:46.120704: done, saving took 0.86 seconds +2022-05-14 15:15:02.393326: This epoch took 479.231677 s + +2022-05-14 15:15:02.393629: +epoch: 81 +2022-05-14 15:22:17.899734: train loss : 0.0451 +2022-05-14 15:22:42.948039: validation loss: 0.0451 +2022-05-14 15:22:42.948659: Average global foreground Dice: [0.9453, 0.9318, 0.9164] +2022-05-14 15:22:42.948744: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 15:22:43.358922: lr: 0.000238 +2022-05-14 15:22:43.359111: saving scheduled checkpoint file... +2022-05-14 15:22:43.407964: saving checkpoint... +2022-05-14 15:22:44.227090: done, saving took 0.87 seconds +2022-05-14 15:22:44.262180: done +2022-05-14 15:23:27.976645: This epoch took 505.582912 s + +2022-05-14 15:23:27.976918: +epoch: 82 +2022-05-14 15:30:43.431351: train loss : 0.0468 +2022-05-14 15:31:08.449099: validation loss: 0.0567 +2022-05-14 15:31:08.449841: Average global foreground Dice: [0.9453, 0.9314, 0.9222] +2022-05-14 15:31:08.449926: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 15:31:08.851963: lr: 0.000237 +2022-05-14 15:31:08.852159: saving scheduled checkpoint file... +2022-05-14 15:31:08.899256: saving checkpoint... +2022-05-14 15:31:09.717399: done, saving took 0.87 seconds +2022-05-14 15:31:09.733700: done +2022-05-14 15:31:29.421741: This epoch took 481.444727 s + +2022-05-14 15:31:29.422019: +epoch: 83 +2022-05-14 15:38:45.597024: train loss : 0.0452 +2022-05-14 15:39:10.701956: validation loss: 0.0498 +2022-05-14 15:39:10.702614: Average global foreground Dice: [0.9521, 0.9411, 0.9232] +2022-05-14 15:39:10.702704: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 15:39:11.111489: lr: 0.000235 +2022-05-14 15:39:11.111692: saving scheduled checkpoint file... +2022-05-14 15:39:11.162793: saving checkpoint... +2022-05-14 15:39:11.919219: done, saving took 0.81 seconds +2022-05-14 15:39:11.934524: done +2022-05-14 15:39:11.969384: saving checkpoint... +2022-05-14 15:39:12.680605: done, saving took 0.75 seconds +2022-05-14 15:39:31.639874: This epoch took 482.217752 s + +2022-05-14 15:39:31.640167: +epoch: 84 +2022-05-14 15:46:47.387680: train loss : 0.0447 +2022-05-14 15:47:12.406368: validation loss: 0.0450 +2022-05-14 15:47:12.406995: Average global foreground Dice: [0.9495, 0.9345, 0.916] +2022-05-14 15:47:12.407174: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 15:47:12.809318: lr: 0.000233 +2022-05-14 15:47:12.809515: saving scheduled checkpoint file... +2022-05-14 15:47:12.857301: saving checkpoint... +2022-05-14 15:47:13.611855: done, saving took 0.80 seconds +2022-05-14 15:47:13.631316: done +2022-05-14 15:47:30.658773: This epoch took 479.018505 s + +2022-05-14 15:47:30.659050: +epoch: 85 +2022-05-14 15:54:46.609056: train loss : 0.0464 +2022-05-14 15:55:11.676665: validation loss: 0.0515 +2022-05-14 15:55:11.677457: Average global foreground Dice: [0.9491, 0.9298, 0.9181] +2022-05-14 15:55:11.677544: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 15:55:12.085676: lr: 0.000232 +2022-05-14 15:55:12.085874: saving scheduled checkpoint file... +2022-05-14 15:55:12.135458: saving checkpoint... +2022-05-14 15:55:12.884207: done, saving took 0.80 seconds +2022-05-14 15:55:12.900041: done +2022-05-14 15:55:35.135420: This epoch took 484.476264 s + +2022-05-14 15:55:35.135748: +epoch: 86 +2022-05-14 16:02:51.422202: train loss : 0.0493 +2022-05-14 16:03:16.516766: validation loss: 0.0631 +2022-05-14 16:03:16.517488: Average global foreground Dice: [0.9507, 0.9313, 0.921] +2022-05-14 16:03:16.517587: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 16:03:16.925303: lr: 0.00023 +2022-05-14 16:03:16.925539: saving scheduled checkpoint file... +2022-05-14 16:03:16.976523: saving checkpoint... +2022-05-14 16:03:17.723072: done, saving took 0.80 seconds +2022-05-14 16:03:17.738148: done +2022-05-14 16:03:33.775785: This epoch took 478.639933 s + +2022-05-14 16:03:33.776045: +epoch: 87 +2022-05-14 16:10:49.639984: train loss : 0.0433 +2022-05-14 16:11:14.622009: validation loss: 0.0530 +2022-05-14 16:11:14.622747: Average global foreground Dice: [0.9515, 0.9365, 0.9126] +2022-05-14 16:11:14.622835: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 16:11:15.027217: lr: 0.000228 +2022-05-14 16:11:15.027419: saving scheduled checkpoint file... +2022-05-14 16:11:15.076259: saving checkpoint... +2022-05-14 16:11:15.821227: done, saving took 0.79 seconds +2022-05-14 16:11:15.841690: done +2022-05-14 16:11:32.193537: This epoch took 478.417389 s + +2022-05-14 16:11:32.193849: +epoch: 88 +2022-05-14 16:18:48.502066: train loss : 0.0453 +2022-05-14 16:19:13.569345: validation loss: 0.0529 +2022-05-14 16:19:13.569975: Average global foreground Dice: [0.9491, 0.9375, 0.9168] +2022-05-14 16:19:13.570065: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 16:19:13.978672: lr: 0.000227 +2022-05-14 16:19:13.978899: saving scheduled checkpoint file... +2022-05-14 16:19:14.032217: saving checkpoint... +2022-05-14 16:19:14.781670: done, saving took 0.80 seconds +2022-05-14 16:19:14.798500: done +2022-05-14 16:19:31.912587: This epoch took 479.718622 s + +2022-05-14 16:19:31.912866: +epoch: 89 +2022-05-14 16:26:48.266207: train loss : 0.0469 +2022-05-14 16:27:13.321880: validation loss: 0.0458 +2022-05-14 16:27:13.322542: Average global foreground Dice: [0.9476, 0.9323, 0.9225] +2022-05-14 16:27:13.322637: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 16:27:13.725563: lr: 0.000225 +2022-05-14 16:27:13.725742: saving scheduled checkpoint file... +2022-05-14 16:27:13.774972: saving checkpoint... +2022-05-14 16:27:14.534640: done, saving took 0.81 seconds +2022-05-14 16:27:14.550890: done +2022-05-14 16:27:33.518408: This epoch took 481.605432 s + +2022-05-14 16:27:33.518698: +epoch: 90 +2022-05-14 16:34:49.688649: train loss : 0.0472 +2022-05-14 16:35:14.733539: validation loss: 0.0501 +2022-05-14 16:35:14.734441: Average global foreground Dice: [0.9512, 0.9444, 0.927] +2022-05-14 16:35:14.734525: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 16:35:15.147619: lr: 0.000223 +2022-05-14 16:35:15.147812: saving scheduled checkpoint file... +2022-05-14 16:35:15.197755: saving checkpoint... +2022-05-14 16:35:15.928483: done, saving took 0.78 seconds +2022-05-14 16:35:15.944551: done +2022-05-14 16:35:15.978912: saving checkpoint... +2022-05-14 16:35:16.668980: done, saving took 0.72 seconds +2022-05-14 16:35:42.146412: This epoch took 488.627614 s + +2022-05-14 16:35:42.146694: +epoch: 91 +2022-05-14 16:42:58.214911: train loss : 0.0436 +2022-05-14 16:43:23.308286: validation loss: 0.0495 +2022-05-14 16:43:23.308900: Average global foreground Dice: [0.9495, 0.94, 0.921] +2022-05-14 16:43:23.308989: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 16:43:23.717398: lr: 0.000222 +2022-05-14 16:43:23.717604: saving scheduled checkpoint file... +2022-05-14 16:43:23.766117: saving checkpoint... +2022-05-14 16:43:24.514363: done, saving took 0.80 seconds +2022-05-14 16:43:24.523816: done +2022-05-14 16:43:24.553052: saving checkpoint... +2022-05-14 16:43:25.250119: done, saving took 0.73 seconds +2022-05-14 16:44:03.778809: This epoch took 501.632000 s + +2022-05-14 16:44:03.779109: +epoch: 92 +2022-05-14 16:51:19.761024: train loss : 0.0447 +2022-05-14 16:51:44.753657: validation loss: 0.0496 +2022-05-14 16:51:44.754694: Average global foreground Dice: [0.9518, 0.9258, 0.9145] +2022-05-14 16:51:44.754785: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 16:51:45.163461: lr: 0.00022 +2022-05-14 16:51:45.163668: saving scheduled checkpoint file... +2022-05-14 16:51:45.216418: saving checkpoint... +2022-05-14 16:51:46.036040: done, saving took 0.87 seconds +2022-05-14 16:51:46.054257: done +2022-05-14 16:52:06.650711: This epoch took 482.871496 s + +2022-05-14 16:52:06.650992: +epoch: 93 +2022-05-14 16:59:23.173148: train loss : 0.0443 +2022-05-14 16:59:48.209521: validation loss: 0.0586 +2022-05-14 16:59:48.210249: Average global foreground Dice: [0.9481, 0.923, 0.9221] +2022-05-14 16:59:48.210373: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 16:59:48.614981: lr: 0.000218 +2022-05-14 16:59:48.615176: saving scheduled checkpoint file... +2022-05-14 16:59:48.665433: saving checkpoint... +2022-05-14 16:59:49.485003: done, saving took 0.87 seconds +2022-05-14 16:59:49.503235: done +2022-05-14 17:00:06.173388: This epoch took 479.522295 s + +2022-05-14 17:00:06.173684: +epoch: 94 +2022-05-14 17:07:22.017112: train loss : 0.0422 +2022-05-14 17:07:47.049740: validation loss: 0.0568 +2022-05-14 17:07:47.050337: Average global foreground Dice: [0.95, 0.9369, 0.92] +2022-05-14 17:07:47.050501: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 17:07:47.457952: lr: 0.000216 +2022-05-14 17:07:47.458142: saving scheduled checkpoint file... +2022-05-14 17:07:47.506006: saving checkpoint... +2022-05-14 17:07:48.331460: done, saving took 0.87 seconds +2022-05-14 17:07:48.349648: done +2022-05-14 17:08:06.640106: This epoch took 480.466321 s + +2022-05-14 17:08:06.640359: +epoch: 95 +2022-05-14 17:15:23.302752: train loss : 0.0459 +2022-05-14 17:15:48.387436: validation loss: 0.0607 +2022-05-14 17:15:48.388093: Average global foreground Dice: [0.948, 0.9368, 0.9216] +2022-05-14 17:15:48.388184: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 17:15:48.814536: lr: 0.000215 +2022-05-14 17:15:48.814736: saving scheduled checkpoint file... +2022-05-14 17:15:48.865904: saving checkpoint... +2022-05-14 17:15:49.690058: done, saving took 0.88 seconds +2022-05-14 17:15:49.708636: done +2022-05-14 17:16:10.392999: This epoch took 483.752540 s + +2022-05-14 17:16:10.393285: +epoch: 96 +2022-05-14 17:23:26.466078: train loss : 0.0456 +2022-05-14 17:23:51.662420: validation loss: 0.0538 +2022-05-14 17:23:51.663100: Average global foreground Dice: [0.9492, 0.9374, 0.9227] +2022-05-14 17:23:51.663184: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 17:23:52.070416: lr: 0.000213 +2022-05-14 17:23:52.070626: saving scheduled checkpoint file... +2022-05-14 17:23:52.118575: saving checkpoint... +2022-05-14 17:23:52.835184: done, saving took 0.76 seconds +2022-05-14 17:23:52.851403: done +2022-05-14 17:24:11.608596: This epoch took 481.215174 s + +2022-05-14 17:24:11.608974: +epoch: 97 +2022-05-14 17:31:28.049122: train loss : 0.0440 +2022-05-14 17:31:53.074143: validation loss: 0.0532 +2022-05-14 17:31:53.074790: Average global foreground Dice: [0.9487, 0.9375, 0.9157] +2022-05-14 17:31:53.074876: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 17:31:53.480171: lr: 0.000211 +2022-05-14 17:31:53.480373: saving scheduled checkpoint file... +2022-05-14 17:31:53.529842: saving checkpoint... +2022-05-14 17:31:54.349167: done, saving took 0.87 seconds +2022-05-14 17:31:54.381073: done +2022-05-14 17:32:12.564385: This epoch took 480.955304 s + +2022-05-14 17:32:12.564896: +epoch: 98 +2022-05-14 17:39:28.242749: train loss : 0.0440 +2022-05-14 17:39:53.259470: validation loss: 0.0562 +2022-05-14 17:39:53.260106: Average global foreground Dice: [0.9504, 0.9362, 0.9208] +2022-05-14 17:39:53.260190: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 17:39:53.665585: lr: 0.000209 +2022-05-14 17:39:53.665776: saving scheduled checkpoint file... +2022-05-14 17:39:53.716365: saving checkpoint... +2022-05-14 17:39:54.532873: done, saving took 0.87 seconds +2022-05-14 17:39:54.549135: done +2022-05-14 17:40:14.399509: This epoch took 481.834505 s + +2022-05-14 17:40:14.399838: +epoch: 99 +2022-05-14 17:47:30.182148: train loss : 0.0474 +2022-05-14 17:47:55.225550: validation loss: 0.0484 +2022-05-14 17:47:55.226214: Average global foreground Dice: [0.9531, 0.9454, 0.9251] +2022-05-14 17:47:55.226297: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 17:47:55.652009: lr: 0.000207 +2022-05-14 17:47:55.652200: saving scheduled checkpoint file... +2022-05-14 17:47:55.699963: saving checkpoint... +2022-05-14 17:47:56.521890: done, saving took 0.87 seconds +2022-05-14 17:47:56.537985: done +2022-05-14 17:47:56.574339: saving checkpoint... +2022-05-14 17:47:57.374172: done, saving took 0.84 seconds +2022-05-14 17:48:16.053893: This epoch took 481.653949 s + +2022-05-14 17:48:16.054175: +epoch: 100 +2022-05-14 17:55:32.042396: train loss : 0.0453 +2022-05-14 17:55:57.097548: validation loss: 0.0593 +2022-05-14 17:55:57.098261: Average global foreground Dice: [0.9495, 0.9251, 0.9187] +2022-05-14 17:55:57.098350: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 17:55:57.510567: lr: 0.000206 +2022-05-14 17:55:57.510765: saving scheduled checkpoint file... +2022-05-14 17:55:57.558693: saving checkpoint... +2022-05-14 17:55:58.376686: done, saving took 0.87 seconds +2022-05-14 17:55:58.395037: done +2022-05-14 17:56:24.315032: This epoch took 488.260756 s + +2022-05-14 17:56:24.315381: +epoch: 101 +2022-05-14 18:03:40.444193: train loss : 0.0435 +2022-05-14 18:04:05.553802: validation loss: 0.0643 +2022-05-14 18:04:05.554501: Average global foreground Dice: [0.9485, 0.931, 0.9198] +2022-05-14 18:04:05.554595: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 18:04:05.963222: lr: 0.000204 +2022-05-14 18:04:05.963420: saving scheduled checkpoint file... +2022-05-14 18:04:06.014623: saving checkpoint... +2022-05-14 18:04:06.845421: done, saving took 0.88 seconds +2022-05-14 18:04:06.865067: done +2022-05-14 18:04:23.742180: This epoch took 479.426690 s + +2022-05-14 18:04:23.742485: +epoch: 102 +2022-05-14 18:11:39.937587: train loss : 0.0434 +2022-05-14 18:12:05.015882: validation loss: 0.0618 +2022-05-14 18:12:05.016533: Average global foreground Dice: [0.9499, 0.928, 0.9211] +2022-05-14 18:12:05.016630: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 18:12:05.423574: lr: 0.000202 +2022-05-14 18:12:05.423774: saving scheduled checkpoint file... +2022-05-14 18:12:05.475214: saving checkpoint... +2022-05-14 18:12:06.293218: done, saving took 0.87 seconds +2022-05-14 18:12:06.304930: done +2022-05-14 18:12:24.464783: This epoch took 480.722199 s + +2022-05-14 18:12:24.465076: +epoch: 103 +2022-05-14 18:19:40.745266: train loss : 0.0455 +2022-05-14 18:20:05.793018: validation loss: 0.0609 +2022-05-14 18:20:05.793648: Average global foreground Dice: [0.9447, 0.9249, 0.9165] +2022-05-14 18:20:05.793732: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 18:20:06.200373: lr: 0.0002 +2022-05-14 18:20:06.200580: saving scheduled checkpoint file... +2022-05-14 18:20:06.253698: saving checkpoint... +2022-05-14 18:20:07.082628: done, saving took 0.88 seconds +2022-05-14 18:20:07.100588: done +2022-05-14 18:20:22.892254: This epoch took 478.427080 s + +2022-05-14 18:20:22.892526: +epoch: 104 +2022-05-14 18:27:39.312812: train loss : 0.0443 +2022-05-14 18:28:04.394960: validation loss: 0.0537 +2022-05-14 18:28:04.395651: Average global foreground Dice: [0.9518, 0.9281, 0.914] +2022-05-14 18:28:04.395740: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 18:28:04.803159: lr: 0.000198 +2022-05-14 18:28:04.803357: saving scheduled checkpoint file... +2022-05-14 18:28:04.854415: saving checkpoint... +2022-05-14 18:28:05.664928: done, saving took 0.86 seconds +2022-05-14 18:28:05.680964: done +2022-05-14 18:28:23.433666: This epoch took 480.541021 s + +2022-05-14 18:28:23.433939: +epoch: 105 +2022-05-14 18:35:40.331568: train loss : 0.0430 +2022-05-14 18:36:05.387938: validation loss: 0.0579 +2022-05-14 18:36:05.388525: Average global foreground Dice: [0.949, 0.9351, 0.9188] +2022-05-14 18:36:05.388632: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 18:36:05.796096: lr: 0.000196 +2022-05-14 18:36:05.796480: saving scheduled checkpoint file... +2022-05-14 18:36:05.844551: saving checkpoint... +2022-05-14 18:36:06.610523: done, saving took 0.81 seconds +2022-05-14 18:36:06.629918: done +2022-05-14 18:36:23.640235: This epoch took 480.206193 s + +2022-05-14 18:36:23.640546: +epoch: 106 +2022-05-14 18:43:39.531628: train loss : 0.0420 +2022-05-14 18:44:04.620725: validation loss: 0.0570 +2022-05-14 18:44:04.621420: Average global foreground Dice: [0.9504, 0.9402, 0.925] +2022-05-14 18:44:04.621608: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 18:44:05.077086: lr: 0.000194 +2022-05-14 18:44:05.077286: saving scheduled checkpoint file... +2022-05-14 18:44:05.132594: saving checkpoint... +2022-05-14 18:44:05.945505: done, saving took 0.87 seconds +2022-05-14 18:44:05.963038: done +2022-05-14 18:44:23.242865: This epoch took 479.602152 s + +2022-05-14 18:44:23.243250: +epoch: 107 +2022-05-14 18:51:38.881088: train loss : 0.0434 +2022-05-14 18:52:03.917140: validation loss: 0.0717 +2022-05-14 18:52:03.917776: Average global foreground Dice: [0.9524, 0.9297, 0.9219] +2022-05-14 18:52:03.917862: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 18:52:04.326402: lr: 0.000193 +2022-05-14 18:52:04.326612: saving scheduled checkpoint file... +2022-05-14 18:52:04.374923: saving checkpoint... +2022-05-14 18:52:05.191061: done, saving took 0.86 seconds +2022-05-14 18:52:05.201735: done +2022-05-14 18:52:32.370134: This epoch took 489.126779 s + +2022-05-14 18:52:32.370405: +epoch: 108 +2022-05-14 18:59:48.494973: train loss : 0.0428 +2022-05-14 19:00:13.550404: validation loss: 0.0630 +2022-05-14 19:00:13.551045: Average global foreground Dice: [0.9481, 0.9384, 0.9188] +2022-05-14 19:00:13.551130: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 19:00:13.962140: lr: 0.000191 +2022-05-14 19:00:13.962381: saving scheduled checkpoint file... +2022-05-14 19:00:14.014889: saving checkpoint... +2022-05-14 19:00:14.830889: done, saving took 0.87 seconds +2022-05-14 19:00:14.846895: done +2022-05-14 19:00:36.180036: This epoch took 483.809512 s + +2022-05-14 19:00:36.180383: +epoch: 109 +2022-05-14 19:07:52.304256: train loss : 0.0441 +2022-05-14 19:08:17.331636: validation loss: 0.0671 +2022-05-14 19:08:17.332369: Average global foreground Dice: [0.9482, 0.9298, 0.9178] +2022-05-14 19:08:17.332457: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 19:08:17.745139: lr: 0.000189 +2022-05-14 19:08:17.745324: saving scheduled checkpoint file... +2022-05-14 19:08:17.794653: saving checkpoint... +2022-05-14 19:08:18.604978: done, saving took 0.86 seconds +2022-05-14 19:08:18.623781: done +2022-05-14 19:08:37.711333: This epoch took 481.530845 s + +2022-05-14 19:08:37.711652: +epoch: 110 +2022-05-14 19:15:53.524433: train loss : 0.0445 +2022-05-14 19:16:18.571456: validation loss: 0.0551 +2022-05-14 19:16:18.572128: Average global foreground Dice: [0.9501, 0.9331, 0.9207] +2022-05-14 19:16:18.572212: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 19:16:18.981318: lr: 0.000187 +2022-05-14 19:16:18.981522: saving scheduled checkpoint file... +2022-05-14 19:16:19.031974: saving checkpoint... +2022-05-14 19:16:19.849809: done, saving took 0.87 seconds +2022-05-14 19:16:19.866214: done +2022-05-14 19:16:37.307584: This epoch took 479.595820 s + +2022-05-14 19:16:37.307869: +epoch: 111 +2022-05-14 19:23:53.234654: train loss : 0.0429 +2022-05-14 19:24:18.323093: validation loss: 0.0446 +2022-05-14 19:24:18.323758: Average global foreground Dice: [0.9498, 0.936, 0.9177] +2022-05-14 19:24:18.323843: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 19:24:18.733184: lr: 0.000185 +2022-05-14 19:24:18.733380: saving scheduled checkpoint file... +2022-05-14 19:24:18.782389: saving checkpoint... +2022-05-14 19:24:19.605344: done, saving took 0.87 seconds +2022-05-14 19:24:19.623394: done +2022-05-14 19:24:41.769691: This epoch took 484.461725 s + +2022-05-14 19:24:41.769962: +epoch: 112 +2022-05-14 19:31:57.692191: train loss : 0.0435 +2022-05-14 19:32:22.705720: validation loss: 0.0495 +2022-05-14 19:32:22.706371: Average global foreground Dice: [0.9507, 0.9365, 0.9219] +2022-05-14 19:32:22.706459: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 19:32:23.124180: lr: 0.000183 +2022-05-14 19:32:23.124372: saving scheduled checkpoint file... +2022-05-14 19:32:23.173693: saving checkpoint... +2022-05-14 19:32:23.993891: done, saving took 0.87 seconds +2022-05-14 19:32:24.010210: done +2022-05-14 19:32:39.563325: This epoch took 477.793261 s + +2022-05-14 19:32:39.563614: +epoch: 113 +2022-05-14 19:39:55.605612: train loss : 0.0406 +2022-05-14 19:40:20.630401: validation loss: 0.0461 +2022-05-14 19:40:20.631112: Average global foreground Dice: [0.9503, 0.9396, 0.9237] +2022-05-14 19:40:20.631198: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 19:40:21.040611: lr: 0.000181 +2022-05-14 19:40:21.040805: saving scheduled checkpoint file... +2022-05-14 19:40:21.090319: saving checkpoint... +2022-05-14 19:40:21.910907: done, saving took 0.87 seconds +2022-05-14 19:40:21.927274: done +2022-05-14 19:40:36.585591: This epoch took 477.021846 s + +2022-05-14 19:40:36.585860: +epoch: 114 +2022-05-14 19:47:52.139545: train loss : 0.0426 +2022-05-14 19:48:17.192298: validation loss: 0.0520 +2022-05-14 19:48:17.193003: Average global foreground Dice: [0.9497, 0.9311, 0.9238] +2022-05-14 19:48:17.193092: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 19:48:17.611896: lr: 0.000179 +2022-05-14 19:48:17.612088: saving scheduled checkpoint file... +2022-05-14 19:48:17.661820: saving checkpoint... +2022-05-14 19:48:18.492146: done, saving took 0.88 seconds +2022-05-14 19:48:18.502486: done +2022-05-14 19:48:35.765006: This epoch took 479.179048 s + +2022-05-14 19:48:35.765269: +epoch: 115 +2022-05-14 19:55:51.421577: train loss : 0.0415 +2022-05-14 19:56:16.414469: validation loss: 0.0506 +2022-05-14 19:56:16.415102: Average global foreground Dice: [0.9504, 0.9336, 0.9192] +2022-05-14 19:56:16.415185: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 19:56:16.824783: lr: 0.000177 +2022-05-14 19:56:16.824977: saving scheduled checkpoint file... +2022-05-14 19:56:16.877398: saving checkpoint... +2022-05-14 19:56:17.709612: done, saving took 0.88 seconds +2022-05-14 19:56:17.726066: done +2022-05-14 19:56:34.300163: This epoch took 478.534795 s + +2022-05-14 19:56:34.300452: +epoch: 116 +2022-05-14 20:03:49.454816: train loss : 0.0430 +2022-05-14 20:04:14.547169: validation loss: 0.0639 +2022-05-14 20:04:14.547919: Average global foreground Dice: [0.9513, 0.9379, 0.9194] +2022-05-14 20:04:14.548007: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 20:04:14.956116: lr: 0.000175 +2022-05-14 20:04:14.956308: saving scheduled checkpoint file... +2022-05-14 20:04:15.008423: saving checkpoint... +2022-05-14 20:04:15.829288: done, saving took 0.87 seconds +2022-05-14 20:04:15.847807: done +2022-05-14 20:04:35.847128: This epoch took 481.546578 s + +2022-05-14 20:04:35.847398: +epoch: 117 +2022-05-14 20:11:50.879069: train loss : 0.0399 +2022-05-14 20:12:15.913433: validation loss: 0.0541 +2022-05-14 20:12:15.914263: Average global foreground Dice: [0.9503, 0.9285, 0.9215] +2022-05-14 20:12:15.914349: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 20:12:16.328545: lr: 0.000173 +2022-05-14 20:12:16.328740: saving scheduled checkpoint file... +2022-05-14 20:12:16.376780: saving checkpoint... +2022-05-14 20:12:17.191322: done, saving took 0.86 seconds +2022-05-14 20:12:17.210304: done +2022-05-14 20:12:33.283708: This epoch took 477.436198 s + +2022-05-14 20:12:33.284017: +epoch: 118 +2022-05-14 20:19:48.457724: train loss : 0.0435 +2022-05-14 20:20:13.474880: validation loss: 0.0500 +2022-05-14 20:20:13.475577: Average global foreground Dice: [0.947, 0.9303, 0.9138] +2022-05-14 20:20:13.475663: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 20:20:13.885219: lr: 0.000172 +2022-05-14 20:20:13.885421: saving scheduled checkpoint file... +2022-05-14 20:20:13.936374: saving checkpoint... +2022-05-14 20:20:14.750916: done, saving took 0.87 seconds +2022-05-14 20:20:14.768830: done +2022-05-14 20:20:29.868506: This epoch took 476.584390 s + +2022-05-14 20:20:29.868797: +epoch: 119 +2022-05-14 20:27:45.401428: train loss : 0.0431 +2022-05-14 20:28:10.416433: validation loss: 0.0623 +2022-05-14 20:28:10.417070: Average global foreground Dice: [0.9473, 0.9323, 0.9117] +2022-05-14 20:28:10.417153: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 20:28:10.827536: lr: 0.00017 +2022-05-14 20:28:10.827723: saving scheduled checkpoint file... +2022-05-14 20:28:10.875145: saving checkpoint... +2022-05-14 20:28:11.688701: done, saving took 0.86 seconds +2022-05-14 20:28:11.706891: done +2022-05-14 20:28:28.650895: This epoch took 478.781999 s + +2022-05-14 20:28:28.651168: +epoch: 120 +2022-05-14 20:35:44.206888: train loss : 0.0411 +2022-05-14 20:36:09.267824: validation loss: 0.0435 +2022-05-14 20:36:09.268452: Average global foreground Dice: [0.9472, 0.9367, 0.9174] +2022-05-14 20:36:09.268538: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 20:36:09.679993: lr: 0.000168 +2022-05-14 20:36:09.680188: saving scheduled checkpoint file... +2022-05-14 20:36:09.729512: saving checkpoint... +2022-05-14 20:36:10.549151: done, saving took 0.87 seconds +2022-05-14 20:36:10.567527: done +2022-05-14 20:36:28.557804: This epoch took 479.906539 s + +2022-05-14 20:36:28.558076: +epoch: 121 +2022-05-14 20:43:43.931593: train loss : 0.0411 +2022-05-14 20:44:08.975190: validation loss: 0.0620 +2022-05-14 20:44:08.975921: Average global foreground Dice: [0.949, 0.9274, 0.9085] +2022-05-14 20:44:08.976007: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 20:44:09.387599: lr: 0.000166 +2022-05-14 20:44:09.387799: saving scheduled checkpoint file... +2022-05-14 20:44:09.438509: saving checkpoint... +2022-05-14 20:44:10.259205: done, saving took 0.87 seconds +2022-05-14 20:44:10.278582: done +2022-05-14 20:44:24.825361: This epoch took 476.267184 s + +2022-05-14 20:44:24.825692: +epoch: 122 +2022-05-14 20:51:40.925524: train loss : 0.0408 +2022-05-14 20:52:05.995085: validation loss: 0.0598 +2022-05-14 20:52:05.995809: Average global foreground Dice: [0.9486, 0.9361, 0.9186] +2022-05-14 20:52:05.995915: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 20:52:06.461100: lr: 0.000164 +2022-05-14 20:52:06.461341: saving scheduled checkpoint file... +2022-05-14 20:52:06.513504: saving checkpoint... +2022-05-14 20:52:07.309757: done, saving took 0.85 seconds +2022-05-14 20:52:07.327708: done +2022-05-14 20:52:24.096898: This epoch took 479.271101 s + +2022-05-14 20:52:24.097197: +epoch: 123 +2022-05-14 20:59:39.866854: train loss : 0.0406 +2022-05-14 21:00:04.842361: validation loss: 0.0529 +2022-05-14 21:00:04.843045: Average global foreground Dice: [0.9506, 0.9391, 0.9209] +2022-05-14 21:00:04.843130: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 21:00:05.253087: lr: 0.000162 +2022-05-14 21:00:05.253285: saving scheduled checkpoint file... +2022-05-14 21:00:05.304387: saving checkpoint... +2022-05-14 21:00:06.120566: done, saving took 0.87 seconds +2022-05-14 21:00:06.137008: done +2022-05-14 21:00:24.675059: This epoch took 480.577762 s + +2022-05-14 21:00:24.675318: +epoch: 124 +2022-05-14 21:07:40.726358: train loss : 0.0404 +2022-05-14 21:08:05.838225: validation loss: 0.0480 +2022-05-14 21:08:05.838866: Average global foreground Dice: [0.9499, 0.9321, 0.9206] +2022-05-14 21:08:05.839216: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 21:08:06.249350: lr: 0.00016 +2022-05-14 21:08:06.249543: saving scheduled checkpoint file... +2022-05-14 21:08:06.297801: saving checkpoint... +2022-05-14 21:08:07.060553: done, saving took 0.81 seconds +2022-05-14 21:08:07.071076: done +2022-05-14 21:08:23.345114: This epoch took 478.669673 s + +2022-05-14 21:08:23.345460: +epoch: 125 +2022-05-14 21:15:38.537543: train loss : 0.0438 +2022-05-14 21:16:03.573159: validation loss: 0.0514 +2022-05-14 21:16:03.573816: Average global foreground Dice: [0.9483, 0.9336, 0.9163] +2022-05-14 21:16:03.573902: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 21:16:03.983755: lr: 0.000158 +2022-05-14 21:16:03.983933: saving scheduled checkpoint file... +2022-05-14 21:16:04.032465: saving checkpoint... +2022-05-14 21:16:04.754893: done, saving took 0.77 seconds +2022-05-14 21:16:04.769866: done +2022-05-14 21:16:20.180006: This epoch took 476.834441 s + +2022-05-14 21:16:20.180279: +epoch: 126 +2022-05-14 21:23:35.720111: train loss : 0.0421 +2022-05-14 21:24:00.832054: validation loss: 0.0552 +2022-05-14 21:24:00.832696: Average global foreground Dice: [0.9508, 0.9283, 0.9147] +2022-05-14 21:24:00.832783: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 21:24:01.243519: lr: 0.000156 +2022-05-14 21:24:01.243711: saving scheduled checkpoint file... +2022-05-14 21:24:01.294032: saving checkpoint... +2022-05-14 21:24:02.089983: done, saving took 0.85 seconds +2022-05-14 21:24:02.107220: done +2022-05-14 21:24:17.525762: This epoch took 477.345377 s + +2022-05-14 21:24:17.526044: +epoch: 127 +2022-05-14 21:31:33.415925: train loss : 0.0468 +2022-05-14 21:31:58.459666: validation loss: 0.0482 +2022-05-14 21:31:58.460353: Average global foreground Dice: [0.9514, 0.9275, 0.916] +2022-05-14 21:31:58.460436: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 21:31:58.871534: lr: 0.000154 +2022-05-14 21:31:58.871728: saving scheduled checkpoint file... +2022-05-14 21:31:58.920630: saving checkpoint... +2022-05-14 21:31:59.733725: done, saving took 0.86 seconds +2022-05-14 21:31:59.751775: done +2022-05-14 21:32:27.391341: This epoch took 489.865191 s + +2022-05-14 21:32:27.391632: +epoch: 128 +2022-05-14 21:39:43.009738: train loss : 0.0436 +2022-05-14 21:40:08.104578: validation loss: 0.0667 +2022-05-14 21:40:08.105254: Average global foreground Dice: [0.9525, 0.93, 0.9149] +2022-05-14 21:40:08.105340: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 21:40:08.517432: lr: 0.000152 +2022-05-14 21:40:08.517638: saving scheduled checkpoint file... +2022-05-14 21:40:08.568119: saving checkpoint... +2022-05-14 21:40:09.379067: done, saving took 0.86 seconds +2022-05-14 21:40:09.396966: done +2022-05-14 21:40:24.425985: This epoch took 477.034230 s + +2022-05-14 21:40:24.426346: +epoch: 129 +2022-05-14 21:47:40.390802: train loss : 0.0414 +2022-05-14 21:48:05.376065: validation loss: 0.0400 +2022-05-14 21:48:05.376692: Average global foreground Dice: [0.9514, 0.9392, 0.9226] +2022-05-14 21:48:05.376779: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 21:48:05.810328: lr: 0.00015 +2022-05-14 21:48:05.810515: saving scheduled checkpoint file... +2022-05-14 21:48:05.858420: saving checkpoint... +2022-05-14 21:48:06.671726: done, saving took 0.86 seconds +2022-05-14 21:48:06.689798: done +2022-05-14 21:48:21.384067: This epoch took 476.957620 s + +2022-05-14 21:48:21.384352: +epoch: 130 +2022-05-14 21:55:37.098601: train loss : 0.0399 +2022-05-14 21:56:02.139446: validation loss: 0.0693 +2022-05-14 21:56:02.140267: Average global foreground Dice: [0.9515, 0.9317, 0.9214] +2022-05-14 21:56:02.140351: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 21:56:02.551702: lr: 0.000148 +2022-05-14 21:56:02.551890: saving scheduled checkpoint file... +2022-05-14 21:56:02.602389: saving checkpoint... +2022-05-14 21:56:03.423317: done, saving took 0.87 seconds +2022-05-14 21:56:03.441607: done +2022-05-14 21:56:19.274513: This epoch took 477.890048 s + +2022-05-14 21:56:19.274823: +epoch: 131 +2022-05-14 22:03:35.134165: train loss : 0.0415 +2022-05-14 22:04:00.207112: validation loss: 0.0538 +2022-05-14 22:04:00.207750: Average global foreground Dice: [0.9507, 0.9356, 0.9204] +2022-05-14 22:04:00.207837: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 22:04:00.620039: lr: 0.000146 +2022-05-14 22:04:00.620228: saving scheduled checkpoint file... +2022-05-14 22:04:00.668853: saving checkpoint... +2022-05-14 22:04:01.491493: done, saving took 0.87 seconds +2022-05-14 22:04:01.503629: done +2022-05-14 22:04:16.620725: This epoch took 477.345794 s + +2022-05-14 22:04:16.620985: +epoch: 132 +2022-05-14 22:11:32.658190: train loss : 0.0410 +2022-05-14 22:11:57.705115: validation loss: 0.0558 +2022-05-14 22:11:57.705743: Average global foreground Dice: [0.9491, 0.9359, 0.92] +2022-05-14 22:11:57.705827: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 22:11:58.118117: lr: 0.000144 +2022-05-14 22:11:58.118317: saving scheduled checkpoint file... +2022-05-14 22:11:58.167390: saving checkpoint... +2022-05-14 22:11:58.991491: done, saving took 0.87 seconds +2022-05-14 22:11:59.009962: done +2022-05-14 22:12:15.370795: This epoch took 478.749703 s + +2022-05-14 22:12:15.371081: +epoch: 133 +2022-05-14 22:19:31.091536: train loss : 0.0425 +2022-05-14 22:19:56.064642: validation loss: 0.0592 +2022-05-14 22:19:56.065320: Average global foreground Dice: [0.9507, 0.9234, 0.9128] +2022-05-14 22:19:56.065404: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 22:19:56.475007: lr: 0.000142 +2022-05-14 22:19:56.475202: saving scheduled checkpoint file... +2022-05-14 22:19:56.524897: saving checkpoint... +2022-05-14 22:19:57.349576: done, saving took 0.87 seconds +2022-05-14 22:19:57.367989: done +2022-05-14 22:20:12.539777: This epoch took 477.168591 s + +2022-05-14 22:20:12.540080: +epoch: 134 +2022-05-14 22:27:28.730876: train loss : 0.0404 +2022-05-14 22:27:53.783569: validation loss: 0.0485 +2022-05-14 22:27:53.784221: Average global foreground Dice: [0.951, 0.9327, 0.911] +2022-05-14 22:27:53.784306: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 22:27:54.201614: lr: 0.00014 +2022-05-14 22:27:54.201815: saving scheduled checkpoint file... +2022-05-14 22:27:54.252419: saving checkpoint... +2022-05-14 22:27:55.069323: done, saving took 0.87 seconds +2022-05-14 22:27:55.085660: done +2022-05-14 22:28:17.067241: This epoch took 484.527055 s + +2022-05-14 22:28:17.067533: +epoch: 135 +2022-05-14 22:35:32.940539: train loss : 0.0408 +2022-05-14 22:35:57.988751: validation loss: 0.0551 +2022-05-14 22:35:57.989418: Average global foreground Dice: [0.9483, 0.9335, 0.9141] +2022-05-14 22:35:57.989502: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 22:35:58.403823: lr: 0.000138 +2022-05-14 22:35:58.404083: saving scheduled checkpoint file... +2022-05-14 22:35:58.453596: saving checkpoint... +2022-05-14 22:35:59.268349: done, saving took 0.86 seconds +2022-05-14 22:35:59.291035: done +2022-05-14 22:36:17.094656: This epoch took 480.027004 s + +2022-05-14 22:36:17.094926: +epoch: 136 +2022-05-14 22:43:32.498487: train loss : 0.0405 +2022-05-14 22:43:57.601321: validation loss: 0.0500 +2022-05-14 22:43:57.602047: Average global foreground Dice: [0.9539, 0.9336, 0.9228] +2022-05-14 22:43:57.602131: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 22:43:58.015252: lr: 0.000136 +2022-05-14 22:43:58.015440: saving scheduled checkpoint file... +2022-05-14 22:43:58.065284: saving checkpoint... +2022-05-14 22:43:58.888773: done, saving took 0.87 seconds +2022-05-14 22:43:58.917371: done +2022-05-14 22:44:15.975620: This epoch took 478.880568 s + +2022-05-14 22:44:15.976002: +epoch: 137 +2022-05-14 22:51:31.672894: train loss : 0.0433 +2022-05-14 22:51:56.700849: validation loss: 0.0577 +2022-05-14 22:51:56.701595: Average global foreground Dice: [0.947, 0.9313, 0.9157] +2022-05-14 22:51:56.701682: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 22:51:57.117867: lr: 0.000134 +2022-05-14 22:51:57.118074: saving scheduled checkpoint file... +2022-05-14 22:51:57.166855: saving checkpoint... +2022-05-14 22:51:57.988350: done, saving took 0.87 seconds +2022-05-14 22:51:58.000238: done +2022-05-14 22:52:15.164885: This epoch took 479.188781 s + +2022-05-14 22:52:15.165173: +epoch: 138 +2022-05-14 22:59:30.987312: train loss : 0.0408 +2022-05-14 22:59:56.032063: validation loss: 0.0465 +2022-05-14 22:59:56.032749: Average global foreground Dice: [0.9497, 0.9288, 0.9155] +2022-05-14 22:59:56.032835: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 22:59:56.446638: lr: 0.000132 +2022-05-14 22:59:56.446840: saving scheduled checkpoint file... +2022-05-14 22:59:56.496910: saving checkpoint... +2022-05-14 22:59:57.313587: done, saving took 0.87 seconds +2022-05-14 22:59:57.332263: done +2022-05-14 23:00:15.970900: This epoch took 480.805614 s + +2022-05-14 23:00:15.971185: +epoch: 139 +2022-05-14 23:07:31.058923: train loss : 0.0405 +2022-05-14 23:07:56.053977: validation loss: 0.0646 +2022-05-14 23:07:56.054606: Average global foreground Dice: [0.952, 0.9279, 0.9154] +2022-05-14 23:07:56.054691: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 23:07:56.466975: lr: 0.00013 +2022-05-14 23:07:56.467173: saving scheduled checkpoint file... +2022-05-14 23:07:56.516445: saving checkpoint... +2022-05-14 23:07:57.337172: done, saving took 0.87 seconds +2022-05-14 23:07:57.355579: done +2022-05-14 23:08:14.606548: This epoch took 478.635243 s + +2022-05-14 23:08:14.606927: +epoch: 140 +2022-05-14 23:15:30.092601: train loss : 0.0431 +2022-05-14 23:15:55.137868: validation loss: 0.0460 +2022-05-14 23:15:55.138498: Average global foreground Dice: [0.9512, 0.9323, 0.9194] +2022-05-14 23:15:55.138593: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 23:15:55.550508: lr: 0.000128 +2022-05-14 23:15:55.550706: saving scheduled checkpoint file... +2022-05-14 23:15:55.601713: saving checkpoint... +2022-05-14 23:15:56.423105: done, saving took 0.87 seconds +2022-05-14 23:15:56.441232: done +2022-05-14 23:16:12.389713: This epoch took 477.782655 s + +2022-05-14 23:16:12.390075: +epoch: 141 +2022-05-14 23:23:28.272308: train loss : 0.0384 +2022-05-14 23:23:53.297197: validation loss: 0.0478 +2022-05-14 23:23:53.297878: Average global foreground Dice: [0.9537, 0.9354, 0.9232] +2022-05-14 23:23:53.297966: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 23:23:53.713041: lr: 0.000127 +2022-05-14 23:23:53.713260: saving scheduled checkpoint file... +2022-05-14 23:23:53.760820: saving checkpoint... +2022-05-14 23:23:54.591698: done, saving took 0.88 seconds +2022-05-14 23:23:54.607752: done +2022-05-14 23:24:27.821045: This epoch took 495.430854 s + +2022-05-14 23:24:27.821409: +epoch: 142 +2022-05-14 23:31:41.904191: train loss : 0.0396 +2022-05-14 23:32:06.956891: validation loss: 0.0677 +2022-05-14 23:32:06.957571: Average global foreground Dice: [0.949, 0.929, 0.9112] +2022-05-14 23:32:06.957661: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 23:32:07.370915: lr: 0.000125 +2022-05-14 23:32:07.371182: saving scheduled checkpoint file... +2022-05-14 23:32:07.422642: saving checkpoint... +2022-05-14 23:32:08.240165: done, saving took 0.87 seconds +2022-05-14 23:32:08.258739: done +2022-05-14 23:32:24.657328: This epoch took 476.835815 s + +2022-05-14 23:32:24.657695: +epoch: 143 +2022-05-14 23:39:40.460490: train loss : 0.0394 +2022-05-14 23:40:05.498808: validation loss: 0.0621 +2022-05-14 23:40:05.499443: Average global foreground Dice: [0.9506, 0.9422, 0.9208] +2022-05-14 23:40:05.499537: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 23:40:05.912725: lr: 0.000123 +2022-05-14 23:40:05.912921: saving scheduled checkpoint file... +2022-05-14 23:40:05.960846: saving checkpoint... +2022-05-14 23:40:06.690953: done, saving took 0.78 seconds +2022-05-14 23:40:06.707373: done +2022-05-14 23:40:23.726279: This epoch took 479.068476 s + +2022-05-14 23:40:23.726588: +epoch: 144 +2022-05-14 23:47:39.417143: train loss : 0.0403 +2022-05-14 23:48:04.532329: validation loss: 0.0450 +2022-05-14 23:48:04.533060: Average global foreground Dice: [0.9531, 0.9327, 0.9225] +2022-05-14 23:48:04.533160: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 23:48:04.945743: lr: 0.000121 +2022-05-14 23:48:04.945923: saving scheduled checkpoint file... +2022-05-14 23:48:04.994432: saving checkpoint... +2022-05-14 23:48:05.810160: done, saving took 0.86 seconds +2022-05-14 23:48:05.826462: done +2022-05-14 23:48:24.726038: This epoch took 480.999347 s + +2022-05-14 23:48:24.726323: +epoch: 145 +2022-05-14 23:55:39.683070: train loss : 0.0400 +2022-05-14 23:56:04.725103: validation loss: 0.0551 +2022-05-14 23:56:04.725785: Average global foreground Dice: [0.9503, 0.9212, 0.9171] +2022-05-14 23:56:04.725874: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 23:56:05.140446: lr: 0.000119 +2022-05-14 23:56:05.140656: saving scheduled checkpoint file... +2022-05-14 23:56:05.191532: saving checkpoint... +2022-05-14 23:56:06.014697: done, saving took 0.87 seconds +2022-05-14 23:56:06.032942: done +2022-05-14 23:56:23.049318: This epoch took 478.322891 s + +2022-05-14 23:56:23.049681: +epoch: 146 +2022-05-15 00:03:39.050681: train loss : 0.0377 +2022-05-15 00:04:04.072880: validation loss: 0.0724 +2022-05-15 00:04:04.073508: Average global foreground Dice: [0.9488, 0.9387, 0.9118] +2022-05-15 00:04:04.073605: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 00:04:04.485408: lr: 0.000117 +2022-05-15 00:04:04.485607: saving scheduled checkpoint file... +2022-05-15 00:04:04.534056: saving checkpoint... +2022-05-15 00:04:05.364410: done, saving took 0.88 seconds +2022-05-15 00:04:05.375669: done +2022-05-15 00:04:27.633680: This epoch took 484.583899 s + +2022-05-15 00:04:27.633956: +epoch: 147 +2022-05-15 00:11:42.778036: train loss : 0.0410 +2022-05-15 00:12:07.840409: validation loss: 0.0495 +2022-05-15 00:12:07.841092: Average global foreground Dice: [0.9499, 0.9344, 0.9199] +2022-05-15 00:12:07.841176: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 00:12:08.255526: lr: 0.000115 +2022-05-15 00:12:08.255769: saving scheduled checkpoint file... +2022-05-15 00:12:08.309052: saving checkpoint... +2022-05-15 00:12:09.136985: done, saving took 0.88 seconds +2022-05-15 00:12:09.153268: done +2022-05-15 00:12:54.161553: This epoch took 506.527491 s + +2022-05-15 00:12:54.161844: +epoch: 148 +2022-05-15 00:20:09.396103: train loss : 0.0382 +2022-05-15 00:20:34.429867: validation loss: 0.0611 +2022-05-15 00:20:34.430568: Average global foreground Dice: [0.952, 0.9294, 0.9205] +2022-05-15 00:20:34.430654: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 00:20:34.847051: lr: 0.000113 +2022-05-15 00:20:34.847251: saving scheduled checkpoint file... +2022-05-15 00:20:34.898664: saving checkpoint... +2022-05-15 00:20:35.713648: done, saving took 0.87 seconds +2022-05-15 00:20:35.731963: done +2022-05-15 00:21:15.403298: This epoch took 501.241334 s + +2022-05-15 00:21:15.403665: +epoch: 149 +2022-05-15 00:28:31.040470: train loss : 0.0389 +2022-05-15 00:28:56.099295: validation loss: 0.0550 +2022-05-15 00:28:56.099973: Average global foreground Dice: [0.9498, 0.9413, 0.919] +2022-05-15 00:28:56.100058: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 00:28:56.513799: lr: 0.000111 +2022-05-15 00:28:56.514000: saving scheduled checkpoint file... +2022-05-15 00:28:56.562563: saving checkpoint... +2022-05-15 00:28:57.383104: done, saving took 0.87 seconds +2022-05-15 00:28:57.395140: done +2022-05-15 00:29:21.187934: This epoch took 485.784036 s + +2022-05-15 00:29:21.188208: +epoch: 150 +2022-05-15 00:36:36.777032: train loss : 0.0388 +2022-05-15 00:37:01.831464: validation loss: 0.0498 +2022-05-15 00:37:01.832112: Average global foreground Dice: [0.9496, 0.9361, 0.9195] +2022-05-15 00:37:01.832196: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 00:37:02.247962: lr: 0.000109 +2022-05-15 00:37:02.248150: saving scheduled checkpoint file... +2022-05-15 00:37:02.297897: saving checkpoint... +2022-05-15 00:37:03.114783: done, saving took 0.87 seconds +2022-05-15 00:37:03.133054: done +2022-05-15 00:37:21.307338: This epoch took 480.119028 s + +2022-05-15 00:37:21.307644: +epoch: 151 +2022-05-15 00:44:36.668218: train loss : 0.0389 +2022-05-15 00:45:01.703328: validation loss: 0.0693 +2022-05-15 00:45:01.704073: Average global foreground Dice: [0.9522, 0.9328, 0.911] +2022-05-15 00:45:01.704161: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 00:45:02.118829: lr: 0.000107 +2022-05-15 00:45:02.119035: saving scheduled checkpoint file... +2022-05-15 00:45:02.168531: saving checkpoint... +2022-05-15 00:45:02.992539: done, saving took 0.87 seconds +2022-05-15 00:45:03.010931: done +2022-05-15 00:45:39.083687: This epoch took 497.775939 s + +2022-05-15 00:45:39.083966: +epoch: 152 +2022-05-15 00:52:54.323138: train loss : 0.0380 +2022-05-15 00:53:19.388855: validation loss: 0.0547 +2022-05-15 00:53:19.389523: Average global foreground Dice: [0.9506, 0.9346, 0.9207] +2022-05-15 00:53:19.389619: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 00:53:19.801832: lr: 0.000106 +2022-05-15 00:53:19.802028: saving scheduled checkpoint file... +2022-05-15 00:53:19.854043: saving checkpoint... +2022-05-15 00:53:20.637480: done, saving took 0.84 seconds +2022-05-15 00:53:20.652686: done +2022-05-15 00:53:36.902531: This epoch took 477.818463 s + +2022-05-15 00:53:36.902837: +epoch: 153 +2022-05-15 01:00:52.706656: train loss : 0.0388 +2022-05-15 01:01:17.773777: validation loss: 0.0443 +2022-05-15 01:01:17.774438: Average global foreground Dice: [0.9512, 0.9208, 0.9129] +2022-05-15 01:01:17.774524: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 01:01:18.209332: lr: 0.000104 +2022-05-15 01:01:18.209528: saving scheduled checkpoint file... +2022-05-15 01:01:18.257447: saving checkpoint... +2022-05-15 01:01:19.012887: done, saving took 0.80 seconds +2022-05-15 01:01:19.022960: done +2022-05-15 01:01:35.792288: This epoch took 478.889349 s + +2022-05-15 01:01:35.792549: +epoch: 154 +2022-05-15 01:08:52.007039: train loss : 0.0408 +2022-05-15 01:09:17.029296: validation loss: 0.0623 +2022-05-15 01:09:17.030061: Average global foreground Dice: [0.9445, 0.9348, 0.92] +2022-05-15 01:09:17.030149: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 01:09:17.507404: lr: 0.000102 +2022-05-15 01:09:17.507622: saving scheduled checkpoint file... +2022-05-15 01:09:17.561703: saving checkpoint... +2022-05-15 01:09:18.373613: done, saving took 0.87 seconds +2022-05-15 01:09:18.389496: done +2022-05-15 01:09:34.450860: This epoch took 478.658190 s + +2022-05-15 01:09:34.451147: +epoch: 155 +2022-05-15 01:16:50.158170: train loss : 0.0380 +2022-05-15 01:17:15.204476: validation loss: 0.0568 +2022-05-15 01:17:15.205261: Average global foreground Dice: [0.95, 0.9367, 0.9212] +2022-05-15 01:17:15.205355: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 01:17:15.619689: lr: 0.0001 +2022-05-15 01:17:15.619881: saving scheduled checkpoint file... +2022-05-15 01:17:15.671925: saving checkpoint... +2022-05-15 01:17:16.423383: done, saving took 0.80 seconds +2022-05-15 01:17:16.442847: done +2022-05-15 01:17:37.665353: This epoch took 483.214107 s + +2022-05-15 01:17:37.665704: +epoch: 156 +2022-05-15 01:24:53.838171: train loss : 0.0395 +2022-05-15 01:25:18.954355: validation loss: 0.0573 +2022-05-15 01:25:18.954982: Average global foreground Dice: [0.9513, 0.931, 0.9187] +2022-05-15 01:25:18.955066: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 01:25:19.374245: lr: 9.8e-05 +2022-05-15 01:25:19.374450: saving scheduled checkpoint file... +2022-05-15 01:25:19.423053: saving checkpoint... +2022-05-15 01:25:20.151260: done, saving took 0.78 seconds +2022-05-15 01:25:20.166553: done +2022-05-15 01:25:43.755589: This epoch took 486.089740 s + +2022-05-15 01:25:43.755889: +epoch: 157 +2022-05-15 01:32:59.318246: train loss : 0.0374 +2022-05-15 01:33:24.347988: validation loss: 0.0514 +2022-05-15 01:33:24.348660: Average global foreground Dice: [0.9493, 0.9301, 0.9212] +2022-05-15 01:33:24.348747: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 01:33:24.765629: lr: 9.6e-05 +2022-05-15 01:33:24.765832: saving scheduled checkpoint file... +2022-05-15 01:33:24.816732: saving checkpoint... +2022-05-15 01:33:25.572195: done, saving took 0.81 seconds +2022-05-15 01:33:25.588090: done +2022-05-15 01:33:42.827660: This epoch took 479.071673 s + +2022-05-15 01:33:42.827922: +epoch: 158 +2022-05-15 01:40:58.727043: train loss : 0.0404 +2022-05-15 01:41:23.693258: validation loss: 0.0460 +2022-05-15 01:41:23.693902: Average global foreground Dice: [0.9482, 0.9416, 0.9233] +2022-05-15 01:41:23.693988: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 01:41:24.107707: lr: 9.4e-05 +2022-05-15 01:41:24.107901: saving scheduled checkpoint file... +2022-05-15 01:41:24.155891: saving checkpoint... +2022-05-15 01:41:24.915197: done, saving took 0.81 seconds +2022-05-15 01:41:24.943837: done +2022-05-15 01:41:47.672388: This epoch took 484.844367 s + +2022-05-15 01:41:47.672657: +epoch: 159 +2022-05-15 01:49:03.881910: train loss : 0.0382 +2022-05-15 01:49:28.867954: validation loss: 0.0637 +2022-05-15 01:49:28.868576: Average global foreground Dice: [0.9486, 0.932, 0.9237] +2022-05-15 01:49:28.868659: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 01:49:29.286285: lr: 9.3e-05 +2022-05-15 01:49:29.286481: saving scheduled checkpoint file... +2022-05-15 01:49:29.335875: saving checkpoint... +2022-05-15 01:49:30.083329: done, saving took 0.80 seconds +2022-05-15 01:49:30.099526: done +2022-05-15 01:49:52.484022: This epoch took 484.811256 s + +2022-05-15 01:49:52.484353: +epoch: 160 +2022-05-15 01:57:08.326671: train loss : 0.0373 +2022-05-15 01:57:33.423001: validation loss: 0.0514 +2022-05-15 01:57:33.423748: Average global foreground Dice: [0.9511, 0.9295, 0.9167] +2022-05-15 01:57:33.423841: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 01:57:33.844152: lr: 9.1e-05 +2022-05-15 01:57:33.844364: saving scheduled checkpoint file... +2022-05-15 01:57:33.894047: saving checkpoint... +2022-05-15 01:57:34.629315: done, saving took 0.78 seconds +2022-05-15 01:57:34.645684: done +2022-05-15 01:57:53.958036: This epoch took 481.473570 s + +2022-05-15 01:57:53.958356: +epoch: 161 +2022-05-15 02:05:09.399614: train loss : 0.0368 +2022-05-15 02:05:34.527720: validation loss: 0.0498 +2022-05-15 02:05:34.528357: Average global foreground Dice: [0.9504, 0.9365, 0.9204] +2022-05-15 02:05:34.528443: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 02:05:34.943111: lr: 8.9e-05 +2022-05-15 02:05:34.943339: saving scheduled checkpoint file... +2022-05-15 02:05:34.994658: saving checkpoint... +2022-05-15 02:05:35.746909: done, saving took 0.80 seconds +2022-05-15 02:05:35.762295: done +2022-05-15 02:05:55.461693: This epoch took 481.503228 s + +2022-05-15 02:05:55.461966: +epoch: 162 +2022-05-15 02:13:10.473014: train loss : 0.0385 +2022-05-15 02:13:35.458347: validation loss: 0.0471 +2022-05-15 02:13:35.459056: Average global foreground Dice: [0.9503, 0.9349, 0.9174] +2022-05-15 02:13:35.459151: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 02:13:35.874505: lr: 8.7e-05 +2022-05-15 02:13:35.874703: saving scheduled checkpoint file... +2022-05-15 02:13:35.923232: saving checkpoint... +2022-05-15 02:13:36.665362: done, saving took 0.79 seconds +2022-05-15 02:13:36.680711: done +2022-05-15 02:14:12.713371: This epoch took 497.251306 s + +2022-05-15 02:14:12.713669: +epoch: 163 +2022-05-15 02:21:28.099636: train loss : 0.0395 +2022-05-15 02:21:53.102575: validation loss: 0.0477 +2022-05-15 02:21:53.103242: Average global foreground Dice: [0.951, 0.9398, 0.9222] +2022-05-15 02:21:53.103326: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 02:21:53.518845: lr: 8.5e-05 +2022-05-15 02:21:53.519038: saving scheduled checkpoint file... +2022-05-15 02:21:53.570006: saving checkpoint... +2022-05-15 02:21:54.396826: done, saving took 0.88 seconds +2022-05-15 02:21:54.412742: done +2022-05-15 02:22:17.650880: This epoch took 484.937103 s + +2022-05-15 02:22:17.651147: +epoch: 164 +2022-05-15 02:29:33.440082: train loss : 0.0374 +2022-05-15 02:29:58.552204: validation loss: 0.0547 +2022-05-15 02:29:58.552892: Average global foreground Dice: [0.9486, 0.9285, 0.9193] +2022-05-15 02:29:58.553044: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 02:29:58.970226: lr: 8.4e-05 +2022-05-15 02:29:58.970425: saving scheduled checkpoint file... +2022-05-15 02:29:59.018456: saving checkpoint... +2022-05-15 02:29:59.858953: done, saving took 0.89 seconds +2022-05-15 02:29:59.876024: done +2022-05-15 02:30:22.723011: This epoch took 485.071761 s + +2022-05-15 02:30:22.723282: +epoch: 165 +2022-05-15 02:37:38.434513: train loss : 0.0377 +2022-05-15 02:38:03.387650: validation loss: 0.0429 +2022-05-15 02:38:03.388328: Average global foreground Dice: [0.9488, 0.9446, 0.9232] +2022-05-15 02:38:03.388412: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 02:38:03.803872: lr: 8.2e-05 +2022-05-15 02:38:03.804061: saving scheduled checkpoint file... +2022-05-15 02:38:03.855191: saving checkpoint... +2022-05-15 02:38:04.697674: done, saving took 0.89 seconds +2022-05-15 02:38:04.714523: done +2022-05-15 02:38:25.636343: This epoch took 482.912959 s + +2022-05-15 02:38:25.636659: +epoch: 166 +2022-05-15 02:45:41.146116: train loss : 0.0391 +2022-05-15 02:46:06.137969: validation loss: 0.0613 +2022-05-15 02:46:06.138639: Average global foreground Dice: [0.9488, 0.9394, 0.9219] +2022-05-15 02:46:06.138727: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 02:46:06.549354: lr: 8e-05 +2022-05-15 02:46:06.549571: saving scheduled checkpoint file... +2022-05-15 02:46:06.598142: saving checkpoint... +2022-05-15 02:46:07.407896: done, saving took 0.86 seconds +2022-05-15 02:46:07.417747: done +2022-05-15 02:46:23.362456: This epoch took 477.725693 s + +2022-05-15 02:46:23.362758: +epoch: 167 +2022-05-15 02:53:39.272552: train loss : 0.0383 +2022-05-15 02:54:04.289729: validation loss: 0.0562 +2022-05-15 02:54:04.290459: Average global foreground Dice: [0.953, 0.9356, 0.9211] +2022-05-15 02:54:04.290546: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 02:54:04.708077: lr: 7.8e-05 +2022-05-15 02:54:04.708268: saving scheduled checkpoint file... +2022-05-15 02:54:04.761525: saving checkpoint... +2022-05-15 02:54:05.585783: done, saving took 0.88 seconds +2022-05-15 02:54:05.617773: done +2022-05-15 02:54:23.805849: This epoch took 480.442995 s + +2022-05-15 02:54:23.806134: +epoch: 168 +2022-05-15 03:01:39.755971: train loss : 0.0388 +2022-05-15 03:02:04.823751: validation loss: 0.0542 +2022-05-15 03:02:04.824395: Average global foreground Dice: [0.9496, 0.9259, 0.9199] +2022-05-15 03:02:04.824480: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 03:02:05.237827: lr: 7.7e-05 +2022-05-15 03:02:05.238014: saving scheduled checkpoint file... +2022-05-15 03:02:05.287018: saving checkpoint... +2022-05-15 03:02:06.055036: done, saving took 0.82 seconds +2022-05-15 03:02:06.070549: done +2022-05-15 03:02:21.375349: This epoch took 477.569116 s + +2022-05-15 03:02:21.375675: +epoch: 169 +2022-05-15 03:09:37.431435: train loss : 0.0385 +2022-05-15 03:10:02.375522: validation loss: 0.0451 +2022-05-15 03:10:02.376176: Average global foreground Dice: [0.9506, 0.9296, 0.9215] +2022-05-15 03:10:02.376264: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 03:10:02.792973: lr: 7.5e-05 +2022-05-15 03:10:02.793180: saving scheduled checkpoint file... +2022-05-15 03:10:02.843971: saving checkpoint... +2022-05-15 03:10:03.669319: done, saving took 0.88 seconds +2022-05-15 03:10:03.687716: done +2022-05-15 03:10:22.005915: This epoch took 480.630120 s + +2022-05-15 03:10:22.006288: +epoch: 170 +2022-05-15 03:17:37.825985: train loss : 0.0393 +2022-05-15 03:18:02.814489: validation loss: 0.0508 +2022-05-15 03:18:02.815096: Average global foreground Dice: [0.9536, 0.9335, 0.9196] +2022-05-15 03:18:02.815184: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 03:18:03.231690: lr: 7.3e-05 +2022-05-15 03:18:03.231879: saving scheduled checkpoint file... +2022-05-15 03:18:03.280195: saving checkpoint... +2022-05-15 03:18:04.098345: done, saving took 0.87 seconds +2022-05-15 03:18:04.110433: done +2022-05-15 03:18:20.654783: This epoch took 478.648391 s + +2022-05-15 03:18:20.655063: +epoch: 171 +2022-05-15 03:25:36.579000: train loss : 0.0400 +2022-05-15 03:26:01.679901: validation loss: 0.0421 +2022-05-15 03:26:01.680520: Average global foreground Dice: [0.9566, 0.9458, 0.9258] +2022-05-15 03:26:01.680610: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 03:26:02.097506: lr: 7.2e-05 +2022-05-15 03:26:02.097707: saving scheduled checkpoint file... +2022-05-15 03:26:02.146366: saving checkpoint... +2022-05-15 03:26:02.967514: done, saving took 0.87 seconds +2022-05-15 03:26:02.983862: done +2022-05-15 03:26:03.016667: saving checkpoint... +2022-05-15 03:26:03.824836: done, saving took 0.84 seconds +2022-05-15 03:26:37.698288: This epoch took 497.043123 s + +2022-05-15 03:26:37.698599: +epoch: 172 +2022-05-15 03:33:53.846192: train loss : 0.0369 +2022-05-15 03:34:18.886384: validation loss: 0.0550 +2022-05-15 03:34:18.887017: Average global foreground Dice: [0.9506, 0.9368, 0.9134] +2022-05-15 03:34:18.887102: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 03:34:19.304644: lr: 7e-05 +2022-05-15 03:34:19.304868: saving scheduled checkpoint file... +2022-05-15 03:34:19.355147: saving checkpoint... +2022-05-15 03:34:20.180160: done, saving took 0.88 seconds +2022-05-15 03:34:20.198457: done +2022-05-15 03:34:38.686212: This epoch took 480.987499 s + +2022-05-15 03:34:38.686518: +epoch: 173 +2022-05-15 03:41:54.515988: train loss : 0.0380 +2022-05-15 03:42:19.556243: validation loss: 0.0500 +2022-05-15 03:42:19.556922: Average global foreground Dice: [0.9533, 0.9295, 0.918] +2022-05-15 03:42:19.557075: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 03:42:19.972229: lr: 6.8e-05 +2022-05-15 03:42:19.972423: saving scheduled checkpoint file... +2022-05-15 03:42:20.022099: saving checkpoint... +2022-05-15 03:42:20.841890: done, saving took 0.87 seconds +2022-05-15 03:42:20.860402: done +2022-05-15 03:42:40.082041: This epoch took 481.395410 s + +2022-05-15 03:42:40.082326: +epoch: 174 +2022-05-15 03:49:55.328724: train loss : 0.0355 +2022-05-15 03:50:20.398109: validation loss: 0.0509 +2022-05-15 03:50:20.398834: Average global foreground Dice: [0.9489, 0.9301, 0.918] +2022-05-15 03:50:20.398927: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 03:50:20.814210: lr: 6.7e-05 +2022-05-15 03:50:20.814432: saving scheduled checkpoint file... +2022-05-15 03:50:20.864581: saving checkpoint... +2022-05-15 03:50:21.684694: done, saving took 0.87 seconds +2022-05-15 03:50:21.702841: done +2022-05-15 03:50:53.035275: This epoch took 492.952849 s + +2022-05-15 03:50:53.035553: +epoch: 175 +2022-05-15 03:58:09.101780: train loss : 0.0369 +2022-05-15 03:58:34.135631: validation loss: 0.0560 +2022-05-15 03:58:34.136352: Average global foreground Dice: [0.9471, 0.9372, 0.9168] +2022-05-15 03:58:34.136452: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 03:58:34.554370: lr: 6.5e-05 +2022-05-15 03:58:34.554566: saving scheduled checkpoint file... +2022-05-15 03:58:34.602270: saving checkpoint... +2022-05-15 03:58:35.427642: done, saving took 0.87 seconds +2022-05-15 03:58:35.443784: done +2022-05-15 03:58:51.819541: This epoch took 478.783855 s + +2022-05-15 03:58:51.819834: +epoch: 176 +2022-05-15 04:06:07.737941: train loss : 0.0384 +2022-05-15 04:06:32.736412: validation loss: 0.0476 +2022-05-15 04:06:32.737022: Average global foreground Dice: [0.9519, 0.9377, 0.9219] +2022-05-15 04:06:32.737107: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 04:06:33.154024: lr: 6.3e-05 +2022-05-15 04:06:33.154220: saving scheduled checkpoint file... +2022-05-15 04:06:33.205420: saving checkpoint... +2022-05-15 04:06:34.029696: done, saving took 0.88 seconds +2022-05-15 04:06:34.047866: done +2022-05-15 04:06:51.587827: This epoch took 479.767895 s + +2022-05-15 04:06:51.588109: +epoch: 177 +2022-05-15 04:14:07.302263: train loss : 0.0372 +2022-05-15 04:14:32.280321: validation loss: 0.0496 +2022-05-15 04:14:32.281006: Average global foreground Dice: [0.9531, 0.9447, 0.9236] +2022-05-15 04:14:32.281090: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 04:14:32.694175: lr: 6.2e-05 +2022-05-15 04:14:32.694585: saving scheduled checkpoint file... +2022-05-15 04:14:32.742632: saving checkpoint... +2022-05-15 04:14:33.473668: done, saving took 0.78 seconds +2022-05-15 04:14:33.484295: done +2022-05-15 04:14:33.513731: saving checkpoint... +2022-05-15 04:14:34.306122: done, saving took 0.82 seconds +2022-05-15 04:14:54.936138: This epoch took 483.347927 s + +2022-05-15 04:14:54.936430: +epoch: 178 +2022-05-15 04:22:10.624983: train loss : 0.0360 +2022-05-15 04:22:35.658273: validation loss: 0.0510 +2022-05-15 04:22:35.658913: Average global foreground Dice: [0.9518, 0.9343, 0.9207] +2022-05-15 04:22:35.659000: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 04:22:36.078749: lr: 6e-05 +2022-05-15 04:22:36.078947: saving scheduled checkpoint file... +2022-05-15 04:22:36.132145: saving checkpoint... +2022-05-15 04:22:36.967224: done, saving took 0.89 seconds +2022-05-15 04:22:36.984358: done +2022-05-15 04:22:37.020976: saving checkpoint... +2022-05-15 04:22:37.840254: done, saving took 0.86 seconds +2022-05-15 04:22:53.433121: This epoch took 478.496588 s + +2022-05-15 04:22:53.433399: +epoch: 179 +2022-05-15 04:30:09.250179: train loss : 0.0373 +2022-05-15 04:30:34.242019: validation loss: 0.0518 +2022-05-15 04:30:34.242677: Average global foreground Dice: [0.9523, 0.9346, 0.9171] +2022-05-15 04:30:34.242761: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 04:30:34.658845: lr: 5.9e-05 +2022-05-15 04:30:34.659048: saving scheduled checkpoint file... +2022-05-15 04:30:34.708297: saving checkpoint... +2022-05-15 04:30:35.525401: done, saving took 0.87 seconds +2022-05-15 04:30:35.543684: done +2022-05-15 04:30:51.352575: This epoch took 477.919067 s + +2022-05-15 04:30:51.352843: +epoch: 180 +2022-05-15 04:38:07.532901: train loss : 0.0361 +2022-05-15 04:38:32.481714: validation loss: 0.0525 +2022-05-15 04:38:32.482358: Average global foreground Dice: [0.9526, 0.9343, 0.9203] +2022-05-15 04:38:32.482443: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 04:38:32.896838: lr: 5.7e-05 +2022-05-15 04:38:32.897043: saving scheduled checkpoint file... +2022-05-15 04:38:32.945365: saving checkpoint... +2022-05-15 04:38:33.652305: done, saving took 0.76 seconds +2022-05-15 04:38:33.661824: done +2022-05-15 04:38:55.515085: This epoch took 484.162140 s + +2022-05-15 04:38:55.515393: +epoch: 181 +2022-05-15 04:46:11.235164: train loss : 0.0392 +2022-05-15 04:46:36.321490: validation loss: 0.0559 +2022-05-15 04:46:36.322163: Average global foreground Dice: [0.9552, 0.9422, 0.9231] +2022-05-15 04:46:36.322249: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 04:46:36.739949: lr: 5.6e-05 +2022-05-15 04:46:36.740142: saving scheduled checkpoint file... +2022-05-15 04:46:36.792974: saving checkpoint... +2022-05-15 04:46:37.611590: done, saving took 0.87 seconds +2022-05-15 04:46:37.629827: done +2022-05-15 04:46:37.671509: saving checkpoint... +2022-05-15 04:46:38.476809: done, saving took 0.85 seconds +2022-05-15 04:46:55.464155: This epoch took 479.948662 s + +2022-05-15 04:46:55.464444: +epoch: 182 +2022-05-15 04:54:10.520013: train loss : 0.0384 +2022-05-15 04:54:35.604431: validation loss: 0.0567 +2022-05-15 04:54:35.605198: Average global foreground Dice: [0.9518, 0.9351, 0.9234] +2022-05-15 04:54:35.605286: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 04:54:36.021354: lr: 5.4e-05 +2022-05-15 04:54:36.021548: saving scheduled checkpoint file... +2022-05-15 04:54:36.070859: saving checkpoint... +2022-05-15 04:54:36.895084: done, saving took 0.87 seconds +2022-05-15 04:54:36.913382: done +2022-05-15 04:54:36.955369: saving checkpoint... +2022-05-15 04:54:37.762508: done, saving took 0.85 seconds +2022-05-15 04:54:58.347055: This epoch took 482.882509 s + +2022-05-15 04:54:58.347328: +epoch: 183 +2022-05-15 05:02:14.222831: train loss : 0.0369 +2022-05-15 05:02:39.218545: validation loss: 0.0569 +2022-05-15 05:02:39.219333: Average global foreground Dice: [0.9495, 0.9353, 0.9196] +2022-05-15 05:02:39.219420: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 05:02:39.635458: lr: 5.3e-05 +2022-05-15 05:02:39.635655: saving scheduled checkpoint file... +2022-05-15 05:02:39.683417: saving checkpoint... +2022-05-15 05:02:40.507697: done, saving took 0.87 seconds +2022-05-15 05:02:40.526749: done +2022-05-15 05:02:56.722974: This epoch took 478.375534 s + +2022-05-15 05:02:56.723266: +epoch: 184 +2022-05-15 05:10:12.005887: train loss : 0.0364 +2022-05-15 05:10:37.114129: validation loss: 0.0474 +2022-05-15 05:10:37.114776: Average global foreground Dice: [0.9546, 0.9367, 0.9178] +2022-05-15 05:10:37.114863: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 05:10:37.528491: lr: 5.1e-05 +2022-05-15 05:10:37.528700: saving scheduled checkpoint file... +2022-05-15 05:10:37.578497: saving checkpoint... +2022-05-15 05:10:38.400522: done, saving took 0.87 seconds +2022-05-15 05:10:38.416741: done +2022-05-15 05:11:02.000175: This epoch took 485.276803 s + +2022-05-15 05:11:02.000459: +epoch: 185 +2022-05-15 05:18:17.977322: train loss : 0.0358 +2022-05-15 05:18:42.991058: validation loss: 0.0560 +2022-05-15 05:18:42.991671: Average global foreground Dice: [0.9532, 0.9427, 0.9279] +2022-05-15 05:18:42.991755: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 05:18:43.405043: lr: 5e-05 +2022-05-15 05:18:43.405244: saving scheduled checkpoint file... +2022-05-15 05:18:43.456057: saving checkpoint... +2022-05-15 05:18:44.279840: done, saving took 0.87 seconds +2022-05-15 05:18:44.303100: done +2022-05-15 05:18:44.347074: saving checkpoint... +2022-05-15 05:18:45.146884: done, saving took 0.84 seconds +2022-05-15 05:19:10.944359: This epoch took 488.943798 s + +2022-05-15 05:19:10.944646: +epoch: 186 +2022-05-15 05:26:26.714237: train loss : 0.0369 +2022-05-15 05:26:51.738877: validation loss: 0.0541 +2022-05-15 05:26:51.739496: Average global foreground Dice: [0.9518, 0.9278, 0.9178] +2022-05-15 05:26:51.739588: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 05:26:52.154382: lr: 4.8e-05 +2022-05-15 05:26:52.154776: saving scheduled checkpoint file... +2022-05-15 05:26:52.203407: saving checkpoint... +2022-05-15 05:26:53.023137: done, saving took 0.87 seconds +2022-05-15 05:26:53.034023: done +2022-05-15 05:27:31.895435: This epoch took 500.950683 s + +2022-05-15 05:27:31.895749: +epoch: 187 +2022-05-15 05:34:47.132200: train loss : 0.0372 +2022-05-15 05:35:12.116455: validation loss: 0.0503 +2022-05-15 05:35:12.117763: Average global foreground Dice: [0.9535, 0.9343, 0.9188] +2022-05-15 05:35:12.117849: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 05:35:12.533251: lr: 4.7e-05 +2022-05-15 05:35:12.533458: saving scheduled checkpoint file... +2022-05-15 05:35:12.582589: saving checkpoint... +2022-05-15 05:35:13.340922: done, saving took 0.81 seconds +2022-05-15 05:35:13.356211: done +2022-05-15 05:35:34.203156: This epoch took 482.307286 s + +2022-05-15 05:35:34.203512: +epoch: 188 +2022-05-15 05:42:50.054816: train loss : 0.0386 +2022-05-15 05:43:15.100820: validation loss: 0.0556 +2022-05-15 05:43:15.101605: Average global foreground Dice: [0.9501, 0.9364, 0.9167] +2022-05-15 05:43:15.101692: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 05:43:15.515376: lr: 4.5e-05 +2022-05-15 05:43:15.515576: saving scheduled checkpoint file... +2022-05-15 05:43:15.565391: saving checkpoint... +2022-05-15 05:43:16.306967: done, saving took 0.79 seconds +2022-05-15 05:43:16.322984: done +2022-05-15 05:43:34.569956: This epoch took 480.366328 s + +2022-05-15 05:43:34.570242: +epoch: 189 +2022-05-15 05:50:50.231953: train loss : 0.0377 +2022-05-15 05:51:15.232754: validation loss: 0.0449 +2022-05-15 05:51:15.233420: Average global foreground Dice: [0.9503, 0.9342, 0.9159] +2022-05-15 05:51:15.233533: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 05:51:15.647637: lr: 4.4e-05 +2022-05-15 05:51:15.647825: saving scheduled checkpoint file... +2022-05-15 05:51:15.698020: saving checkpoint... +2022-05-15 05:51:16.454103: done, saving took 0.81 seconds +2022-05-15 05:51:16.469630: done +2022-05-15 05:51:35.174162: This epoch took 480.603816 s + +2022-05-15 05:51:35.174428: +epoch: 190 +2022-05-15 05:58:49.999119: train loss : 0.0356 +2022-05-15 05:59:14.996297: validation loss: 0.0617 +2022-05-15 05:59:14.997014: Average global foreground Dice: [0.9528, 0.9324, 0.9199] +2022-05-15 05:59:14.997097: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 05:59:15.411795: lr: 4.3e-05 +2022-05-15 05:59:15.411986: saving scheduled checkpoint file... +2022-05-15 05:59:15.460413: saving checkpoint... +2022-05-15 05:59:16.199684: done, saving took 0.79 seconds +2022-05-15 05:59:16.210093: done +2022-05-15 05:59:48.050815: This epoch took 492.876282 s + +2022-05-15 05:59:48.051139: +epoch: 191 +2022-05-15 06:07:03.258366: train loss : 0.0379 +2022-05-15 06:07:28.239123: validation loss: 0.0491 +2022-05-15 06:07:28.239774: Average global foreground Dice: [0.9522, 0.9365, 0.9215] +2022-05-15 06:07:28.239866: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 06:07:28.656312: lr: 4.1e-05 +2022-05-15 06:07:28.656513: saving scheduled checkpoint file... +2022-05-15 06:07:28.709660: saving checkpoint... +2022-05-15 06:07:29.486792: done, saving took 0.83 seconds +2022-05-15 06:07:29.503079: done +2022-05-15 06:07:56.432281: This epoch took 488.381042 s + +2022-05-15 06:07:56.432550: +epoch: 192 +2022-05-15 06:15:10.809160: train loss : 0.0389 +2022-05-15 06:15:35.788899: validation loss: 0.0560 +2022-05-15 06:15:35.789491: Average global foreground Dice: [0.946, 0.9288, 0.9192] +2022-05-15 06:15:35.789586: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 06:15:36.226840: lr: 4e-05 +2022-05-15 06:15:36.227051: saving scheduled checkpoint file... +2022-05-15 06:15:36.277021: saving checkpoint... +2022-05-15 06:15:37.042650: done, saving took 0.82 seconds +2022-05-15 06:15:37.057912: done +2022-05-15 06:15:54.285335: This epoch took 477.852667 s + +2022-05-15 06:15:54.285646: +epoch: 193 +2022-05-15 06:23:09.117692: train loss : 0.0369 +2022-05-15 06:23:34.095152: validation loss: 0.0496 +2022-05-15 06:23:34.095876: Average global foreground Dice: [0.9499, 0.934, 0.9165] +2022-05-15 06:23:34.095960: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 06:23:34.515046: lr: 3.9e-05 +2022-05-15 06:23:34.515241: saving scheduled checkpoint file... +2022-05-15 06:23:34.563760: saving checkpoint... +2022-05-15 06:23:35.291689: done, saving took 0.78 seconds +2022-05-15 06:23:35.319023: done +2022-05-15 06:23:55.295331: This epoch took 481.009584 s + +2022-05-15 06:23:55.295614: +epoch: 194 +2022-05-15 06:31:10.858582: train loss : 0.0352 +2022-05-15 06:31:35.822434: validation loss: 0.0515 +2022-05-15 06:31:35.823078: Average global foreground Dice: [0.9519, 0.9211, 0.9176] +2022-05-15 06:31:35.823164: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 06:31:36.246412: lr: 3.7e-05 +2022-05-15 06:31:36.246616: saving scheduled checkpoint file... +2022-05-15 06:31:36.299777: saving checkpoint... +2022-05-15 06:31:37.049633: done, saving took 0.80 seconds +2022-05-15 06:31:37.066507: done +2022-05-15 06:32:04.409148: This epoch took 489.113424 s + +2022-05-15 06:32:04.409434: +epoch: 195 +2022-05-15 06:39:19.387719: train loss : 0.0365 +2022-05-15 06:39:44.365045: validation loss: 0.0531 +2022-05-15 06:39:44.365703: Average global foreground Dice: [0.9487, 0.9304, 0.9154] +2022-05-15 06:39:44.365789: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 06:39:44.781132: lr: 3.6e-05 +2022-05-15 06:39:44.781326: saving scheduled checkpoint file... +2022-05-15 06:39:44.832001: saving checkpoint... +2022-05-15 06:39:45.655888: done, saving took 0.87 seconds +2022-05-15 06:39:45.672075: done +2022-05-15 06:40:02.658487: This epoch took 478.248951 s + +2022-05-15 06:40:02.658772: +epoch: 196 +2022-05-15 06:47:17.790580: train loss : 0.0400 +2022-05-15 06:47:42.742015: validation loss: 0.0585 +2022-05-15 06:47:42.742737: Average global foreground Dice: [0.9513, 0.9357, 0.9199] +2022-05-15 06:47:42.742822: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 06:47:43.160219: lr: 3.5e-05 +2022-05-15 06:47:43.160449: saving scheduled checkpoint file... +2022-05-15 06:47:43.208584: saving checkpoint... +2022-05-15 06:47:44.025366: done, saving took 0.86 seconds +2022-05-15 06:47:44.041751: done +2022-05-15 06:48:06.946481: This epoch took 484.287609 s + +2022-05-15 06:48:06.946816: +epoch: 197 +2022-05-15 06:55:22.279918: train loss : 0.0370 +2022-05-15 06:55:47.284760: validation loss: 0.0571 +2022-05-15 06:55:47.285516: Average global foreground Dice: [0.949, 0.9252, 0.9129] +2022-05-15 06:55:47.285611: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 06:55:47.702045: lr: 3.3e-05 +2022-05-15 06:55:47.702240: saving scheduled checkpoint file... +2022-05-15 06:55:47.751533: saving checkpoint... +2022-05-15 06:55:48.511159: done, saving took 0.81 seconds +2022-05-15 06:55:48.526567: done +2022-05-15 06:56:03.410502: This epoch took 476.463582 s + +2022-05-15 06:56:03.410789: +epoch: 198 +2022-05-15 07:03:18.834496: train loss : 0.0365 +2022-05-15 07:03:43.802289: validation loss: 0.0463 +2022-05-15 07:03:43.803442: Average global foreground Dice: [0.9535, 0.9298, 0.9191] +2022-05-15 07:03:43.803525: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 07:03:44.226648: lr: 3.2e-05 +2022-05-15 07:03:44.226843: saving scheduled checkpoint file... +2022-05-15 07:03:44.277197: saving checkpoint... +2022-05-15 07:03:45.109296: done, saving took 0.88 seconds +2022-05-15 07:03:45.127880: done +2022-05-15 07:04:08.686710: This epoch took 485.275800 s + +2022-05-15 07:04:08.687076: +epoch: 199 +2022-05-15 07:11:24.437480: train loss : 0.0346 +2022-05-15 07:11:49.424022: validation loss: 0.0468 +2022-05-15 07:11:49.424650: Average global foreground Dice: [0.9507, 0.9301, 0.9173] +2022-05-15 07:11:49.424751: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 07:11:49.844073: lr: 3.1e-05 +2022-05-15 07:11:49.844252: saving scheduled checkpoint file... +2022-05-15 07:11:49.891421: saving checkpoint... +2022-05-15 07:11:50.702091: done, saving took 0.86 seconds +2022-05-15 07:11:50.719989: done +2022-05-15 07:12:20.212897: This epoch took 491.525715 s + +2022-05-15 07:12:20.213178: +epoch: 200 +2022-05-15 07:19:35.889523: train loss : 0.0376 +2022-05-15 07:20:00.940571: validation loss: 0.0549 +2022-05-15 07:20:00.941259: Average global foreground Dice: [0.9507, 0.939, 0.9238] +2022-05-15 07:20:00.941342: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 07:20:01.358663: lr: 3e-05 +2022-05-15 07:20:01.358857: saving scheduled checkpoint file... +2022-05-15 07:20:01.408668: saving checkpoint... +2022-05-15 07:20:02.227159: done, saving took 0.87 seconds +2022-05-15 07:20:02.245073: done +2022-05-15 07:20:38.164773: This epoch took 497.951474 s + +2022-05-15 07:20:38.165277: +epoch: 201 +2022-05-15 07:27:53.005304: train loss : 0.0380 +2022-05-15 07:28:18.017902: validation loss: 0.0487 +2022-05-15 07:28:18.018553: Average global foreground Dice: [0.9494, 0.9414, 0.9222] +2022-05-15 07:28:18.018651: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 07:28:18.438210: lr: 2.9e-05 +2022-05-15 07:28:18.438406: saving scheduled checkpoint file... +2022-05-15 07:28:18.486577: saving checkpoint... +2022-05-15 07:28:19.309730: done, saving took 0.87 seconds +2022-05-15 07:28:19.323133: done +2022-05-15 07:28:42.537919: This epoch took 484.372462 s + +2022-05-15 07:28:42.538209: +epoch: 202 +2022-05-15 07:35:58.393770: train loss : 0.0382 +2022-05-15 07:36:23.341347: validation loss: 0.0559 +2022-05-15 07:36:23.342040: Average global foreground Dice: [0.9467, 0.9396, 0.9191] +2022-05-15 07:36:23.342125: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 07:36:23.759959: lr: 2.8e-05 +2022-05-15 07:36:23.760165: saving scheduled checkpoint file... +2022-05-15 07:36:23.809179: saving checkpoint... +2022-05-15 07:36:24.630140: done, saving took 0.87 seconds +2022-05-15 07:36:24.648168: done +2022-05-15 07:36:46.574608: This epoch took 484.036268 s + +2022-05-15 07:36:46.574901: +epoch: 203 +2022-05-15 07:44:01.772193: train loss : 0.0354 +2022-05-15 07:44:26.787109: validation loss: 0.0412 +2022-05-15 07:44:26.787737: Average global foreground Dice: [0.9523, 0.9392, 0.9191] +2022-05-15 07:44:26.787825: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 07:44:27.207529: lr: 2.6e-05 +2022-05-15 07:44:27.207730: saving scheduled checkpoint file... +2022-05-15 07:44:27.256672: saving checkpoint... +2022-05-15 07:44:28.084305: done, saving took 0.88 seconds +2022-05-15 07:44:28.102405: done +2022-05-15 07:44:50.480231: This epoch took 483.905209 s + +2022-05-15 07:44:50.480609: +epoch: 204 +2022-05-15 07:52:06.097186: train loss : 0.0360 +2022-05-15 07:52:31.056467: validation loss: 0.0458 +2022-05-15 07:52:31.057211: Average global foreground Dice: [0.9521, 0.94, 0.9231] +2022-05-15 07:52:31.057296: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 07:52:31.474163: lr: 2.5e-05 +2022-05-15 07:52:31.474345: saving scheduled checkpoint file... +2022-05-15 07:52:31.524863: saving checkpoint... +2022-05-15 07:52:32.342099: done, saving took 0.87 seconds +2022-05-15 07:52:32.360045: done +2022-05-15 07:52:49.504972: This epoch took 479.024245 s + +2022-05-15 07:52:49.505347: +epoch: 205 +2022-05-15 08:00:05.362933: train loss : 0.0391 +2022-05-15 08:00:30.355473: validation loss: 0.0552 +2022-05-15 08:00:30.356259: Average global foreground Dice: [0.9535, 0.9389, 0.9224] +2022-05-15 08:00:30.356346: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 08:00:30.766829: lr: 2.4e-05 +2022-05-15 08:00:30.767038: saving scheduled checkpoint file... +2022-05-15 08:00:30.815094: saving checkpoint... +2022-05-15 08:00:31.636047: done, saving took 0.87 seconds +2022-05-15 08:00:31.653717: done +2022-05-15 08:01:09.150798: This epoch took 499.645328 s + +2022-05-15 08:01:09.151103: +epoch: 206 +2022-05-15 08:08:24.780820: train loss : 0.0387 +2022-05-15 08:08:49.809999: validation loss: 0.0600 +2022-05-15 08:08:49.810770: Average global foreground Dice: [0.9523, 0.9311, 0.9206] +2022-05-15 08:08:49.810855: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 08:08:50.221448: lr: 2.3e-05 +2022-05-15 08:08:50.221650: saving scheduled checkpoint file... +2022-05-15 08:08:50.271786: saving checkpoint... +2022-05-15 08:08:51.086668: done, saving took 0.86 seconds +2022-05-15 08:08:51.102722: done +2022-05-15 08:09:08.778020: This epoch took 479.626819 s + +2022-05-15 08:09:08.778294: +epoch: 207 +2022-05-15 08:16:22.962937: train loss : 0.0359 +2022-05-15 08:16:47.980716: validation loss: 0.0467 +2022-05-15 08:16:47.981363: Average global foreground Dice: [0.9521, 0.9324, 0.9165] +2022-05-15 08:16:47.981448: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 08:16:48.391251: lr: 2.2e-05 +2022-05-15 08:16:48.391440: saving scheduled checkpoint file... +2022-05-15 08:16:48.439070: saving checkpoint... +2022-05-15 08:16:49.258487: done, saving took 0.87 seconds +2022-05-15 08:16:49.273206: done +2022-05-15 08:17:28.047440: This epoch took 499.269046 s + +2022-05-15 08:17:28.047784: +epoch: 208 +2022-05-15 08:24:43.596999: train loss : 0.0343 +2022-05-15 08:25:08.545601: validation loss: 0.0479 +2022-05-15 08:25:08.546255: Average global foreground Dice: [0.952, 0.9271, 0.9201] +2022-05-15 08:25:08.546339: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 08:25:08.958231: lr: 2.1e-05 +2022-05-15 08:25:08.958406: saving scheduled checkpoint file... +2022-05-15 08:25:09.008244: saving checkpoint... +2022-05-15 08:25:09.828452: done, saving took 0.87 seconds +2022-05-15 08:25:09.844489: done +2022-05-15 08:25:43.729711: This epoch took 495.681822 s + +2022-05-15 08:25:43.730044: +epoch: 209 +2022-05-15 08:32:59.305598: train loss : 0.0372 +2022-05-15 08:33:24.248249: validation loss: 0.0463 +2022-05-15 08:33:24.248926: Average global foreground Dice: [0.9519, 0.9369, 0.9251] +2022-05-15 08:33:24.249012: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 08:33:24.661459: lr: 2e-05 +2022-05-15 08:33:24.661662: saving scheduled checkpoint file... +2022-05-15 08:33:24.712157: saving checkpoint... +2022-05-15 08:33:25.468760: done, saving took 0.81 seconds +2022-05-15 08:33:25.483838: done +2022-05-15 08:33:46.355247: This epoch took 482.625102 s + +2022-05-15 08:33:46.355513: +epoch: 210 +2022-05-15 08:41:01.547489: train loss : 0.0370 +2022-05-15 08:41:26.514625: validation loss: 0.0533 +2022-05-15 08:41:26.515290: Average global foreground Dice: [0.9513, 0.9355, 0.9225] +2022-05-15 08:41:26.515373: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 08:41:26.925023: lr: 1.9e-05 +2022-05-15 08:41:26.925221: saving scheduled checkpoint file... +2022-05-15 08:41:26.973587: saving checkpoint... +2022-05-15 08:41:27.731440: done, saving took 0.81 seconds +2022-05-15 08:41:27.742701: done +2022-05-15 08:41:46.483346: This epoch took 480.127681 s + +2022-05-15 08:41:46.483616: +epoch: 211 +2022-05-15 08:49:02.533055: train loss : 0.0374 +2022-05-15 08:49:27.614228: validation loss: 0.0544 +2022-05-15 08:49:27.614989: Average global foreground Dice: [0.9523, 0.9272, 0.9183] +2022-05-15 08:49:27.615073: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 08:49:28.025126: lr: 1.8e-05 +2022-05-15 08:49:28.025329: saving scheduled checkpoint file... +2022-05-15 08:49:28.078207: saving checkpoint... +2022-05-15 08:49:28.814154: done, saving took 0.79 seconds +2022-05-15 08:49:28.831172: done +2022-05-15 08:49:48.313171: This epoch took 481.829453 s + +2022-05-15 08:49:48.313453: +epoch: 212 +2022-05-15 08:57:03.775923: train loss : 0.0366 +2022-05-15 08:57:28.854430: validation loss: 0.0620 +2022-05-15 08:57:28.855093: Average global foreground Dice: [0.9481, 0.9316, 0.9178] +2022-05-15 08:57:28.855179: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 08:57:29.264895: lr: 1.7e-05 +2022-05-15 08:57:29.265087: saving scheduled checkpoint file... +2022-05-15 08:57:29.315491: saving checkpoint... +2022-05-15 08:57:30.084822: done, saving took 0.82 seconds +2022-05-15 08:57:30.101295: done +2022-05-15 08:57:46.800259: This epoch took 478.486704 s + +2022-05-15 08:57:46.800540: +epoch: 213 +2022-05-15 09:05:02.864237: train loss : 0.0361 +2022-05-15 09:05:27.926488: validation loss: 0.0543 +2022-05-15 09:05:27.927102: Average global foreground Dice: [0.9508, 0.9262, 0.9217] +2022-05-15 09:05:27.927188: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 09:05:28.337820: lr: 1.6e-05 +2022-05-15 09:05:28.338023: saving scheduled checkpoint file... +2022-05-15 09:05:28.386599: saving checkpoint... +2022-05-15 09:05:29.184961: done, saving took 0.85 seconds +2022-05-15 09:05:29.194518: done +2022-05-15 09:05:51.000310: This epoch took 484.199653 s + +2022-05-15 09:05:51.000611: +epoch: 214 +2022-05-15 09:13:06.946517: train loss : 0.0360 +2022-05-15 09:13:31.992230: validation loss: 0.0466 +2022-05-15 09:13:31.992870: Average global foreground Dice: [0.9485, 0.9294, 0.9179] +2022-05-15 09:13:31.992956: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 09:13:32.408364: lr: 1.5e-05 +2022-05-15 09:13:32.408545: saving scheduled checkpoint file... +2022-05-15 09:13:32.456837: saving checkpoint... +2022-05-15 09:13:33.171739: done, saving took 0.76 seconds +2022-05-15 09:13:33.188460: done +2022-05-15 09:13:50.689392: This epoch took 479.688665 s + +2022-05-15 09:13:50.689759: +epoch: 215 +2022-05-15 09:21:06.191499: train loss : 0.0368 +2022-05-15 09:21:31.249788: validation loss: 0.0582 +2022-05-15 09:21:31.250498: Average global foreground Dice: [0.9499, 0.9331, 0.9202] +2022-05-15 09:21:31.250593: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 09:21:31.660841: lr: 1.5e-05 +2022-05-15 09:21:31.661050: saving scheduled checkpoint file... +2022-05-15 09:21:31.711026: saving checkpoint... +2022-05-15 09:21:32.458420: done, saving took 0.80 seconds +2022-05-15 09:21:32.473470: done +2022-05-15 09:21:52.693511: This epoch took 482.003651 s + +2022-05-15 09:21:52.693806: +epoch: 216 +2022-05-15 09:29:08.280115: train loss : 0.0360 +2022-05-15 09:29:33.370654: validation loss: 0.0490 +2022-05-15 09:29:33.371262: Average global foreground Dice: [0.9506, 0.9351, 0.9239] +2022-05-15 09:29:33.371344: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 09:29:33.786949: lr: 1.4e-05 +2022-05-15 09:29:33.787134: saving scheduled checkpoint file... +2022-05-15 09:29:33.836576: saving checkpoint... +2022-05-15 09:29:34.588345: done, saving took 0.80 seconds +2022-05-15 09:29:34.605162: done +2022-05-15 09:29:58.779168: This epoch took 486.085262 s + +2022-05-15 09:29:58.779517: +epoch: 217 +2022-05-15 09:37:13.546309: train loss : 0.0379 +2022-05-15 09:37:38.603475: validation loss: 0.0535 +2022-05-15 09:37:38.604188: Average global foreground Dice: [0.9512, 0.929, 0.9183] +2022-05-15 09:37:38.604271: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 09:37:39.037574: lr: 1.3e-05 +2022-05-15 09:37:39.037776: saving scheduled checkpoint file... +2022-05-15 09:37:39.087139: saving checkpoint... +2022-05-15 09:37:39.905454: done, saving took 0.87 seconds +2022-05-15 09:37:39.923941: done +2022-05-15 09:38:08.997515: This epoch took 490.217879 s + +2022-05-15 09:38:08.997786: +epoch: 218 +2022-05-15 09:45:24.335414: train loss : 0.0356 +2022-05-15 09:45:49.423175: validation loss: 0.0475 +2022-05-15 09:45:49.423936: Average global foreground Dice: [0.9517, 0.936, 0.9233] +2022-05-15 09:45:49.424059: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 09:45:49.898923: lr: 1.2e-05 +2022-05-15 09:45:49.899114: saving scheduled checkpoint file... +2022-05-15 09:45:49.953782: saving checkpoint... +2022-05-15 09:45:50.808013: done, saving took 0.91 seconds +2022-05-15 09:45:50.824414: done +2022-05-15 09:46:18.638694: This epoch took 489.640797 s + +2022-05-15 09:46:18.638968: +epoch: 219 +2022-05-15 09:53:34.844466: train loss : 0.0393 +2022-05-15 09:53:59.883483: validation loss: 0.0526 +2022-05-15 09:53:59.884218: Average global foreground Dice: [0.952, 0.9344, 0.9258] +2022-05-15 09:53:59.884303: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 09:54:00.298672: lr: 1.1e-05 +2022-05-15 09:54:00.298867: saving scheduled checkpoint file... +2022-05-15 09:54:00.346723: saving checkpoint... +2022-05-15 09:54:01.163468: done, saving took 0.86 seconds +2022-05-15 09:54:01.181616: done +2022-05-15 09:54:20.526885: This epoch took 481.887819 s + +2022-05-15 09:54:20.527177: +epoch: 220 +2022-05-15 10:01:36.411660: train loss : 0.0364 +2022-05-15 10:02:01.491039: validation loss: 0.0569 +2022-05-15 10:02:01.491779: Average global foreground Dice: [0.9482, 0.9335, 0.9197] +2022-05-15 10:02:01.491867: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 10:02:01.902171: lr: 1.1e-05 +2022-05-15 10:02:01.902367: saving scheduled checkpoint file... +2022-05-15 10:02:01.952146: saving checkpoint... +2022-05-15 10:02:02.780157: done, saving took 0.88 seconds +2022-05-15 10:02:02.798960: done +2022-05-15 10:02:24.086868: This epoch took 483.559585 s + +2022-05-15 10:02:24.087160: +epoch: 221 +2022-05-15 10:09:39.627606: train loss : 0.0359 +2022-05-15 10:10:04.703516: validation loss: 0.0584 +2022-05-15 10:10:04.704160: Average global foreground Dice: [0.9496, 0.9332, 0.921] +2022-05-15 10:10:04.704245: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 10:10:05.118819: lr: 1e-05 +2022-05-15 10:10:05.119034: saving scheduled checkpoint file... +2022-05-15 10:10:05.169023: saving checkpoint... +2022-05-15 10:10:05.998222: done, saving took 0.88 seconds +2022-05-15 10:10:06.014583: done +2022-05-15 10:10:25.908365: This epoch took 481.821106 s + +2022-05-15 10:10:25.908633: +epoch: 222 +2022-05-15 10:17:41.827397: train loss : 0.0348 +2022-05-15 10:18:06.866477: validation loss: 0.0490 +2022-05-15 10:18:06.867206: Average global foreground Dice: [0.9504, 0.9366, 0.9232] +2022-05-15 10:18:06.867292: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 10:18:07.278836: lr: 9e-06 +2022-05-15 10:18:07.279018: saving scheduled checkpoint file... +2022-05-15 10:18:07.330000: saving checkpoint... +2022-05-15 10:18:08.041275: done, saving took 0.76 seconds +2022-05-15 10:18:08.057658: done +2022-05-15 10:18:25.814725: This epoch took 479.905972 s + +2022-05-15 10:18:25.815048: +epoch: 223 +2022-05-15 10:25:41.535014: train loss : 0.0360 +2022-05-15 10:26:06.587187: validation loss: 0.0656 +2022-05-15 10:26:06.587887: Average global foreground Dice: [0.952, 0.9318, 0.9219] +2022-05-15 10:26:06.587974: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 10:26:06.998754: lr: 9e-06 +2022-05-15 10:26:06.998945: saving scheduled checkpoint file... +2022-05-15 10:26:07.047116: saving checkpoint... +2022-05-15 10:26:07.863696: done, saving took 0.86 seconds +2022-05-15 10:26:07.875572: done +2022-05-15 10:26:26.882282: This epoch took 481.067136 s + +2022-05-15 10:26:26.882606: +epoch: 224 +2022-05-15 10:33:42.707476: train loss : 0.0359 +2022-05-15 10:34:07.742742: validation loss: 0.0472 +2022-05-15 10:34:07.743361: Average global foreground Dice: [0.9478, 0.9385, 0.9226] +2022-05-15 10:34:07.743455: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 10:34:08.160681: lr: 8e-06 +2022-05-15 10:34:08.160871: saving scheduled checkpoint file... +2022-05-15 10:34:08.214021: saving checkpoint... +2022-05-15 10:34:09.024823: done, saving took 0.86 seconds +2022-05-15 10:34:09.042880: done +2022-05-15 10:34:25.594705: This epoch took 478.711987 s + +2022-05-15 10:34:25.594988: +epoch: 225 +2022-05-15 10:41:42.520641: train loss : 0.0356 +2022-05-15 10:42:07.570288: validation loss: 0.0511 +2022-05-15 10:42:07.570946: Average global foreground Dice: [0.9493, 0.9299, 0.9219] +2022-05-15 10:42:07.571035: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 10:42:07.988201: lr: 7e-06 +2022-05-15 10:42:07.988395: saving scheduled checkpoint file... +2022-05-15 10:42:08.037842: saving checkpoint... +2022-05-15 10:42:08.857832: done, saving took 0.87 seconds +2022-05-15 10:42:08.873874: done +2022-05-15 10:42:24.950150: This epoch took 479.355059 s + +2022-05-15 10:42:24.950432: +epoch: 226 +2022-05-15 10:49:40.434459: train loss : 0.0367 +2022-05-15 10:50:05.542319: validation loss: 0.0522 +2022-05-15 10:50:05.542964: Average global foreground Dice: [0.9493, 0.9313, 0.9179] +2022-05-15 10:50:05.543052: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 10:50:05.960448: lr: 7e-06 +2022-05-15 10:50:05.960656: saving scheduled checkpoint file... +2022-05-15 10:50:06.009563: saving checkpoint... +2022-05-15 10:50:06.828782: done, saving took 0.87 seconds +2022-05-15 10:50:06.846849: done +2022-05-15 10:50:29.141023: This epoch took 484.190491 s + +2022-05-15 10:50:29.141310: +epoch: 227 +2022-05-15 10:57:45.493638: train loss : 0.0382 +2022-05-15 10:58:10.529570: validation loss: 0.0547 +2022-05-15 10:58:10.530207: Average global foreground Dice: [0.9493, 0.9301, 0.9197] +2022-05-15 10:58:10.530291: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 10:58:10.940047: lr: 6e-06 +2022-05-15 10:58:10.940247: saving scheduled checkpoint file... +2022-05-15 10:58:10.989516: saving checkpoint... +2022-05-15 10:58:11.813657: done, saving took 0.87 seconds +2022-05-15 10:58:11.829952: done +2022-05-15 10:58:31.203484: This epoch took 482.062054 s + +2022-05-15 10:58:31.203947: +epoch: 228 +2022-05-15 11:05:47.223631: train loss : 0.0363 +2022-05-15 11:06:12.235457: validation loss: 0.0503 +2022-05-15 11:06:12.236153: Average global foreground Dice: [0.953, 0.9418, 0.9241] +2022-05-15 11:06:12.236240: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 11:06:12.647591: lr: 6e-06 +2022-05-15 11:06:12.647779: saving scheduled checkpoint file... +2022-05-15 11:06:12.697831: saving checkpoint... +2022-05-15 11:06:13.542253: done, saving took 0.89 seconds +2022-05-15 11:06:13.552946: done +2022-05-15 11:06:34.995505: This epoch took 483.791446 s + +2022-05-15 11:06:34.995790: +epoch: 229 +2022-05-15 11:13:51.375784: train loss : 0.0366 +2022-05-15 11:14:16.373983: validation loss: 0.0542 +2022-05-15 11:14:16.374646: Average global foreground Dice: [0.9496, 0.9316, 0.9161] +2022-05-15 11:14:16.374734: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 11:14:16.784981: lr: 5e-06 +2022-05-15 11:14:16.785176: saving scheduled checkpoint file... +2022-05-15 11:14:16.833402: saving checkpoint... +2022-05-15 11:14:17.645550: done, saving took 0.86 seconds +2022-05-15 11:14:17.656318: done +2022-05-15 11:14:34.888585: This epoch took 479.892676 s + +2022-05-15 11:14:34.888864: +epoch: 230 +2022-05-15 11:21:50.818128: train loss : 0.0352 +2022-05-15 11:22:15.811493: validation loss: 0.0486 +2022-05-15 11:22:15.812192: Average global foreground Dice: [0.9518, 0.9338, 0.9222] +2022-05-15 11:22:15.812276: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 11:22:16.226908: lr: 5e-06 +2022-05-15 11:22:16.227097: saving scheduled checkpoint file... +2022-05-15 11:22:16.277449: saving checkpoint... +2022-05-15 11:22:17.115908: done, saving took 0.89 seconds +2022-05-15 11:22:17.134141: done +2022-05-15 11:22:34.457625: This epoch took 479.568609 s + +2022-05-15 11:22:34.457952: +epoch: 231 +2022-05-15 11:29:50.199736: train loss : 0.0341 +2022-05-15 11:30:15.238173: validation loss: 0.0651 +2022-05-15 11:30:15.238793: Average global foreground Dice: [0.9509, 0.934, 0.9275] +2022-05-15 11:30:15.238877: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 11:30:15.664826: lr: 4e-06 +2022-05-15 11:30:15.665020: saving scheduled checkpoint file... +2022-05-15 11:30:15.717156: saving checkpoint... +2022-05-15 11:30:16.536197: done, saving took 0.87 seconds +2022-05-15 11:30:16.554106: done +2022-05-15 11:30:35.421410: This epoch took 480.963356 s + +2022-05-15 11:30:35.421723: +epoch: 232 +2022-05-15 11:37:51.615944: train loss : 0.0358 +2022-05-15 11:38:16.690305: validation loss: 0.0602 +2022-05-15 11:38:16.690934: Average global foreground Dice: [0.9479, 0.9357, 0.9231] +2022-05-15 11:38:16.691152: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 11:38:17.103986: lr: 4e-06 +2022-05-15 11:38:17.104165: saving scheduled checkpoint file... +2022-05-15 11:38:17.151781: saving checkpoint... +2022-05-15 11:38:17.963236: done, saving took 0.86 seconds +2022-05-15 11:38:17.979380: done +2022-05-15 11:38:36.473113: This epoch took 481.051289 s + +2022-05-15 11:38:36.473388: +epoch: 233 +2022-05-15 11:45:51.325040: train loss : 0.0363 +2022-05-15 11:46:16.402973: validation loss: 0.0579 +2022-05-15 11:46:16.403647: Average global foreground Dice: [0.9554, 0.9347, 0.9221] +2022-05-15 11:46:16.403733: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 11:46:16.814671: lr: 3e-06 +2022-05-15 11:46:16.814867: saving scheduled checkpoint file... +2022-05-15 11:46:16.864781: saving checkpoint... +2022-05-15 11:46:17.684041: done, saving took 0.87 seconds +2022-05-15 11:46:17.700368: done +2022-05-15 11:46:38.189613: This epoch took 481.716120 s + +2022-05-15 11:46:38.189889: +epoch: 234 +2022-05-15 11:53:53.795597: train loss : 0.0365 +2022-05-15 11:54:18.821951: validation loss: 0.0448 +2022-05-15 11:54:18.822697: Average global foreground Dice: [0.956, 0.9424, 0.9261] +2022-05-15 11:54:18.822786: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 11:54:19.238725: lr: 3e-06 +2022-05-15 11:54:19.238918: saving scheduled checkpoint file... +2022-05-15 11:54:19.289765: saving checkpoint... +2022-05-15 11:54:20.112958: done, saving took 0.87 seconds +2022-05-15 11:54:20.130949: done +2022-05-15 11:54:38.099495: This epoch took 479.909508 s + +2022-05-15 11:54:38.099793: +epoch: 235 +2022-05-15 12:01:53.074145: train loss : 0.0345 +2022-05-15 12:02:18.102525: validation loss: 0.0567 +2022-05-15 12:02:18.103247: Average global foreground Dice: [0.9519, 0.9143, 0.9172] +2022-05-15 12:02:18.103331: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 12:02:18.525082: lr: 3e-06 +2022-05-15 12:02:18.525277: saving scheduled checkpoint file... +2022-05-15 12:02:18.573314: saving checkpoint... +2022-05-15 12:02:19.381939: done, saving took 0.86 seconds +2022-05-15 12:02:19.393641: done +2022-05-15 12:02:37.741507: This epoch took 479.641539 s + +2022-05-15 12:02:37.741832: +epoch: 236 +2022-05-15 12:09:53.422242: train loss : 0.0369 +2022-05-15 12:10:18.534998: validation loss: 0.0530 +2022-05-15 12:10:18.535702: Average global foreground Dice: [0.9508, 0.9349, 0.9176] +2022-05-15 12:10:18.535788: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 12:10:18.952898: lr: 2e-06 +2022-05-15 12:10:18.953091: saving scheduled checkpoint file... +2022-05-15 12:10:19.005858: saving checkpoint... +2022-05-15 12:10:19.825912: done, saving took 0.87 seconds +2022-05-15 12:10:19.844404: done +2022-05-15 12:10:44.373891: This epoch took 486.631958 s + +2022-05-15 12:10:44.374175: +epoch: 237 +2022-05-15 12:17:59.756845: train loss : 0.0373 +2022-05-15 12:18:24.795362: validation loss: 0.0511 +2022-05-15 12:18:24.796052: Average global foreground Dice: [0.9524, 0.9409, 0.9251] +2022-05-15 12:18:24.796148: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 12:18:25.210150: lr: 2e-06 +2022-05-15 12:18:25.210355: saving scheduled checkpoint file... +2022-05-15 12:18:25.260024: saving checkpoint... +2022-05-15 12:18:26.086198: done, saving took 0.88 seconds +2022-05-15 12:18:26.104427: done +2022-05-15 12:18:41.877221: This epoch took 477.502946 s + +2022-05-15 12:18:41.877509: +epoch: 238 +2022-05-15 12:25:56.935703: train loss : 0.0356 +2022-05-15 12:26:21.984458: validation loss: 0.0550 +2022-05-15 12:26:21.985076: Average global foreground Dice: [0.9536, 0.9372, 0.9206] +2022-05-15 12:26:21.985192: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 12:26:22.398610: lr: 2e-06 +2022-05-15 12:26:22.398808: saving scheduled checkpoint file... +2022-05-15 12:26:22.450106: saving checkpoint... +2022-05-15 12:26:23.280327: done, saving took 0.88 seconds +2022-05-15 12:26:23.296460: done +2022-05-15 12:26:45.091907: This epoch took 483.214253 s + +2022-05-15 12:26:45.092244: +epoch: 239 +2022-05-15 12:34:00.790616: train loss : 0.0366 +2022-05-15 12:34:25.828134: validation loss: 0.0558 +2022-05-15 12:34:25.828859: Average global foreground Dice: [0.9508, 0.9408, 0.9259] +2022-05-15 12:34:25.828945: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 12:34:26.241752: lr: 1e-06 +2022-05-15 12:34:26.241942: saving scheduled checkpoint file... +2022-05-15 12:34:26.290483: saving checkpoint... +2022-05-15 12:34:27.113239: done, saving took 0.87 seconds +2022-05-15 12:34:27.125240: done +2022-05-15 12:34:44.369706: This epoch took 479.277347 s + +2022-05-15 12:34:44.369997: +epoch: 240 +2022-05-15 12:41:59.789044: train loss : 0.0368 +2022-05-15 12:42:24.808699: validation loss: 0.0548 +2022-05-15 12:42:24.809394: Average global foreground Dice: [0.9519, 0.9364, 0.9258] +2022-05-15 12:42:24.809481: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 12:42:25.229234: lr: 1e-06 +2022-05-15 12:42:25.229439: saving scheduled checkpoint file... +2022-05-15 12:42:25.282744: saving checkpoint... +2022-05-15 12:42:26.105185: done, saving took 0.88 seconds +2022-05-15 12:42:26.123532: done +2022-05-15 12:42:43.861931: This epoch took 479.491834 s + +2022-05-15 12:42:43.862211: +epoch: 241 +2022-05-15 12:49:59.837996: train loss : 0.0369 +2022-05-15 12:50:24.848631: validation loss: 0.0485 +2022-05-15 12:50:24.849371: Average global foreground Dice: [0.9521, 0.9345, 0.9227] +2022-05-15 12:50:24.849457: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 12:50:25.316078: lr: 1e-06 +2022-05-15 12:50:25.316262: saving scheduled checkpoint file... +2022-05-15 12:50:25.370573: saving checkpoint... +2022-05-15 12:50:26.261854: done, saving took 0.95 seconds +2022-05-15 12:50:26.280830: done +2022-05-15 12:50:44.249664: This epoch took 480.387350 s + +2022-05-15 12:50:44.249953: +epoch: 242 +2022-05-15 12:57:59.500136: train loss : 0.0366 +2022-05-15 12:58:24.583237: validation loss: 0.0529 +2022-05-15 12:58:24.583947: Average global foreground Dice: [0.9518, 0.9385, 0.9227] +2022-05-15 12:58:24.584033: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 12:58:24.996439: lr: 1e-06 +2022-05-15 12:58:24.996649: saving scheduled checkpoint file... +2022-05-15 12:58:25.045505: saving checkpoint... +2022-05-15 12:58:25.873563: done, saving took 0.88 seconds +2022-05-15 12:58:25.889029: done +2022-05-15 12:58:46.714173: This epoch took 482.464120 s + +2022-05-15 12:58:46.714494: +epoch: 243 +2022-05-15 13:06:02.158926: train loss : 0.0366 +2022-05-15 13:06:27.222476: validation loss: 0.0607 +2022-05-15 13:06:27.223184: Average global foreground Dice: [0.9516, 0.9255, 0.9203] +2022-05-15 13:06:27.223271: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 13:06:27.644257: lr: 0.0 +2022-05-15 13:06:27.644444: saving scheduled checkpoint file... +2022-05-15 13:06:27.694085: saving checkpoint... +2022-05-15 13:06:28.522524: done, saving took 0.88 seconds +2022-05-15 13:06:28.542947: done +2022-05-15 13:06:49.443603: This epoch took 482.728852 s + +2022-05-15 13:06:49.443930: +epoch: 244 +2022-05-15 13:14:04.728892: train loss : 0.0381 +2022-05-15 13:14:29.701458: validation loss: 0.0562 +2022-05-15 13:14:29.702123: Average global foreground Dice: [0.9515, 0.9324, 0.9237] +2022-05-15 13:14:29.702210: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 13:14:30.118659: lr: 0.0 +2022-05-15 13:14:30.118851: saving scheduled checkpoint file... +2022-05-15 13:14:30.172141: saving checkpoint... +2022-05-15 13:14:30.996458: done, saving took 0.88 seconds +2022-05-15 13:14:31.014731: done +2022-05-15 13:14:50.821321: This epoch took 481.377290 s + +2022-05-15 13:14:50.821612: +epoch: 245 +2022-05-15 13:22:05.890761: train loss : 0.0361 +2022-05-15 13:22:30.858722: validation loss: 0.0491 +2022-05-15 13:22:30.859335: Average global foreground Dice: [0.9509, 0.9352, 0.9193] +2022-05-15 13:22:30.859418: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 13:22:31.274185: lr: 0.0 +2022-05-15 13:22:31.274379: saving scheduled checkpoint file... +2022-05-15 13:22:31.323796: saving checkpoint... +2022-05-15 13:22:32.093477: done, saving took 0.82 seconds +2022-05-15 13:22:32.111742: done +2022-05-15 13:22:53.756593: This epoch took 482.934850 s + +2022-05-15 13:22:53.756995: +epoch: 246 +2022-05-15 13:30:09.317459: train loss : 0.0363 +2022-05-15 13:30:34.298730: validation loss: 0.0537 +2022-05-15 13:30:34.299376: Average global foreground Dice: [0.9514, 0.9358, 0.9244] +2022-05-15 13:30:34.299459: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 13:30:34.718442: lr: 0.0 +2022-05-15 13:30:34.718644: saving scheduled checkpoint file... +2022-05-15 13:30:34.769226: saving checkpoint... +2022-05-15 13:30:35.544666: done, saving took 0.83 seconds +2022-05-15 13:30:35.561519: done +2022-05-15 13:30:56.327924: This epoch took 482.570828 s + +2022-05-15 13:30:56.328210: +epoch: 247 +2022-05-15 13:38:12.151462: train loss : 0.0346 +2022-05-15 13:38:37.133783: validation loss: 0.0553 +2022-05-15 13:38:37.134419: Average global foreground Dice: [0.9509, 0.9337, 0.9206] +2022-05-15 13:38:37.134506: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 13:38:37.550047: lr: 0.0 +2022-05-15 13:38:37.550229: saving scheduled checkpoint file... +2022-05-15 13:38:37.598184: saving checkpoint... +2022-05-15 13:38:38.423790: done, saving took 0.87 seconds +2022-05-15 13:38:38.440045: done +2022-05-15 13:39:17.805525: This epoch took 501.477213 s + +2022-05-15 13:39:17.805855: +epoch: 248 +2022-05-15 13:46:33.782455: train loss : 0.0336 +2022-05-15 13:46:58.877351: validation loss: 0.0445 +2022-05-15 13:46:58.877966: Average global foreground Dice: [0.9506, 0.9314, 0.9209] +2022-05-15 13:46:58.878050: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 13:46:59.291456: lr: 0.0 +2022-05-15 13:46:59.291697: saving scheduled checkpoint file... +2022-05-15 13:46:59.342269: saving checkpoint... +2022-05-15 13:47:00.166389: done, saving took 0.87 seconds +2022-05-15 13:47:00.184738: done +2022-05-15 13:47:35.493026: This epoch took 497.687065 s + +2022-05-15 13:47:35.493323: +epoch: 249 +2022-05-15 13:54:51.534534: train loss : 0.0335 +2022-05-15 13:55:16.516414: validation loss: 0.0513 +2022-05-15 13:55:16.517064: Average global foreground Dice: [0.95, 0.9343, 0.9202] +2022-05-15 13:55:16.517160: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 13:55:16.936258: lr: 0.0 +2022-05-15 13:55:16.936460: saving scheduled checkpoint file... +2022-05-15 13:55:16.984953: saving checkpoint... +2022-05-15 13:55:17.744514: done, saving took 0.81 seconds +2022-05-15 13:55:17.754759: done +2022-05-15 13:55:50.142931: This epoch took 494.649486 s + +2022-05-15 13:55:50.189810: saving checkpoint... +2022-05-15 13:55:50.789546: done, saving took 0.65 seconds diff --git a/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_2/debug.json b/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_2/debug.json new file mode 100644 index 0000000000000000000000000000000000000000..de54ca540a40c39d8f4096d76d8fb63c73360d5f --- /dev/null +++ b/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_2/debug.json @@ -0,0 +1,107 @@ +{ + "all_tr_losses": "[]", + "all_val_eval_metrics": "[]", + "all_val_losses": "[]", + "all_val_losses_tr_mode": "[]", + "also_val_in_tr_mode": "False", + "amp_grad_scaler": "None", + "args": "Namespace(batch_size=2, config='./configs/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250.yaml', continue_training=False, crop_size=[128, 128, 128], dbs=False, deep_supervision_scales=[], deterministic=False, disable_decoder=False, disable_ds=True, disable_postprocessing_on_folds=False, disable_saving=False, drop_path=0.0, find_lr=False, find_zero_weight_decay=False, fix_ds_net_numpool=False, fold='2', fp32=False, hdfs_base='GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250', initial_lr=0.0003, is_sigmoid=False, is_spatial_aug_only=False, is_ssl=False, layer_decay=1.0, local_rank=0, loss_name='', lrschedule='warmup_cosine', mask_ratio=0.75, max_loss_cal='v1', max_num_epochs=250, merge_femur=False, min_lr=0, model='Generic_TransUNet_max_ppbp', model_params={'is_masked_attn': True, 'max_dec_layers': 3, 'is_max_bottleneck_transformer': False, 'max_msda': '', 'is_max_ms': True, 'max_ms_idxs': [-4, -3, -2], 'max_hidden_dim': 192, 'mw': 1.0, 'is_max_ds': True, 'is_masking': True, 'is_max_hungarian': True, 'num_queries': 20, 'is_max_cls': True, 'is_mhsa_float32': True}, n_class=17, network='3d_fullres', network_trainer='nnUNetTrainerV2_DDP', npz=False, opt_betas=None, opt_eps=1e-08, optim_name='adamw', p='nnUNetPlansv2.1', plan_update='', pretrained=False, pretrained_config=None, pretrained_weights=None, rank=0, reclip=None, resume='auto', skip_grad_nan=False, task='Task500_BraTS2021', task_pretrained='Task801_WORD', total_batch_size=None, use_compressed_data=False, val_final=False, val_folder='validation_raw', valbest=False, validation_only=False, vallatest=False, warmup_epochs=10, weight_decay=3e-05, world_size=8)", + "base_num_features": "32", + "basic_generator_patch_size": "[182 182 182]", + "batch_dice": "True", + "batch_size": "2", + "best_MA_tr_loss_for_patience": "None", + "best_epoch_based_on_MA_tr_loss": "None", + "best_val_eval_criterion_MA": "None", + "classes": "[1, 2, 3]", + "conv_per_stage": "2", + "data_aug_params": "{'selected_data_channels': None, 'selected_seg_channels': [0], 'do_elastic': True, 'elastic_deform_alpha': (0.0, 900.0), 'elastic_deform_sigma': (9.0, 13.0), 'p_eldef': 0.3, 'do_scaling': True, 'scale_range': (0.65, 1.6), 'independent_scale_factor_for_each_axis': True, 'p_independent_scale_per_axis': 0.3, 'p_scale': 0.3, 'do_rotation': True, 'rotation_x': (-1.5707963267948966, 1.5707963267948966), 'rotation_y': (-1.5707963267948966, 1.5707963267948966), 'rotation_z': (-1.5707963267948966, 1.5707963267948966), 'rotation_p_per_axis': 1, 'p_rot': 0.3, 'random_crop': False, 'random_crop_dist_to_border': None, 'do_gamma': True, 'gamma_retain_stats': True, 'gamma_range': (0.5, 1.6), 'p_gamma': 0.3, 'do_mirror': True, 'mirror_axes': (0, 1, 2), 'dummy_2D': False, 'mask_was_used_for_normalization': OrderedDict([(0, True), (1, True), (2, True), (3, True)]), 'border_mode_data': 'constant', 'all_segmentation_labels': None, 'move_last_seg_chanel_to_data': False, 'cascade_do_cascade_augmentations': False, 'cascade_random_binary_transform_p': 0.4, 'cascade_random_binary_transform_p_per_label': 1, 'cascade_random_binary_transform_size': (1, 8), 'cascade_remove_conn_comp_p': 0.2, 'cascade_remove_conn_comp_max_size_percent_threshold': 0.15, 'cascade_remove_conn_comp_fill_with_other_class_p': 0.0, 'do_additive_brightness': True, 'additive_brightness_p_per_sample': 0.3, 'additive_brightness_p_per_channel': 0.5, 'additive_brightness_mu': 0, 'additive_brightness_sigma': 0.2, 'num_threads': 12, 'num_cached_per_thread': 4, 'patch_size_for_spatialtransform': array([128, 128, 128]), 'eldef_deformation_scale': (0, 0.25)}", + "dataset_directory": "/opt/tiger/project/data/nnUNet_preprocessed/Task500_BraTS2021", + "deep_supervision_scales": "[[1, 1, 1], [0.5, 0.5, 0.5], [0.25, 0.25, 0.25], [0.125, 0.125, 0.125], [0.0625, 0.0625, 0.0625]]", + "deterministic": "False", + "disable_ds": "True", + "distribute_batch_size": "False", + "dl_tr": "", + "dl_val": "", + "do_dummy_2D_aug": "False", + "ds_loss_weights": "[1. 0. 0. 0. 0.]", + "epoch": "0", + "experiment_name": "nnUNetTrainerV2_DDP", + "find_zero_weight_decay": "False", + "fold": "2", + "folder_with_preprocessed_data": "/opt/tiger/project/data/nnUNet_preprocessed/Task500_BraTS2021/nnUNetData_plans_v2.1_stage0", + "fp16": "True", + "global_batch_size": "16", + "gt_niftis_folder": "/opt/tiger/project/data/nnUNet_preprocessed/Task500_BraTS2021/gt_segmentations", + "inference_pad_border_mode": "constant", + "inference_pad_kwargs": "{'constant_values': 0}", + "init_args": "('/opt/tiger/project/data/nnUNet_preprocessed/Task500_BraTS2021/nnUNetPlansv2.1_plans_3D.pkl', 2, 0, '/opt/tiger/project/data/nnUNet_trained_models/UNet_IN_NANFang/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250', '/opt/tiger/project/data/nnUNet_preprocessed/Task500_BraTS2021', True, 0, True, False, False, True)", + "initial_lr": "0.0003", + "input_size": "[128, 128, 128]", + "is_spatial_aug_only": "False", + "is_ssl": "False", + "layer_decay": "1.0", + "local_rank": "0", + "log_file": "/opt/tiger/project/data/nnUNet_trained_models/UNet_IN_NANFang/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_2/training_log_2022_5_14_04_20_21.txt", + "loss": "None", + "lr_scheduler": "", + "lr_scheduler_eps": "0.001", + "lr_scheduler_name": "warmup_cosine", + "lr_scheduler_patience": "30", + "lr_threshold": "1e-06", + "max_num_epochs": "250", + "min_lr": "0", + "min_region_size_per_class": "None", + "min_size_per_class": "None", + "model": "Generic_TransUNet_max_ppbp", + "model_params": "{'is_masked_attn': True, 'max_dec_layers': 3, 'is_max_bottleneck_transformer': False, 'max_msda': '', 'is_max_ms': True, 'max_ms_idxs': [-4, -3, -2], 'max_hidden_dim': 192, 'mw': 1.0, 'is_max_ds': True, 'is_masking': True, 'is_max_hungarian': True, 'num_queries': 20, 'is_max_cls': True, 'is_mhsa_float32': True}", + "net_conv_kernel_sizes": "[[3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3]]", + "net_num_pool_op_kernel_sizes": "[[2, 2, 2], [2, 2, 2], [2, 2, 2], [2, 2, 2], [2, 2, 2]]", + "net_pool_per_axis": "[5, 5, 5]", + "normalization_schemes": "OrderedDict([(0, 'nonCT'), (1, 'nonCT'), (2, 'nonCT'), (3, 'nonCT')])", + "num_batches_per_epoch": "250", + "num_classes": "3", + "num_input_channels": "4", + "num_val_batches_per_epoch": "50", + "online_eval_fn": "[]", + "online_eval_foreground_dc": "[]", + "online_eval_fp": "[]", + "online_eval_tp": "[]", + "only_keep_largest_connected_component": "None", + "optim_name": "adamw", + "optimizer": "AdamW (\nParameter Group 0\n amsgrad: False\n betas: (0.9, 0.999)\n eps: 1e-08\n initial_lr: 0.0003\n lr: 0.0\n weight_decay: 3e-05\n)", + "output_folder": "/opt/tiger/project/data/nnUNet_trained_models/UNet_IN_NANFang/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_2", + "output_folder_base": "/opt/tiger/project/data/nnUNet_trained_models/UNet_IN_NANFang/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250", + "oversample_foreground_percent": "0.0", + "pad_all_sides": "None", + "patch_size": "[128 128 128]", + "patience": "50", + "pin_memory": "True", + "plans_file": "/opt/tiger/project/data/nnUNet_preprocessed/Task500_BraTS2021/nnUNetPlansv2.1_plans_3D.pkl", + "reclip": "None", + "regions": "{'whole tumor': (1, 2, 3), 'tumor core': (2, 3), 'enhancing tumor': (3,)}", + "regions_class_order": "(1, 2, 3)", + "resume": "auto", + "save_best_checkpoint": "True", + "save_every": "1", + "save_final_checkpoint": "True", + "save_intermediate_checkpoints": "True", + "save_latest_only": "True", + "stage": "0", + "threeD": "True", + "tr_gen": "", + "train_loss_MA": "None", + "train_loss_MA_alpha": "0.93", + "train_loss_MA_eps": "0.0005", + "transpose_backward": "[0, 1, 2]", + "transpose_forward": "[0, 1, 2]", + "unpack_data": "True", + "use_mask_for_norm": "OrderedDict([(0, True), (1, True), (2, True), (3, True)])", + "use_progress_bar": "False", + "val_eval_criterion_MA": "None", + "val_eval_criterion_alpha": "0.9", + "val_gen": "", + "warmup_epochs": "10", + "was_initialized": "True", + "weight_decay": "3e-05" +} \ No newline at end of file diff --git a/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_2/model_best.model b/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_2/model_best.model new file mode 100644 index 0000000000000000000000000000000000000000..f4ef9bceaa7fe5f4b781f35a045d868d30644b7f --- /dev/null +++ b/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_2/model_best.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1e525f09c2ee38a4838904fcdf8c859f3dc688b6e0153a00e0fc5c691891127f +size 409149664 diff --git a/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_2/model_best.model.pkl b/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_2/model_best.model.pkl new file mode 100644 index 0000000000000000000000000000000000000000..e34407987f43a84196765b953d03a988bc0b14e6 --- /dev/null +++ b/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_2/model_best.model.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:84e3a5584ba24abb2ea30e699bcdf195e1a139e05c344332c6a72805728baf9d +size 393843 diff --git a/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_2/model_final_checkpoint.model b/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_2/model_final_checkpoint.model new file mode 100644 index 0000000000000000000000000000000000000000..310d6787be47f6f5ffbc1bc5e943f1050bc0a6ea --- /dev/null +++ b/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_2/model_final_checkpoint.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:96bdde5c89f287f088ebce7f59118f80e33dc8e32d47f7667781d74297e4e34e +size 409163745 diff --git a/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_2/model_final_checkpoint.model.pkl b/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_2/model_final_checkpoint.model.pkl new file mode 100644 index 0000000000000000000000000000000000000000..e34407987f43a84196765b953d03a988bc0b14e6 --- /dev/null +++ b/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_2/model_final_checkpoint.model.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:84e3a5584ba24abb2ea30e699bcdf195e1a139e05c344332c6a72805728baf9d +size 393843 diff --git a/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_2/model_latest.model b/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_2/model_latest.model new file mode 100644 index 0000000000000000000000000000000000000000..e0b5b6401cc6058c8ef2aaaf1c545863216618e9 --- /dev/null +++ b/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_2/model_latest.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:40263def2fd5181c28fc6a96e6eaf3a3a72f296b80d0d9f3ec87ad97c10b9137 +size 409163745 diff --git a/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_2/model_latest.model.pkl b/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_2/model_latest.model.pkl new file mode 100644 index 0000000000000000000000000000000000000000..e34407987f43a84196765b953d03a988bc0b14e6 --- /dev/null +++ b/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_2/model_latest.model.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:84e3a5584ba24abb2ea30e699bcdf195e1a139e05c344332c6a72805728baf9d +size 393843 diff --git a/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_2/progress.png b/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_2/progress.png new file mode 100644 index 0000000000000000000000000000000000000000..c9e240bd8b2f3c743eb844bc0329c229de981b6f --- /dev/null +++ b/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_2/progress.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:482a09bcaf412ed069a34a6f3cb32afecda05c5e63ca2d8624fe61fa0310f05b +size 318916 diff --git a/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_2/training_log_2022_5_14_04_20_21.txt b/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_2/training_log_2022_5_14_04_20_21.txt new file mode 100644 index 0000000000000000000000000000000000000000..2a03bc58da69ebe97156e83673d3db1d5dae180e --- /dev/null +++ b/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_2/training_log_2022_5_14_04_20_21.txt @@ -0,0 +1,3404 @@ +Starting... +2022-05-14 04:20:21.457793: Using splits from existing split file: /opt/tiger/project/data/nnUNet_preprocessed/Task500_BraTS2021/splits_final.pkl +2022-05-14 04:20:21.458345: The split file contains 5 splits. +2022-05-14 04:20:21.458398: Desired fold for training: 2 +2022-05-14 04:20:21.458445: This split has 1001 training and 250 validation cases. +2022-05-14 04:21:47.278324: TRAINING KEYS: + odict_keys(['BraTS2021_00000', 'BraTS2021_00002', 'BraTS2021_00003', 'BraTS2021_00005', 'BraTS2021_00006', 'BraTS2021_00008', 'BraTS2021_00009', 'BraTS2021_00011', 'BraTS2021_00012', 'BraTS2021_00014', 'BraTS2021_00016', 'BraTS2021_00018', 'BraTS2021_00019', 'BraTS2021_00020', 'BraTS2021_00021', 'BraTS2021_00022', 'BraTS2021_00024', 'BraTS2021_00025', 'BraTS2021_00026', 'BraTS2021_00028', 'BraTS2021_00031', 'BraTS2021_00032', 'BraTS2021_00033', 'BraTS2021_00035', 'BraTS2021_00036', 'BraTS2021_00044', 'BraTS2021_00045', 'BraTS2021_00046', 'BraTS2021_00051', 'BraTS2021_00052', 'BraTS2021_00053', 'BraTS2021_00054', 'BraTS2021_00058', 'BraTS2021_00059', 'BraTS2021_00060', 'BraTS2021_00061', 'BraTS2021_00062', 'BraTS2021_00063', 'BraTS2021_00064', 'BraTS2021_00066', 'BraTS2021_00070', 'BraTS2021_00072', 'BraTS2021_00074', 'BraTS2021_00077', 'BraTS2021_00078', 'BraTS2021_00081', 'BraTS2021_00084', 'BraTS2021_00085', 'BraTS2021_00087', 'BraTS2021_00088', 'BraTS2021_00089', 'BraTS2021_00090', 'BraTS2021_00095', 'BraTS2021_00097', 'BraTS2021_00099', 'BraTS2021_00101', 'BraTS2021_00102', 'BraTS2021_00103', 'BraTS2021_00104', 'BraTS2021_00105', 'BraTS2021_00106', 'BraTS2021_00107', 'BraTS2021_00108', 'BraTS2021_00109', 'BraTS2021_00110', 'BraTS2021_00111', 'BraTS2021_00112', 'BraTS2021_00113', 'BraTS2021_00115', 'BraTS2021_00116', 'BraTS2021_00117', 'BraTS2021_00118', 'BraTS2021_00120', 'BraTS2021_00122', 'BraTS2021_00123', 'BraTS2021_00124', 'BraTS2021_00126', 'BraTS2021_00127', 'BraTS2021_00128', 'BraTS2021_00130', 'BraTS2021_00131', 'BraTS2021_00132', 'BraTS2021_00133', 'BraTS2021_00134', 'BraTS2021_00136', 'BraTS2021_00137', 'BraTS2021_00140', 'BraTS2021_00143', 'BraTS2021_00144', 'BraTS2021_00146', 'BraTS2021_00147', 'BraTS2021_00148', 'BraTS2021_00149', 'BraTS2021_00152', 'BraTS2021_00154', 'BraTS2021_00157', 'BraTS2021_00158', 'BraTS2021_00159', 'BraTS2021_00160', 'BraTS2021_00162', 'BraTS2021_00165', 'BraTS2021_00166', 'BraTS2021_00167', 'BraTS2021_00170', 'BraTS2021_00171', 'BraTS2021_00177', 'BraTS2021_00178', 'BraTS2021_00183', 'BraTS2021_00184', 'BraTS2021_00185', 'BraTS2021_00186', 'BraTS2021_00188', 'BraTS2021_00191', 'BraTS2021_00192', 'BraTS2021_00193', 'BraTS2021_00194', 'BraTS2021_00196', 'BraTS2021_00199', 'BraTS2021_00201', 'BraTS2021_00204', 'BraTS2021_00206', 'BraTS2021_00207', 'BraTS2021_00210', 'BraTS2021_00211', 'BraTS2021_00212', 'BraTS2021_00214', 'BraTS2021_00216', 'BraTS2021_00217', 'BraTS2021_00218', 'BraTS2021_00219', 'BraTS2021_00220', 'BraTS2021_00222', 'BraTS2021_00228', 'BraTS2021_00230', 'BraTS2021_00231', 'BraTS2021_00233', 'BraTS2021_00234', 'BraTS2021_00236', 'BraTS2021_00237', 'BraTS2021_00239', 'BraTS2021_00240', 'BraTS2021_00241', 'BraTS2021_00242', 'BraTS2021_00243', 'BraTS2021_00246', 'BraTS2021_00247', 'BraTS2021_00249', 'BraTS2021_00251', 'BraTS2021_00258', 'BraTS2021_00259', 'BraTS2021_00260', 'BraTS2021_00262', 'BraTS2021_00263', 'BraTS2021_00266', 'BraTS2021_00267', 'BraTS2021_00269', 'BraTS2021_00270', 'BraTS2021_00274', 'BraTS2021_00275', 'BraTS2021_00281', 'BraTS2021_00282', 'BraTS2021_00283', 'BraTS2021_00286', 'BraTS2021_00289', 'BraTS2021_00290', 'BraTS2021_00291', 'BraTS2021_00292', 'BraTS2021_00293', 'BraTS2021_00294', 'BraTS2021_00296', 'BraTS2021_00297', 'BraTS2021_00298', 'BraTS2021_00299', 'BraTS2021_00301', 'BraTS2021_00303', 'BraTS2021_00304', 'BraTS2021_00306', 'BraTS2021_00309', 'BraTS2021_00310', 'BraTS2021_00311', 'BraTS2021_00312', 'BraTS2021_00313', 'BraTS2021_00314', 'BraTS2021_00316', 'BraTS2021_00317', 'BraTS2021_00318', 'BraTS2021_00320', 'BraTS2021_00321', 'BraTS2021_00322', 'BraTS2021_00324', 'BraTS2021_00325', 'BraTS2021_00327', 'BraTS2021_00328', 'BraTS2021_00329', 'BraTS2021_00331', 'BraTS2021_00332', 'BraTS2021_00334', 'BraTS2021_00336', 'BraTS2021_00339', 'BraTS2021_00341', 'BraTS2021_00343', 'BraTS2021_00344', 'BraTS2021_00346', 'BraTS2021_00347', 'BraTS2021_00348', 'BraTS2021_00350', 'BraTS2021_00351', 'BraTS2021_00352', 'BraTS2021_00353', 'BraTS2021_00359', 'BraTS2021_00360', 'BraTS2021_00366', 'BraTS2021_00367', 'BraTS2021_00369', 'BraTS2021_00370', 'BraTS2021_00373', 'BraTS2021_00375', 'BraTS2021_00376', 'BraTS2021_00377', 'BraTS2021_00378', 'BraTS2021_00379', 'BraTS2021_00380', 'BraTS2021_00383', 'BraTS2021_00386', 'BraTS2021_00387', 'BraTS2021_00388', 'BraTS2021_00390', 'BraTS2021_00391', 'BraTS2021_00392', 'BraTS2021_00395', 'BraTS2021_00397', 'BraTS2021_00399', 'BraTS2021_00400', 'BraTS2021_00401', 'BraTS2021_00402', 'BraTS2021_00406', 'BraTS2021_00409', 'BraTS2021_00413', 'BraTS2021_00414', 'BraTS2021_00417', 'BraTS2021_00418', 'BraTS2021_00419', 'BraTS2021_00421', 'BraTS2021_00423', 'BraTS2021_00425', 'BraTS2021_00426', 'BraTS2021_00429', 'BraTS2021_00430', 'BraTS2021_00432', 'BraTS2021_00433', 'BraTS2021_00436', 'BraTS2021_00440', 'BraTS2021_00442', 'BraTS2021_00443', 'BraTS2021_00444', 'BraTS2021_00445', 'BraTS2021_00448', 'BraTS2021_00449', 'BraTS2021_00451', 'BraTS2021_00452', 'BraTS2021_00453', 'BraTS2021_00454', 'BraTS2021_00455', 'BraTS2021_00457', 'BraTS2021_00459', 'BraTS2021_00464', 'BraTS2021_00466', 'BraTS2021_00468', 'BraTS2021_00469', 'BraTS2021_00477', 'BraTS2021_00478', 'BraTS2021_00480', 'BraTS2021_00481', 'BraTS2021_00483', 'BraTS2021_00485', 'BraTS2021_00488', 'BraTS2021_00491', 'BraTS2021_00493', 'BraTS2021_00494', 'BraTS2021_00495', 'BraTS2021_00496', 'BraTS2021_00498', 'BraTS2021_00500', 'BraTS2021_00501', 'BraTS2021_00502', 'BraTS2021_00505', 'BraTS2021_00506', 'BraTS2021_00507', 'BraTS2021_00510', 'BraTS2021_00511', 'BraTS2021_00514', 'BraTS2021_00516', 'BraTS2021_00517', 'BraTS2021_00518', 'BraTS2021_00519', 'BraTS2021_00520', 'BraTS2021_00523', 'BraTS2021_00525', 'BraTS2021_00528', 'BraTS2021_00529', 'BraTS2021_00532', 'BraTS2021_00533', 'BraTS2021_00537', 'BraTS2021_00543', 'BraTS2021_00544', 'BraTS2021_00545', 'BraTS2021_00547', 'BraTS2021_00548', 'BraTS2021_00549', 'BraTS2021_00552', 'BraTS2021_00554', 'BraTS2021_00555', 'BraTS2021_00556', 'BraTS2021_00557', 'BraTS2021_00558', 'BraTS2021_00559', 'BraTS2021_00561', 'BraTS2021_00563', 'BraTS2021_00565', 'BraTS2021_00567', 'BraTS2021_00568', 'BraTS2021_00569', 'BraTS2021_00570', 'BraTS2021_00571', 'BraTS2021_00572', 'BraTS2021_00574', 'BraTS2021_00575', 'BraTS2021_00576', 'BraTS2021_00577', 'BraTS2021_00578', 'BraTS2021_00579', 'BraTS2021_00580', 'BraTS2021_00581', 'BraTS2021_00582', 'BraTS2021_00583', 'BraTS2021_00586', 'BraTS2021_00587', 'BraTS2021_00590', 'BraTS2021_00591', 'BraTS2021_00593', 'BraTS2021_00594', 'BraTS2021_00596', 'BraTS2021_00597', 'BraTS2021_00598', 'BraTS2021_00599', 'BraTS2021_00601', 'BraTS2021_00602', 'BraTS2021_00604', 'BraTS2021_00605', 'BraTS2021_00606', 'BraTS2021_00607', 'BraTS2021_00608', 'BraTS2021_00610', 'BraTS2021_00611', 'BraTS2021_00612', 'BraTS2021_00615', 'BraTS2021_00616', 'BraTS2021_00618', 'BraTS2021_00619', 'BraTS2021_00621', 'BraTS2021_00622', 'BraTS2021_00623', 'BraTS2021_00624', 'BraTS2021_00628', 'BraTS2021_00630', 'BraTS2021_00631', 'BraTS2021_00636', 'BraTS2021_00639', 'BraTS2021_00640', 'BraTS2021_00641', 'BraTS2021_00642', 'BraTS2021_00645', 'BraTS2021_00646', 'BraTS2021_00650', 'BraTS2021_00652', 'BraTS2021_00654', 'BraTS2021_00655', 'BraTS2021_00656', 'BraTS2021_00657', 'BraTS2021_00658', 'BraTS2021_00661', 'BraTS2021_00663', 'BraTS2021_00667', 'BraTS2021_00668', 'BraTS2021_00675', 'BraTS2021_00677', 'BraTS2021_00679', 'BraTS2021_00682', 'BraTS2021_00684', 'BraTS2021_00686', 'BraTS2021_00688', 'BraTS2021_00690', 'BraTS2021_00691', 'BraTS2021_00692', 'BraTS2021_00693', 'BraTS2021_00694', 'BraTS2021_00698', 'BraTS2021_00703', 'BraTS2021_00704', 'BraTS2021_00705', 'BraTS2021_00707', 'BraTS2021_00708', 'BraTS2021_00709', 'BraTS2021_00714', 'BraTS2021_00715', 'BraTS2021_00716', 'BraTS2021_00718', 'BraTS2021_00724', 'BraTS2021_00725', 'BraTS2021_00727', 'BraTS2021_00730', 'BraTS2021_00731', 'BraTS2021_00732', 'BraTS2021_00733', 'BraTS2021_00734', 'BraTS2021_00735', 'BraTS2021_00736', 'BraTS2021_00739', 'BraTS2021_00740', 'BraTS2021_00744', 'BraTS2021_00746', 'BraTS2021_00750', 'BraTS2021_00751', 'BraTS2021_00753', 'BraTS2021_00757', 'BraTS2021_00758', 'BraTS2021_00759', 'BraTS2021_00760', 'BraTS2021_00765', 'BraTS2021_00768', 'BraTS2021_00772', 'BraTS2021_00773', 'BraTS2021_00775', 'BraTS2021_00777', 'BraTS2021_00778', 'BraTS2021_00780', 'BraTS2021_00781', 'BraTS2021_00782', 'BraTS2021_00784', 'BraTS2021_00787', 'BraTS2021_00788', 'BraTS2021_00789', 'BraTS2021_00791', 'BraTS2021_00792', 'BraTS2021_00793', 'BraTS2021_00795', 'BraTS2021_00796', 'BraTS2021_00800', 'BraTS2021_00803', 'BraTS2021_00805', 'BraTS2021_00806', 'BraTS2021_00808', 'BraTS2021_00809', 'BraTS2021_00810', 'BraTS2021_00811', 'BraTS2021_00814', 'BraTS2021_00818', 'BraTS2021_00819', 'BraTS2021_00820', 'BraTS2021_00823', 'BraTS2021_00824', 'BraTS2021_00830', 'BraTS2021_00831', 'BraTS2021_00834', 'BraTS2021_00836', 'BraTS2021_00837', 'BraTS2021_00838', 'BraTS2021_00839', 'BraTS2021_00840', 'BraTS2021_00999', 'BraTS2021_01000', 'BraTS2021_01001', 'BraTS2021_01002', 'BraTS2021_01003', 'BraTS2021_01004', 'BraTS2021_01005', 'BraTS2021_01008', 'BraTS2021_01009', 'BraTS2021_01010', 'BraTS2021_01011', 'BraTS2021_01013', 'BraTS2021_01016', 'BraTS2021_01018', 'BraTS2021_01019', 'BraTS2021_01020', 'BraTS2021_01021', 'BraTS2021_01023', 'BraTS2021_01024', 'BraTS2021_01026', 'BraTS2021_01027', 'BraTS2021_01028', 'BraTS2021_01029', 'BraTS2021_01030', 'BraTS2021_01031', 'BraTS2021_01032', 'BraTS2021_01033', 'BraTS2021_01034', 'BraTS2021_01035', 'BraTS2021_01036', 'BraTS2021_01037', 'BraTS2021_01038', 'BraTS2021_01039', 'BraTS2021_01040', 'BraTS2021_01041', 'BraTS2021_01042', 'BraTS2021_01043', 'BraTS2021_01045', 'BraTS2021_01046', 'BraTS2021_01047', 'BraTS2021_01048', 'BraTS2021_01050', 'BraTS2021_01051', 'BraTS2021_01052', 'BraTS2021_01053', 'BraTS2021_01054', 'BraTS2021_01055', 'BraTS2021_01056', 'BraTS2021_01058', 'BraTS2021_01059', 'BraTS2021_01060', 'BraTS2021_01061', 'BraTS2021_01063', 'BraTS2021_01065', 'BraTS2021_01066', 'BraTS2021_01067', 'BraTS2021_01068', 'BraTS2021_01069', 'BraTS2021_01070', 'BraTS2021_01072', 'BraTS2021_01073', 'BraTS2021_01076', 'BraTS2021_01077', 'BraTS2021_01079', 'BraTS2021_01081', 'BraTS2021_01084', 'BraTS2021_01085', 'BraTS2021_01086', 'BraTS2021_01087', 'BraTS2021_01088', 'BraTS2021_01089', 'BraTS2021_01090', 'BraTS2021_01092', 'BraTS2021_01094', 'BraTS2021_01095', 'BraTS2021_01096', 'BraTS2021_01097', 'BraTS2021_01098', 'BraTS2021_01099', 'BraTS2021_01100', 'BraTS2021_01101', 'BraTS2021_01102', 'BraTS2021_01104', 'BraTS2021_01106', 'BraTS2021_01107', 'BraTS2021_01109', 'BraTS2021_01110', 'BraTS2021_01111', 'BraTS2021_01112', 'BraTS2021_01114', 'BraTS2021_01115', 'BraTS2021_01117', 'BraTS2021_01118', 'BraTS2021_01119', 'BraTS2021_01120', 'BraTS2021_01123', 'BraTS2021_01124', 'BraTS2021_01125', 'BraTS2021_01128', 'BraTS2021_01129', 'BraTS2021_01130', 'BraTS2021_01132', 'BraTS2021_01133', 'BraTS2021_01135', 'BraTS2021_01136', 'BraTS2021_01137', 'BraTS2021_01139', 'BraTS2021_01140', 'BraTS2021_01141', 'BraTS2021_01142', 'BraTS2021_01143', 'BraTS2021_01144', 'BraTS2021_01145', 'BraTS2021_01146', 'BraTS2021_01147', 'BraTS2021_01148', 'BraTS2021_01151', 'BraTS2021_01152', 'BraTS2021_01153', 'BraTS2021_01155', 'BraTS2021_01157', 'BraTS2021_01158', 'BraTS2021_01159', 'BraTS2021_01160', 'BraTS2021_01161', 'BraTS2021_01162', 'BraTS2021_01163', 'BraTS2021_01164', 'BraTS2021_01165', 'BraTS2021_01166', 'BraTS2021_01167', 'BraTS2021_01168', 'BraTS2021_01169', 'BraTS2021_01170', 'BraTS2021_01171', 'BraTS2021_01172', 'BraTS2021_01173', 'BraTS2021_01174', 'BraTS2021_01175', 'BraTS2021_01176', 'BraTS2021_01178', 'BraTS2021_01179', 'BraTS2021_01180', 'BraTS2021_01181', 'BraTS2021_01183', 'BraTS2021_01184', 'BraTS2021_01185', 'BraTS2021_01186', 'BraTS2021_01187', 'BraTS2021_01188', 'BraTS2021_01189', 'BraTS2021_01190', 'BraTS2021_01191', 'BraTS2021_01193', 'BraTS2021_01194', 'BraTS2021_01195', 'BraTS2021_01196', 'BraTS2021_01197', 'BraTS2021_01198', 'BraTS2021_01199', 'BraTS2021_01200', 'BraTS2021_01204', 'BraTS2021_01205', 'BraTS2021_01206', 'BraTS2021_01207', 'BraTS2021_01208', 'BraTS2021_01209', 'BraTS2021_01210', 'BraTS2021_01211', 'BraTS2021_01212', 'BraTS2021_01213', 'BraTS2021_01214', 'BraTS2021_01215', 'BraTS2021_01216', 'BraTS2021_01217', 'BraTS2021_01218', 'BraTS2021_01219', 'BraTS2021_01220', 'BraTS2021_01221', 'BraTS2021_01223', 'BraTS2021_01224', 'BraTS2021_01225', 'BraTS2021_01227', 'BraTS2021_01228', 'BraTS2021_01229', 'BraTS2021_01231', 'BraTS2021_01232', 'BraTS2021_01233', 'BraTS2021_01234', 'BraTS2021_01235', 'BraTS2021_01237', 'BraTS2021_01238', 'BraTS2021_01239', 'BraTS2021_01240', 'BraTS2021_01241', 'BraTS2021_01243', 'BraTS2021_01244', 'BraTS2021_01245', 'BraTS2021_01246', 'BraTS2021_01247', 'BraTS2021_01248', 'BraTS2021_01249', 'BraTS2021_01250', 'BraTS2021_01251', 'BraTS2021_01252', 'BraTS2021_01254', 'BraTS2021_01255', 'BraTS2021_01257', 'BraTS2021_01258', 'BraTS2021_01259', 'BraTS2021_01260', 'BraTS2021_01261', 'BraTS2021_01262', 'BraTS2021_01263', 'BraTS2021_01264', 'BraTS2021_01265', 'BraTS2021_01266', 'BraTS2021_01267', 'BraTS2021_01269', 'BraTS2021_01270', 'BraTS2021_01271', 'BraTS2021_01272', 'BraTS2021_01273', 'BraTS2021_01274', 'BraTS2021_01275', 'BraTS2021_01276', 'BraTS2021_01277', 'BraTS2021_01279', 'BraTS2021_01281', 'BraTS2021_01282', 'BraTS2021_01283', 'BraTS2021_01284', 'BraTS2021_01285', 'BraTS2021_01286', 'BraTS2021_01287', 'BraTS2021_01288', 'BraTS2021_01289', 'BraTS2021_01292', 'BraTS2021_01293', 'BraTS2021_01294', 'BraTS2021_01295', 'BraTS2021_01296', 'BraTS2021_01297', 'BraTS2021_01298', 'BraTS2021_01299', 'BraTS2021_01300', 'BraTS2021_01301', 'BraTS2021_01302', 'BraTS2021_01303', 'BraTS2021_01304', 'BraTS2021_01305', 'BraTS2021_01306', 'BraTS2021_01307', 'BraTS2021_01308', 'BraTS2021_01309', 'BraTS2021_01310', 'BraTS2021_01311', 'BraTS2021_01312', 'BraTS2021_01313', 'BraTS2021_01316', 'BraTS2021_01318', 'BraTS2021_01319', 'BraTS2021_01320', 'BraTS2021_01323', 'BraTS2021_01324', 'BraTS2021_01325', 'BraTS2021_01326', 'BraTS2021_01327', 'BraTS2021_01328', 'BraTS2021_01329', 'BraTS2021_01331', 'BraTS2021_01333', 'BraTS2021_01335', 'BraTS2021_01336', 'BraTS2021_01337', 'BraTS2021_01338', 'BraTS2021_01339', 'BraTS2021_01340', 'BraTS2021_01341', 'BraTS2021_01342', 'BraTS2021_01343', 'BraTS2021_01344', 'BraTS2021_01345', 'BraTS2021_01346', 'BraTS2021_01347', 'BraTS2021_01348', 'BraTS2021_01349', 'BraTS2021_01350', 'BraTS2021_01351', 'BraTS2021_01352', 'BraTS2021_01353', 'BraTS2021_01354', 'BraTS2021_01355', 'BraTS2021_01356', 'BraTS2021_01357', 'BraTS2021_01359', 'BraTS2021_01360', 'BraTS2021_01361', 'BraTS2021_01362', 'BraTS2021_01363', 'BraTS2021_01364', 'BraTS2021_01365', 'BraTS2021_01367', 'BraTS2021_01368', 'BraTS2021_01369', 'BraTS2021_01370', 'BraTS2021_01375', 'BraTS2021_01376', 'BraTS2021_01377', 'BraTS2021_01381', 'BraTS2021_01382', 'BraTS2021_01386', 'BraTS2021_01387', 'BraTS2021_01388', 'BraTS2021_01389', 'BraTS2021_01390', 'BraTS2021_01391', 'BraTS2021_01392', 'BraTS2021_01393', 'BraTS2021_01394', 'BraTS2021_01397', 'BraTS2021_01398', 'BraTS2021_01399', 'BraTS2021_01400', 'BraTS2021_01401', 'BraTS2021_01402', 'BraTS2021_01404', 'BraTS2021_01408', 'BraTS2021_01410', 'BraTS2021_01411', 'BraTS2021_01412', 'BraTS2021_01421', 'BraTS2021_01422', 'BraTS2021_01423', 'BraTS2021_01424', 'BraTS2021_01425', 'BraTS2021_01426', 'BraTS2021_01427', 'BraTS2021_01428', 'BraTS2021_01429', 'BraTS2021_01430', 'BraTS2021_01432', 'BraTS2021_01433', 'BraTS2021_01434', 'BraTS2021_01435', 'BraTS2021_01437', 'BraTS2021_01438', 'BraTS2021_01439', 'BraTS2021_01440', 'BraTS2021_01442', 'BraTS2021_01443', 'BraTS2021_01445', 'BraTS2021_01446', 'BraTS2021_01447', 'BraTS2021_01450', 'BraTS2021_01451', 'BraTS2021_01452', 'BraTS2021_01453', 'BraTS2021_01454', 'BraTS2021_01455', 'BraTS2021_01457', 'BraTS2021_01458', 'BraTS2021_01460', 'BraTS2021_01461', 'BraTS2021_01462', 'BraTS2021_01463', 'BraTS2021_01465', 'BraTS2021_01466', 'BraTS2021_01467', 'BraTS2021_01468', 'BraTS2021_01469', 'BraTS2021_01471', 'BraTS2021_01472', 'BraTS2021_01474', 'BraTS2021_01475', 'BraTS2021_01476', 'BraTS2021_01477', 'BraTS2021_01479', 'BraTS2021_01480', 'BraTS2021_01481', 'BraTS2021_01482', 'BraTS2021_01483', 'BraTS2021_01485', 'BraTS2021_01486', 'BraTS2021_01487', 'BraTS2021_01488', 'BraTS2021_01489', 'BraTS2021_01490', 'BraTS2021_01491', 'BraTS2021_01492', 'BraTS2021_01494', 'BraTS2021_01495', 'BraTS2021_01496', 'BraTS2021_01497', 'BraTS2021_01498', 'BraTS2021_01499', 'BraTS2021_01500', 'BraTS2021_01501', 'BraTS2021_01502', 'BraTS2021_01503', 'BraTS2021_01504', 'BraTS2021_01505', 'BraTS2021_01506', 'BraTS2021_01507', 'BraTS2021_01508', 'BraTS2021_01509', 'BraTS2021_01510', 'BraTS2021_01512', 'BraTS2021_01513', 'BraTS2021_01514', 'BraTS2021_01515', 'BraTS2021_01516', 'BraTS2021_01517', 'BraTS2021_01518', 'BraTS2021_01520', 'BraTS2021_01521', 'BraTS2021_01522', 'BraTS2021_01523', 'BraTS2021_01524', 'BraTS2021_01525', 'BraTS2021_01526', 'BraTS2021_01528', 'BraTS2021_01529', 'BraTS2021_01530', 'BraTS2021_01531', 'BraTS2021_01532', 'BraTS2021_01536', 'BraTS2021_01538', 'BraTS2021_01539', 'BraTS2021_01540', 'BraTS2021_01541', 'BraTS2021_01542', 'BraTS2021_01544', 'BraTS2021_01546', 'BraTS2021_01547', 'BraTS2021_01548', 'BraTS2021_01550', 'BraTS2021_01551', 'BraTS2021_01552', 'BraTS2021_01553', 'BraTS2021_01554', 'BraTS2021_01555', 'BraTS2021_01557', 'BraTS2021_01558', 'BraTS2021_01559', 'BraTS2021_01560', 'BraTS2021_01562', 'BraTS2021_01563', 'BraTS2021_01564', 'BraTS2021_01565', 'BraTS2021_01566', 'BraTS2021_01567', 'BraTS2021_01569', 'BraTS2021_01571', 'BraTS2021_01572', 'BraTS2021_01573', 'BraTS2021_01575', 'BraTS2021_01576', 'BraTS2021_01577', 'BraTS2021_01578', 'BraTS2021_01579', 'BraTS2021_01580', 'BraTS2021_01581', 'BraTS2021_01582', 'BraTS2021_01583', 'BraTS2021_01584', 'BraTS2021_01585', 'BraTS2021_01586', 'BraTS2021_01587', 'BraTS2021_01589', 'BraTS2021_01590', 'BraTS2021_01591', 'BraTS2021_01592', 'BraTS2021_01593', 'BraTS2021_01594', 'BraTS2021_01595', 'BraTS2021_01596', 'BraTS2021_01597', 'BraTS2021_01599', 'BraTS2021_01601', 'BraTS2021_01602', 'BraTS2021_01603', 'BraTS2021_01604', 'BraTS2021_01605', 'BraTS2021_01606', 'BraTS2021_01607', 'BraTS2021_01608', 'BraTS2021_01609', 'BraTS2021_01610', 'BraTS2021_01611', 'BraTS2021_01613', 'BraTS2021_01614', 'BraTS2021_01615', 'BraTS2021_01617', 'BraTS2021_01618', 'BraTS2021_01619', 'BraTS2021_01620', 'BraTS2021_01622', 'BraTS2021_01624', 'BraTS2021_01625', 'BraTS2021_01626', 'BraTS2021_01627', 'BraTS2021_01630', 'BraTS2021_01631', 'BraTS2021_01632', 'BraTS2021_01633', 'BraTS2021_01634', 'BraTS2021_01635', 'BraTS2021_01636', 'BraTS2021_01637', 'BraTS2021_01639', 'BraTS2021_01640', 'BraTS2021_01641', 'BraTS2021_01642', 'BraTS2021_01644', 'BraTS2021_01646', 'BraTS2021_01647', 'BraTS2021_01648', 'BraTS2021_01649', 'BraTS2021_01650', 'BraTS2021_01651', 'BraTS2021_01652', 'BraTS2021_01653', 'BraTS2021_01654', 'BraTS2021_01655', 'BraTS2021_01658', 'BraTS2021_01660', 'BraTS2021_01661', 'BraTS2021_01662', 'BraTS2021_01663', 'BraTS2021_01664', 'BraTS2021_01666']) +2022-05-14 04:21:47.278523: VALIDATION KEYS: + odict_keys(['BraTS2021_00017', 'BraTS2021_00030', 'BraTS2021_00043', 'BraTS2021_00048', 'BraTS2021_00049', 'BraTS2021_00056', 'BraTS2021_00068', 'BraTS2021_00071', 'BraTS2021_00094', 'BraTS2021_00096', 'BraTS2021_00098', 'BraTS2021_00100', 'BraTS2021_00121', 'BraTS2021_00138', 'BraTS2021_00139', 'BraTS2021_00142', 'BraTS2021_00150', 'BraTS2021_00151', 'BraTS2021_00155', 'BraTS2021_00156', 'BraTS2021_00172', 'BraTS2021_00176', 'BraTS2021_00187', 'BraTS2021_00195', 'BraTS2021_00203', 'BraTS2021_00209', 'BraTS2021_00221', 'BraTS2021_00227', 'BraTS2021_00235', 'BraTS2021_00238', 'BraTS2021_00250', 'BraTS2021_00253', 'BraTS2021_00254', 'BraTS2021_00261', 'BraTS2021_00271', 'BraTS2021_00273', 'BraTS2021_00280', 'BraTS2021_00284', 'BraTS2021_00285', 'BraTS2021_00288', 'BraTS2021_00300', 'BraTS2021_00305', 'BraTS2021_00338', 'BraTS2021_00340', 'BraTS2021_00349', 'BraTS2021_00356', 'BraTS2021_00364', 'BraTS2021_00371', 'BraTS2021_00382', 'BraTS2021_00389', 'BraTS2021_00403', 'BraTS2021_00404', 'BraTS2021_00405', 'BraTS2021_00407', 'BraTS2021_00410', 'BraTS2021_00412', 'BraTS2021_00416', 'BraTS2021_00431', 'BraTS2021_00441', 'BraTS2021_00446', 'BraTS2021_00456', 'BraTS2021_00470', 'BraTS2021_00472', 'BraTS2021_00479', 'BraTS2021_00499', 'BraTS2021_00504', 'BraTS2021_00512', 'BraTS2021_00513', 'BraTS2021_00524', 'BraTS2021_00526', 'BraTS2021_00530', 'BraTS2021_00538', 'BraTS2021_00539', 'BraTS2021_00540', 'BraTS2021_00542', 'BraTS2021_00550', 'BraTS2021_00551', 'BraTS2021_00584', 'BraTS2021_00588', 'BraTS2021_00589', 'BraTS2021_00613', 'BraTS2021_00620', 'BraTS2021_00625', 'BraTS2021_00626', 'BraTS2021_00638', 'BraTS2021_00649', 'BraTS2021_00651', 'BraTS2021_00659', 'BraTS2021_00674', 'BraTS2021_00676', 'BraTS2021_00680', 'BraTS2021_00683', 'BraTS2021_00685', 'BraTS2021_00687', 'BraTS2021_00689', 'BraTS2021_00697', 'BraTS2021_00706', 'BraTS2021_00723', 'BraTS2021_00728', 'BraTS2021_00729', 'BraTS2021_00737', 'BraTS2021_00742', 'BraTS2021_00747', 'BraTS2021_00756', 'BraTS2021_00764', 'BraTS2021_00767', 'BraTS2021_00774', 'BraTS2021_00797', 'BraTS2021_00799', 'BraTS2021_00801', 'BraTS2021_00802', 'BraTS2021_00804', 'BraTS2021_00807', 'BraTS2021_00816', 'BraTS2021_00828', 'BraTS2021_01007', 'BraTS2021_01012', 'BraTS2021_01014', 'BraTS2021_01015', 'BraTS2021_01017', 'BraTS2021_01022', 'BraTS2021_01025', 'BraTS2021_01044', 'BraTS2021_01049', 'BraTS2021_01057', 'BraTS2021_01062', 'BraTS2021_01064', 'BraTS2021_01071', 'BraTS2021_01074', 'BraTS2021_01075', 'BraTS2021_01078', 'BraTS2021_01080', 'BraTS2021_01082', 'BraTS2021_01083', 'BraTS2021_01091', 'BraTS2021_01093', 'BraTS2021_01103', 'BraTS2021_01105', 'BraTS2021_01108', 'BraTS2021_01113', 'BraTS2021_01116', 'BraTS2021_01121', 'BraTS2021_01122', 'BraTS2021_01126', 'BraTS2021_01127', 'BraTS2021_01131', 'BraTS2021_01134', 'BraTS2021_01138', 'BraTS2021_01149', 'BraTS2021_01150', 'BraTS2021_01154', 'BraTS2021_01156', 'BraTS2021_01177', 'BraTS2021_01182', 'BraTS2021_01192', 'BraTS2021_01201', 'BraTS2021_01202', 'BraTS2021_01203', 'BraTS2021_01222', 'BraTS2021_01226', 'BraTS2021_01230', 'BraTS2021_01236', 'BraTS2021_01242', 'BraTS2021_01253', 'BraTS2021_01256', 'BraTS2021_01268', 'BraTS2021_01278', 'BraTS2021_01280', 'BraTS2021_01290', 'BraTS2021_01291', 'BraTS2021_01314', 'BraTS2021_01315', 'BraTS2021_01317', 'BraTS2021_01321', 'BraTS2021_01322', 'BraTS2021_01330', 'BraTS2021_01332', 'BraTS2021_01334', 'BraTS2021_01358', 'BraTS2021_01366', 'BraTS2021_01371', 'BraTS2021_01372', 'BraTS2021_01373', 'BraTS2021_01374', 'BraTS2021_01378', 'BraTS2021_01379', 'BraTS2021_01380', 'BraTS2021_01383', 'BraTS2021_01384', 'BraTS2021_01385', 'BraTS2021_01395', 'BraTS2021_01396', 'BraTS2021_01403', 'BraTS2021_01405', 'BraTS2021_01406', 'BraTS2021_01407', 'BraTS2021_01409', 'BraTS2021_01413', 'BraTS2021_01414', 'BraTS2021_01415', 'BraTS2021_01416', 'BraTS2021_01417', 'BraTS2021_01418', 'BraTS2021_01419', 'BraTS2021_01420', 'BraTS2021_01431', 'BraTS2021_01436', 'BraTS2021_01441', 'BraTS2021_01444', 'BraTS2021_01448', 'BraTS2021_01449', 'BraTS2021_01456', 'BraTS2021_01459', 'BraTS2021_01464', 'BraTS2021_01470', 'BraTS2021_01473', 'BraTS2021_01478', 'BraTS2021_01484', 'BraTS2021_01493', 'BraTS2021_01511', 'BraTS2021_01519', 'BraTS2021_01527', 'BraTS2021_01533', 'BraTS2021_01534', 'BraTS2021_01535', 'BraTS2021_01537', 'BraTS2021_01543', 'BraTS2021_01545', 'BraTS2021_01549', 'BraTS2021_01556', 'BraTS2021_01561', 'BraTS2021_01568', 'BraTS2021_01570', 'BraTS2021_01574', 'BraTS2021_01588', 'BraTS2021_01598', 'BraTS2021_01600', 'BraTS2021_01612', 'BraTS2021_01616', 'BraTS2021_01621', 'BraTS2021_01623', 'BraTS2021_01628', 'BraTS2021_01629', 'BraTS2021_01638', 'BraTS2021_01643', 'BraTS2021_01645', 'BraTS2021_01656', 'BraTS2021_01657', 'BraTS2021_01659', 'BraTS2021_01665']) +2022-05-14 04:21:50.630793: lr: 3.3e-05 +2022-05-14 04:22:06.443777: +epoch: 0 +2022-05-14 04:29:51.105087: train loss : 0.3123 +2022-05-14 04:30:16.354802: validation loss: 0.1550 +2022-05-14 04:30:16.355433: Average global foreground Dice: [0.8935, 0.8168, 0.7994] +2022-05-14 04:30:16.355539: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 04:30:16.686342: lr: 3.3e-05 +2022-05-14 04:30:16.686559: saving scheduled checkpoint file... +2022-05-14 04:30:16.733365: saving checkpoint... +2022-05-14 04:30:17.295859: done, saving took 0.61 seconds +2022-05-14 04:30:17.312503: done +2022-05-14 04:30:27.690610: This epoch took 501.246168 s + +2022-05-14 04:30:27.690885: +epoch: 1 +2022-05-14 04:37:45.188638: train loss : 0.1550 +2022-05-14 04:38:10.445534: validation loss: 0.1331 +2022-05-14 04:38:10.446271: Average global foreground Dice: [0.9068, 0.8598, 0.8445] +2022-05-14 04:38:10.446373: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 04:38:10.852355: lr: 6.7e-05 +2022-05-14 04:38:10.852555: saving scheduled checkpoint file... +2022-05-14 04:38:10.897147: saving checkpoint... +2022-05-14 04:38:11.564206: done, saving took 0.71 seconds +2022-05-14 04:38:11.578598: done +2022-05-14 04:38:11.598156: saving checkpoint... +2022-05-14 04:38:12.149095: done, saving took 0.57 seconds +2022-05-14 04:38:52.020569: This epoch took 504.329583 s + +2022-05-14 04:38:52.020832: +epoch: 2 +2022-05-14 04:46:09.748151: train loss : 0.1197 +2022-05-14 04:46:34.960407: validation loss: 0.0989 +2022-05-14 04:46:34.961028: Average global foreground Dice: [0.9159, 0.9, 0.8728] +2022-05-14 04:46:34.961113: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 04:46:35.357258: lr: 0.0001 +2022-05-14 04:46:35.357451: saving scheduled checkpoint file... +2022-05-14 04:46:35.402177: saving checkpoint... +2022-05-14 04:46:36.174928: done, saving took 0.82 seconds +2022-05-14 04:46:36.193054: done +2022-05-14 04:46:36.215031: saving checkpoint... +2022-05-14 04:46:36.955763: done, saving took 0.76 seconds +2022-05-14 04:47:10.025135: This epoch took 498.004184 s + +2022-05-14 04:47:10.025531: +epoch: 3 +2022-05-14 04:54:27.153553: train loss : 0.0982 +2022-05-14 04:54:52.269129: validation loss: 0.0917 +2022-05-14 04:54:52.269779: Average global foreground Dice: [0.9233, 0.9037, 0.8835] +2022-05-14 04:54:52.269868: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 04:54:52.690608: lr: 0.000133 +2022-05-14 04:54:52.690795: saving scheduled checkpoint file... +2022-05-14 04:54:52.734969: saving checkpoint... +2022-05-14 04:54:53.396805: done, saving took 0.71 seconds +2022-05-14 04:54:53.417359: done +2022-05-14 04:54:53.437708: saving checkpoint... +2022-05-14 04:54:54.130179: done, saving took 0.71 seconds +2022-05-14 04:55:14.182853: This epoch took 484.156980 s + +2022-05-14 04:55:14.183218: +epoch: 4 +2022-05-14 05:02:31.002375: train loss : 0.0915 +2022-05-14 05:02:56.207330: validation loss: 0.0791 +2022-05-14 05:02:56.207963: Average global foreground Dice: [0.9084, 0.8922, 0.88] +2022-05-14 05:02:56.208053: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 05:02:56.696564: lr: 0.000167 +2022-05-14 05:02:56.696754: saving scheduled checkpoint file... +2022-05-14 05:02:56.740694: saving checkpoint... +2022-05-14 05:02:57.451148: done, saving took 0.75 seconds +2022-05-14 05:02:57.467387: done +2022-05-14 05:02:57.486226: saving checkpoint... +2022-05-14 05:02:58.161823: done, saving took 0.69 seconds +2022-05-14 05:03:24.783058: This epoch took 490.599720 s + +2022-05-14 05:03:24.783346: +epoch: 5 +2022-05-14 05:10:42.434385: train loss : 0.0854 +2022-05-14 05:11:07.753999: validation loss: 0.0657 +2022-05-14 05:11:07.754852: Average global foreground Dice: [0.9354, 0.9196, 0.8947] +2022-05-14 05:11:07.754941: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 05:11:08.244263: lr: 0.0002 +2022-05-14 05:11:08.244452: saving scheduled checkpoint file... +2022-05-14 05:11:08.288821: saving checkpoint... +2022-05-14 05:11:09.010383: done, saving took 0.77 seconds +2022-05-14 05:11:09.038045: done +2022-05-14 05:11:09.056962: saving checkpoint... +2022-05-14 05:11:09.756629: done, saving took 0.72 seconds +2022-05-14 05:11:28.291045: This epoch took 483.507597 s + +2022-05-14 05:11:28.291318: +epoch: 6 +2022-05-14 05:18:46.604656: train loss : 0.0730 +2022-05-14 05:19:11.911774: validation loss: 0.0875 +2022-05-14 05:19:11.912494: Average global foreground Dice: [0.9286, 0.9138, 0.8961] +2022-05-14 05:19:11.912602: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 05:19:12.459316: lr: 0.000233 +2022-05-14 05:19:12.459512: saving scheduled checkpoint file... +2022-05-14 05:19:12.508629: saving checkpoint... +2022-05-14 05:19:13.291546: done, saving took 0.83 seconds +2022-05-14 05:19:13.315956: done +2022-05-14 05:19:13.336410: saving checkpoint... +2022-05-14 05:19:14.050605: done, saving took 0.73 seconds +2022-05-14 05:19:33.767999: This epoch took 485.476563 s + +2022-05-14 05:19:33.768325: +epoch: 7 +2022-05-14 05:26:51.691021: train loss : 0.0751 +2022-05-14 05:27:16.835343: validation loss: 0.0707 +2022-05-14 05:27:16.836025: Average global foreground Dice: [0.9264, 0.921, 0.8994] +2022-05-14 05:27:16.836114: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 05:27:17.247773: lr: 0.000267 +2022-05-14 05:27:17.247975: saving scheduled checkpoint file... +2022-05-14 05:27:17.293045: saving checkpoint... +2022-05-14 05:27:18.043839: done, saving took 0.80 seconds +2022-05-14 05:27:18.064168: done +2022-05-14 05:27:18.095063: saving checkpoint... +2022-05-14 05:27:18.779979: done, saving took 0.72 seconds +2022-05-14 05:27:41.224257: This epoch took 487.455826 s + +2022-05-14 05:27:41.224556: +epoch: 8 +2022-05-14 05:34:59.160678: train loss : 0.0762 +2022-05-14 05:35:24.299977: validation loss: 0.0721 +2022-05-14 05:35:24.300630: Average global foreground Dice: [0.9351, 0.9174, 0.8961] +2022-05-14 05:35:24.300729: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 05:35:24.706328: lr: 0.0003 +2022-05-14 05:35:24.706528: saving scheduled checkpoint file... +2022-05-14 05:35:24.750834: saving checkpoint... +2022-05-14 05:35:25.561401: done, saving took 0.85 seconds +2022-05-14 05:35:25.585778: done +2022-05-14 05:35:25.624191: saving checkpoint... +2022-05-14 05:35:26.393260: done, saving took 0.81 seconds +2022-05-14 05:35:46.453030: This epoch took 485.228371 s + +2022-05-14 05:35:46.453322: +epoch: 9 +2022-05-14 05:43:04.263839: train loss : 0.0706 +2022-05-14 05:43:29.459134: validation loss: 0.0747 +2022-05-14 05:43:29.459833: Average global foreground Dice: [0.9399, 0.9205, 0.9042] +2022-05-14 05:43:29.459920: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 05:43:29.878779: lr: 0.0003 +2022-05-14 05:43:29.878978: saving scheduled checkpoint file... +2022-05-14 05:43:29.923906: saving checkpoint... +2022-05-14 05:43:30.704371: done, saving took 0.83 seconds +2022-05-14 05:43:30.722572: done +2022-05-14 05:43:30.745400: saving checkpoint... +2022-05-14 05:43:31.522605: done, saving took 0.80 seconds +2022-05-14 05:43:50.466041: This epoch took 484.012617 s + +2022-05-14 05:43:50.466307: +epoch: 10 +2022-05-14 05:51:08.526568: train loss : 0.0682 +2022-05-14 05:51:33.756546: validation loss: 0.0672 +2022-05-14 05:51:33.757250: Average global foreground Dice: [0.9412, 0.93, 0.9059] +2022-05-14 05:51:33.757336: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 05:51:34.151734: lr: 0.0003 +2022-05-14 05:51:34.151919: saving scheduled checkpoint file... +2022-05-14 05:51:34.200271: saving checkpoint... +2022-05-14 05:51:34.990527: done, saving took 0.84 seconds +2022-05-14 05:51:35.009676: done +2022-05-14 05:51:35.034101: saving checkpoint... +2022-05-14 05:51:35.815947: done, saving took 0.81 seconds +2022-05-14 05:51:56.017187: This epoch took 485.550753 s + +2022-05-14 05:51:56.017757: +epoch: 11 +2022-05-14 05:59:13.565644: train loss : 0.0690 +2022-05-14 05:59:38.878569: validation loss: 0.0646 +2022-05-14 05:59:38.879240: Average global foreground Dice: [0.9358, 0.9135, 0.9133] +2022-05-14 05:59:38.879342: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 05:59:39.272202: lr: 0.0003 +2022-05-14 05:59:39.272397: saving scheduled checkpoint file... +2022-05-14 05:59:39.316703: saving checkpoint... +2022-05-14 05:59:40.113891: done, saving took 0.84 seconds +2022-05-14 05:59:40.133309: done +2022-05-14 05:59:40.157157: saving checkpoint... +2022-05-14 05:59:40.938964: done, saving took 0.81 seconds +2022-05-14 05:59:56.567837: This epoch took 480.549898 s + +2022-05-14 05:59:56.568128: +epoch: 12 +2022-05-14 06:07:13.596488: train loss : 0.0676 +2022-05-14 06:07:38.888342: validation loss: 0.0529 +2022-05-14 06:07:38.889016: Average global foreground Dice: [0.9374, 0.9348, 0.9136] +2022-05-14 06:07:38.889102: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 06:07:39.285533: lr: 0.0003 +2022-05-14 06:07:39.285731: saving scheduled checkpoint file... +2022-05-14 06:07:39.329967: saving checkpoint... +2022-05-14 06:07:40.097272: done, saving took 0.81 seconds +2022-05-14 06:07:40.117486: done +2022-05-14 06:07:40.140525: saving checkpoint... +2022-05-14 06:07:40.892858: done, saving took 0.78 seconds +2022-05-14 06:08:09.748779: This epoch took 493.180549 s + +2022-05-14 06:08:09.749069: +epoch: 13 +2022-05-14 06:15:27.670892: train loss : 0.0616 +2022-05-14 06:15:52.864700: validation loss: 0.0745 +2022-05-14 06:15:52.865365: Average global foreground Dice: [0.9328, 0.9262, 0.9058] +2022-05-14 06:15:52.865465: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 06:15:53.261280: lr: 0.0003 +2022-05-14 06:15:53.261482: saving scheduled checkpoint file... +2022-05-14 06:15:53.306445: saving checkpoint... +2022-05-14 06:15:54.082147: done, saving took 0.82 seconds +2022-05-14 06:15:54.103381: done +2022-05-14 06:15:54.126430: saving checkpoint... +2022-05-14 06:15:54.834848: done, saving took 0.73 seconds +2022-05-14 06:16:14.506811: This epoch took 484.757641 s + +2022-05-14 06:16:14.507070: +epoch: 14 +2022-05-14 06:23:32.267543: train loss : 0.0674 +2022-05-14 06:23:57.525159: validation loss: 0.0610 +2022-05-14 06:23:57.525823: Average global foreground Dice: [0.9403, 0.9364, 0.9111] +2022-05-14 06:23:57.525912: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 06:23:57.923553: lr: 0.0003 +2022-05-14 06:23:57.923744: saving scheduled checkpoint file... +2022-05-14 06:23:57.968284: saving checkpoint... +2022-05-14 06:23:58.760325: done, saving took 0.84 seconds +2022-05-14 06:23:58.776900: done +2022-05-14 06:23:58.796637: saving checkpoint... +2022-05-14 06:23:59.552003: done, saving took 0.77 seconds +2022-05-14 06:24:21.405722: This epoch took 486.898551 s + +2022-05-14 06:24:21.406009: +epoch: 15 +2022-05-14 06:31:39.516253: train loss : 0.0647 +2022-05-14 06:32:04.882635: validation loss: 0.0562 +2022-05-14 06:32:04.883292: Average global foreground Dice: [0.941, 0.9343, 0.9122] +2022-05-14 06:32:04.883380: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 06:32:05.282660: lr: 0.0003 +2022-05-14 06:32:05.282854: saving scheduled checkpoint file... +2022-05-14 06:32:05.327546: saving checkpoint... +2022-05-14 06:32:06.098878: done, saving took 0.82 seconds +2022-05-14 06:32:06.119246: done +2022-05-14 06:32:06.142547: saving checkpoint... +2022-05-14 06:32:06.903380: done, saving took 0.78 seconds +2022-05-14 06:32:30.297050: This epoch took 488.890918 s + +2022-05-14 06:32:30.297426: +epoch: 16 +2022-05-14 06:39:48.449329: train loss : 0.0600 +2022-05-14 06:40:13.779921: validation loss: 0.0549 +2022-05-14 06:40:13.780614: Average global foreground Dice: [0.942, 0.9318, 0.9162] +2022-05-14 06:40:13.780705: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 06:40:14.190696: lr: 0.000299 +2022-05-14 06:40:14.190901: saving scheduled checkpoint file... +2022-05-14 06:40:14.235570: saving checkpoint... +2022-05-14 06:40:15.019228: done, saving took 0.83 seconds +2022-05-14 06:40:15.043467: done +2022-05-14 06:40:15.071834: saving checkpoint... +2022-05-14 06:40:15.703886: done, saving took 0.66 seconds +2022-05-14 06:40:40.124307: This epoch took 489.826762 s + +2022-05-14 06:40:40.124585: +epoch: 17 +2022-05-14 06:47:58.358447: train loss : 0.0587 +2022-05-14 06:48:23.575565: validation loss: 0.0554 +2022-05-14 06:48:23.576154: Average global foreground Dice: [0.9442, 0.9293, 0.9046] +2022-05-14 06:48:23.576246: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 06:48:23.980468: lr: 0.000299 +2022-05-14 06:48:23.980656: saving scheduled checkpoint file... +2022-05-14 06:48:24.024868: saving checkpoint... +2022-05-14 06:48:24.797296: done, saving took 0.82 seconds +2022-05-14 06:48:24.816319: done +2022-05-14 06:48:24.838985: saving checkpoint... +2022-05-14 06:48:25.599463: done, saving took 0.78 seconds +2022-05-14 06:48:42.948003: This epoch took 482.823315 s + +2022-05-14 06:48:42.948264: +epoch: 18 +2022-05-14 06:56:00.930721: train loss : 0.0613 +2022-05-14 06:56:26.211843: validation loss: 0.0523 +2022-05-14 06:56:26.212469: Average global foreground Dice: [0.938, 0.9331, 0.9199] +2022-05-14 06:56:26.212558: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 06:56:26.612805: lr: 0.000299 +2022-05-14 06:56:26.612994: saving scheduled checkpoint file... +2022-05-14 06:56:26.657016: saving checkpoint... +2022-05-14 06:56:27.446667: done, saving took 0.83 seconds +2022-05-14 06:56:27.464992: done +2022-05-14 06:56:27.487722: saving checkpoint... +2022-05-14 06:56:28.256984: done, saving took 0.79 seconds +2022-05-14 06:56:48.199415: This epoch took 485.251052 s + +2022-05-14 06:56:48.199683: +epoch: 19 +2022-05-14 07:04:06.670077: train loss : 0.0597 +2022-05-14 07:04:31.938657: validation loss: 0.0533 +2022-05-14 07:04:31.939317: Average global foreground Dice: [0.9418, 0.9489, 0.9212] +2022-05-14 07:04:31.939407: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 07:04:32.344128: lr: 0.000299 +2022-05-14 07:04:32.344314: saving scheduled checkpoint file... +2022-05-14 07:04:32.389573: saving checkpoint... +2022-05-14 07:04:33.192377: done, saving took 0.85 seconds +2022-05-14 07:04:33.209726: done +2022-05-14 07:04:33.230207: saving checkpoint... +2022-05-14 07:04:34.008353: done, saving took 0.80 seconds +2022-05-14 07:05:04.136641: This epoch took 495.936857 s + +2022-05-14 07:05:04.136893: +epoch: 20 +2022-05-14 07:12:22.352340: train loss : 0.0613 +2022-05-14 07:12:47.604120: validation loss: 0.0433 +2022-05-14 07:12:47.604746: Average global foreground Dice: [0.9438, 0.9406, 0.918] +2022-05-14 07:12:47.604831: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 07:12:48.007533: lr: 0.000298 +2022-05-14 07:12:48.007740: saving scheduled checkpoint file... +2022-05-14 07:12:48.053102: saving checkpoint... +2022-05-14 07:12:48.837441: done, saving took 0.83 seconds +2022-05-14 07:12:48.856364: done +2022-05-14 07:12:48.879429: saving checkpoint... +2022-05-14 07:12:49.612973: done, saving took 0.76 seconds +2022-05-14 07:13:05.435194: This epoch took 481.298203 s + +2022-05-14 07:13:05.435488: +epoch: 21 +2022-05-14 07:20:23.737387: train loss : 0.0632 +2022-05-14 07:20:48.973839: validation loss: 0.0599 +2022-05-14 07:20:48.974451: Average global foreground Dice: [0.9445, 0.9348, 0.922] +2022-05-14 07:20:48.974537: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 07:20:49.372980: lr: 0.000298 +2022-05-14 07:20:49.373168: saving scheduled checkpoint file... +2022-05-14 07:20:49.417204: saving checkpoint... +2022-05-14 07:20:50.199849: done, saving took 0.83 seconds +2022-05-14 07:20:50.218434: done +2022-05-14 07:20:50.241119: saving checkpoint... +2022-05-14 07:20:50.998761: done, saving took 0.78 seconds +2022-05-14 07:21:06.259308: This epoch took 480.823713 s + +2022-05-14 07:21:06.259596: +epoch: 22 +2022-05-14 07:28:24.431362: train loss : 0.0589 +2022-05-14 07:28:49.611688: validation loss: 0.0562 +2022-05-14 07:28:49.612546: Average global foreground Dice: [0.9392, 0.9385, 0.9209] +2022-05-14 07:28:49.612634: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 07:28:50.019104: lr: 0.000298 +2022-05-14 07:28:50.019289: saving scheduled checkpoint file... +2022-05-14 07:28:50.064183: saving checkpoint... +2022-05-14 07:28:50.848470: done, saving took 0.83 seconds +2022-05-14 07:28:50.886115: done +2022-05-14 07:28:50.912203: saving checkpoint... +2022-05-14 07:28:51.683525: done, saving took 0.80 seconds +2022-05-14 07:29:12.255513: This epoch took 485.995810 s + +2022-05-14 07:29:12.255794: +epoch: 23 +2022-05-14 07:36:29.942899: train loss : 0.0586 +2022-05-14 07:36:55.113723: validation loss: 0.0637 +2022-05-14 07:36:55.114333: Average global foreground Dice: [0.944, 0.9352, 0.9179] +2022-05-14 07:36:55.114420: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 07:36:55.511764: lr: 0.000297 +2022-05-14 07:36:55.511960: saving scheduled checkpoint file... +2022-05-14 07:36:55.556518: saving checkpoint... +2022-05-14 07:36:56.287033: done, saving took 0.77 seconds +2022-05-14 07:36:56.305690: done +2022-05-14 07:36:56.325304: saving checkpoint... +2022-05-14 07:36:56.973860: done, saving took 0.67 seconds +2022-05-14 07:37:24.522483: This epoch took 492.266579 s + +2022-05-14 07:37:24.522759: +epoch: 24 +2022-05-14 07:44:42.498276: train loss : 0.0538 +2022-05-14 07:45:07.644319: validation loss: 0.0622 +2022-05-14 07:45:07.644949: Average global foreground Dice: [0.9467, 0.9349, 0.9194] +2022-05-14 07:45:07.645035: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 07:45:08.041734: lr: 0.000297 +2022-05-14 07:45:08.041930: saving scheduled checkpoint file... +2022-05-14 07:45:08.086230: saving checkpoint... +2022-05-14 07:45:08.860652: done, saving took 0.82 seconds +2022-05-14 07:45:08.877069: done +2022-05-14 07:45:08.895982: saving checkpoint... +2022-05-14 07:45:09.593419: done, saving took 0.72 seconds +2022-05-14 07:45:26.774872: This epoch took 482.252012 s + +2022-05-14 07:45:26.775158: +epoch: 25 +2022-05-14 07:52:44.737013: train loss : 0.0558 +2022-05-14 07:53:10.000914: validation loss: 0.0634 +2022-05-14 07:53:10.001586: Average global foreground Dice: [0.9418, 0.94, 0.9174] +2022-05-14 07:53:10.001673: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 07:53:10.400971: lr: 0.000297 +2022-05-14 07:53:10.401163: saving scheduled checkpoint file... +2022-05-14 07:53:10.445598: saving checkpoint... +2022-05-14 07:53:11.168398: done, saving took 0.77 seconds +2022-05-14 07:53:11.188308: done +2022-05-14 07:53:11.208388: saving checkpoint... +2022-05-14 07:53:11.872459: done, saving took 0.68 seconds +2022-05-14 07:53:38.948849: This epoch took 492.173589 s + +2022-05-14 07:53:38.949126: +epoch: 26 +2022-05-14 08:00:56.589947: train loss : 0.0569 +2022-05-14 08:01:21.784792: validation loss: 0.0551 +2022-05-14 08:01:21.785669: Average global foreground Dice: [0.9442, 0.9432, 0.917] +2022-05-14 08:01:21.785756: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 08:01:22.182583: lr: 0.000296 +2022-05-14 08:01:22.182787: saving scheduled checkpoint file... +2022-05-14 08:01:22.227817: saving checkpoint... +2022-05-14 08:01:22.921342: done, saving took 0.74 seconds +2022-05-14 08:01:22.937700: done +2022-05-14 08:01:22.959179: saving checkpoint... +2022-05-14 08:01:23.681929: done, saving took 0.74 seconds +2022-05-14 08:01:39.923712: This epoch took 480.974483 s + +2022-05-14 08:01:39.923975: +epoch: 27 +2022-05-14 08:08:58.057022: train loss : 0.0570 +2022-05-14 08:09:23.255688: validation loss: 0.0576 +2022-05-14 08:09:23.256322: Average global foreground Dice: [0.9417, 0.9406, 0.9191] +2022-05-14 08:09:23.256419: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 08:09:23.655599: lr: 0.000296 +2022-05-14 08:09:23.655793: saving scheduled checkpoint file... +2022-05-14 08:09:23.700002: saving checkpoint... +2022-05-14 08:09:24.426619: done, saving took 0.77 seconds +2022-05-14 08:09:24.443343: done +2022-05-14 08:09:24.462167: saving checkpoint... +2022-05-14 08:09:25.130616: done, saving took 0.69 seconds +2022-05-14 08:09:47.237102: This epoch took 487.313022 s + +2022-05-14 08:09:47.237396: +epoch: 28 +2022-05-14 08:17:05.758219: train loss : 0.0579 +2022-05-14 08:17:31.023205: validation loss: 0.0574 +2022-05-14 08:17:31.023889: Average global foreground Dice: [0.9313, 0.9293, 0.9203] +2022-05-14 08:17:31.023979: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 08:17:31.434720: lr: 0.000295 +2022-05-14 08:17:31.434905: saving scheduled checkpoint file... +2022-05-14 08:17:31.479674: saving checkpoint... +2022-05-14 08:17:32.253792: done, saving took 0.82 seconds +2022-05-14 08:17:32.273018: done +2022-05-14 08:17:32.297019: saving checkpoint... +2022-05-14 08:17:33.073965: done, saving took 0.80 seconds +2022-05-14 08:17:59.149725: This epoch took 491.912196 s + +2022-05-14 08:17:59.150013: +epoch: 29 +2022-05-14 08:25:17.228977: train loss : 0.0603 +2022-05-14 08:25:42.453463: validation loss: 0.0470 +2022-05-14 08:25:42.454149: Average global foreground Dice: [0.9441, 0.9436, 0.9173] +2022-05-14 08:25:42.454236: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 08:25:42.863846: lr: 0.000295 +2022-05-14 08:25:42.864038: saving scheduled checkpoint file... +2022-05-14 08:25:42.909283: saving checkpoint... +2022-05-14 08:25:43.694137: done, saving took 0.83 seconds +2022-05-14 08:25:43.712426: done +2022-05-14 08:25:43.735204: saving checkpoint... +2022-05-14 08:25:44.500973: done, saving took 0.79 seconds +2022-05-14 08:26:00.468827: This epoch took 481.318702 s + +2022-05-14 08:26:00.469131: +epoch: 30 +2022-05-14 08:33:18.664980: train loss : 0.0543 +2022-05-14 08:33:43.800677: validation loss: 0.0544 +2022-05-14 08:33:43.801419: Average global foreground Dice: [0.9462, 0.9332, 0.9122] +2022-05-14 08:33:43.801566: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 08:33:44.204571: lr: 0.000294 +2022-05-14 08:33:44.204777: saving scheduled checkpoint file... +2022-05-14 08:33:44.249448: saving checkpoint... +2022-05-14 08:33:45.033541: done, saving took 0.83 seconds +2022-05-14 08:33:45.052401: done +2022-05-14 08:33:45.075738: saving checkpoint... +2022-05-14 08:33:45.847732: done, saving took 0.80 seconds +2022-05-14 08:34:05.170523: This epoch took 484.701289 s + +2022-05-14 08:34:05.170800: +epoch: 31 +2022-05-14 08:41:22.930480: train loss : 0.0519 +2022-05-14 08:41:48.128380: validation loss: 0.0501 +2022-05-14 08:41:48.129150: Average global foreground Dice: [0.946, 0.9372, 0.9181] +2022-05-14 08:41:48.129236: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 08:41:48.579471: lr: 0.000294 +2022-05-14 08:41:48.579704: saving scheduled checkpoint file... +2022-05-14 08:41:48.629837: saving checkpoint... +2022-05-14 08:41:49.487213: done, saving took 0.91 seconds +2022-05-14 08:41:49.506603: done +2022-05-14 08:41:49.528265: saving checkpoint... +2022-05-14 08:41:50.320407: done, saving took 0.81 seconds +2022-05-14 08:42:07.933948: This epoch took 482.763037 s + +2022-05-14 08:42:07.934235: +epoch: 32 +2022-05-14 08:49:26.353698: train loss : 0.0570 +2022-05-14 08:49:51.549603: validation loss: 0.0539 +2022-05-14 08:49:51.550465: Average global foreground Dice: [0.9428, 0.9377, 0.9181] +2022-05-14 08:49:51.550552: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 08:49:51.950545: lr: 0.000293 +2022-05-14 08:49:51.950742: saving scheduled checkpoint file... +2022-05-14 08:49:51.995520: saving checkpoint... +2022-05-14 08:49:52.776590: done, saving took 0.83 seconds +2022-05-14 08:49:52.796548: done +2022-05-14 08:49:52.819158: saving checkpoint... +2022-05-14 08:49:53.593188: done, saving took 0.80 seconds +2022-05-14 08:50:10.122033: This epoch took 482.187693 s + +2022-05-14 08:50:10.122308: +epoch: 33 +2022-05-14 08:57:28.294659: train loss : 0.0557 +2022-05-14 08:57:53.605249: validation loss: 0.0554 +2022-05-14 08:57:53.605926: Average global foreground Dice: [0.9489, 0.9259, 0.917] +2022-05-14 08:57:53.606012: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 08:57:54.012584: lr: 0.000293 +2022-05-14 08:57:54.012775: saving scheduled checkpoint file... +2022-05-14 08:57:54.057263: saving checkpoint... +2022-05-14 08:57:54.851449: done, saving took 0.84 seconds +2022-05-14 08:57:54.870880: done +2022-05-14 08:57:54.894152: saving checkpoint... +2022-05-14 08:57:55.616229: done, saving took 0.75 seconds +2022-05-14 08:58:17.948708: This epoch took 487.826298 s + +2022-05-14 08:58:17.948981: +epoch: 34 +2022-05-14 09:05:36.004380: train loss : 0.0541 +2022-05-14 09:06:01.166288: validation loss: 0.0583 +2022-05-14 09:06:01.166955: Average global foreground Dice: [0.9436, 0.9402, 0.926] +2022-05-14 09:06:01.167044: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 09:06:01.570555: lr: 0.000292 +2022-05-14 09:06:01.570750: saving scheduled checkpoint file... +2022-05-14 09:06:01.615105: saving checkpoint... +2022-05-14 09:06:02.417431: done, saving took 0.85 seconds +2022-05-14 09:06:02.442515: done +2022-05-14 09:06:02.480800: saving checkpoint... +2022-05-14 09:06:03.245238: done, saving took 0.80 seconds +2022-05-14 09:06:36.642712: This epoch took 498.693629 s + +2022-05-14 09:06:36.642989: +epoch: 35 +2022-05-14 09:13:55.242823: train loss : 0.0522 +2022-05-14 09:14:20.412664: validation loss: 0.0440 +2022-05-14 09:14:20.413412: Average global foreground Dice: [0.9491, 0.943, 0.9249] +2022-05-14 09:14:20.413509: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 09:14:20.815907: lr: 0.000291 +2022-05-14 09:14:20.816100: saving scheduled checkpoint file... +2022-05-14 09:14:20.860300: saving checkpoint... +2022-05-14 09:14:21.646077: done, saving took 0.83 seconds +2022-05-14 09:14:21.664737: done +2022-05-14 09:14:21.687476: saving checkpoint... +2022-05-14 09:14:22.448699: done, saving took 0.78 seconds +2022-05-14 09:14:46.815570: This epoch took 490.172478 s + +2022-05-14 09:14:46.815847: +epoch: 36 +2022-05-14 09:22:04.116201: train loss : 0.0549 +2022-05-14 09:22:29.327832: validation loss: 0.0489 +2022-05-14 09:22:29.328523: Average global foreground Dice: [0.946, 0.9386, 0.9255] +2022-05-14 09:22:29.328613: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 09:22:29.734911: lr: 0.000291 +2022-05-14 09:22:29.735086: saving scheduled checkpoint file... +2022-05-14 09:22:29.780246: saving checkpoint... +2022-05-14 09:22:30.560070: done, saving took 0.82 seconds +2022-05-14 09:22:30.579637: done +2022-05-14 09:22:30.604226: saving checkpoint... +2022-05-14 09:22:31.383376: done, saving took 0.80 seconds +2022-05-14 09:23:20.465134: This epoch took 513.649189 s + +2022-05-14 09:23:20.465407: +epoch: 37 +2022-05-14 09:30:38.553055: train loss : 0.0533 +2022-05-14 09:31:03.781325: validation loss: 0.0577 +2022-05-14 09:31:03.781965: Average global foreground Dice: [0.9427, 0.9446, 0.921] +2022-05-14 09:31:03.782052: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 09:31:04.189054: lr: 0.00029 +2022-05-14 09:31:04.189463: saving scheduled checkpoint file... +2022-05-14 09:31:04.234795: saving checkpoint... +2022-05-14 09:31:05.006325: done, saving took 0.82 seconds +2022-05-14 09:31:05.028848: done +2022-05-14 09:31:05.060133: saving checkpoint... +2022-05-14 09:31:05.784504: done, saving took 0.76 seconds +2022-05-14 09:31:23.257932: This epoch took 482.792368 s + +2022-05-14 09:31:23.258350: +epoch: 38 +2022-05-14 09:38:41.226588: train loss : 0.0537 +2022-05-14 09:39:06.433905: validation loss: 0.0491 +2022-05-14 09:39:06.434539: Average global foreground Dice: [0.9515, 0.9474, 0.9279] +2022-05-14 09:39:06.434628: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 09:39:06.836018: lr: 0.000289 +2022-05-14 09:39:06.836211: saving scheduled checkpoint file... +2022-05-14 09:39:06.880911: saving checkpoint... +2022-05-14 09:39:07.629888: done, saving took 0.79 seconds +2022-05-14 09:39:07.651596: done +2022-05-14 09:39:07.685647: saving checkpoint... +2022-05-14 09:39:08.379767: done, saving took 0.73 seconds +2022-05-14 09:39:29.583888: This epoch took 486.325433 s + +2022-05-14 09:39:29.584222: +epoch: 39 +2022-05-14 09:46:47.601183: train loss : 0.0548 +2022-05-14 09:47:12.857470: validation loss: 0.0515 +2022-05-14 09:47:12.858096: Average global foreground Dice: [0.9489, 0.943, 0.9218] +2022-05-14 09:47:12.858183: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 09:47:13.262371: lr: 0.000289 +2022-05-14 09:47:13.262583: saving scheduled checkpoint file... +2022-05-14 09:47:13.309790: saving checkpoint... +2022-05-14 09:47:14.028258: done, saving took 0.77 seconds +2022-05-14 09:47:14.046648: done +2022-05-14 09:47:14.067796: saving checkpoint... +2022-05-14 09:47:14.781705: done, saving took 0.73 seconds +2022-05-14 09:47:32.169242: This epoch took 482.584895 s + +2022-05-14 09:47:32.169580: +epoch: 40 +2022-05-14 09:54:50.176030: train loss : 0.0481 +2022-05-14 09:55:15.517111: validation loss: 0.0498 +2022-05-14 09:55:15.517906: Average global foreground Dice: [0.9484, 0.9374, 0.9183] +2022-05-14 09:55:15.517992: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 09:55:15.934453: lr: 0.000288 +2022-05-14 09:55:15.934654: saving scheduled checkpoint file... +2022-05-14 09:55:15.979593: saving checkpoint... +2022-05-14 09:55:16.690818: done, saving took 0.76 seconds +2022-05-14 09:55:16.710221: done +2022-05-14 09:55:16.730517: saving checkpoint... +2022-05-14 09:55:17.424788: done, saving took 0.71 seconds +2022-05-14 09:55:38.638860: This epoch took 486.469169 s + +2022-05-14 09:55:38.639136: +epoch: 41 +2022-05-14 10:02:56.950636: train loss : 0.0504 +2022-05-14 10:03:22.195544: validation loss: 0.0543 +2022-05-14 10:03:22.196163: Average global foreground Dice: [0.9462, 0.937, 0.9263] +2022-05-14 10:03:22.196251: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 10:03:22.602478: lr: 0.000287 +2022-05-14 10:03:22.602676: saving scheduled checkpoint file... +2022-05-14 10:03:22.646883: saving checkpoint... +2022-05-14 10:03:23.346205: done, saving took 0.74 seconds +2022-05-14 10:03:23.362683: done +2022-05-14 10:03:23.381709: saving checkpoint... +2022-05-14 10:03:24.074444: done, saving took 0.71 seconds +2022-05-14 10:03:48.943304: This epoch took 490.304071 s + +2022-05-14 10:03:48.943611: +epoch: 42 +2022-05-14 10:11:07.608013: train loss : 0.0515 +2022-05-14 10:11:32.746723: validation loss: 0.0539 +2022-05-14 10:11:32.747316: Average global foreground Dice: [0.9438, 0.9313, 0.918] +2022-05-14 10:11:32.747404: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 10:11:33.144253: lr: 0.000286 +2022-05-14 10:11:33.144458: saving scheduled checkpoint file... +2022-05-14 10:11:33.188835: saving checkpoint... +2022-05-14 10:11:33.981640: done, saving took 0.84 seconds +2022-05-14 10:11:34.000630: done +2022-05-14 10:11:53.558414: This epoch took 484.614702 s + +2022-05-14 10:11:53.558683: +epoch: 43 +2022-05-14 10:19:11.962467: train loss : 0.0505 +2022-05-14 10:19:37.208252: validation loss: 0.0473 +2022-05-14 10:19:37.208919: Average global foreground Dice: [0.9487, 0.9442, 0.9249] +2022-05-14 10:19:37.209007: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 10:19:37.607820: lr: 0.000285 +2022-05-14 10:19:37.608008: saving scheduled checkpoint file... +2022-05-14 10:19:37.652627: saving checkpoint... +2022-05-14 10:19:38.426168: done, saving took 0.82 seconds +2022-05-14 10:19:38.444584: done +2022-05-14 10:19:38.467010: saving checkpoint... +2022-05-14 10:19:39.240205: done, saving took 0.80 seconds +2022-05-14 10:19:56.710955: This epoch took 483.152169 s + +2022-05-14 10:19:56.711244: +epoch: 44 +2022-05-14 10:27:14.384019: train loss : 0.0519 +2022-05-14 10:27:39.566983: validation loss: 0.0535 +2022-05-14 10:27:39.567706: Average global foreground Dice: [0.9495, 0.9402, 0.9244] +2022-05-14 10:27:39.567793: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 10:27:39.965754: lr: 0.000285 +2022-05-14 10:27:39.965950: saving scheduled checkpoint file... +2022-05-14 10:27:40.010401: saving checkpoint... +2022-05-14 10:27:40.805974: done, saving took 0.84 seconds +2022-05-14 10:27:40.829566: done +2022-05-14 10:27:40.856468: saving checkpoint... +2022-05-14 10:27:41.633301: done, saving took 0.80 seconds +2022-05-14 10:28:00.955950: This epoch took 484.244603 s + +2022-05-14 10:28:00.956245: +epoch: 45 +2022-05-14 10:35:18.296296: train loss : 0.0513 +2022-05-14 10:35:43.483722: validation loss: 0.0544 +2022-05-14 10:35:43.484324: Average global foreground Dice: [0.9465, 0.9431, 0.9249] +2022-05-14 10:35:43.484411: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 10:35:43.886325: lr: 0.000284 +2022-05-14 10:35:43.886533: saving scheduled checkpoint file... +2022-05-14 10:35:43.931168: saving checkpoint... +2022-05-14 10:35:44.719319: done, saving took 0.83 seconds +2022-05-14 10:35:44.737765: done +2022-05-14 10:35:44.760617: saving checkpoint... +2022-05-14 10:35:45.526352: done, saving took 0.79 seconds +2022-05-14 10:36:00.898759: This epoch took 479.942411 s + +2022-05-14 10:36:00.899050: +epoch: 46 +2022-05-14 10:43:18.512292: train loss : 0.0497 +2022-05-14 10:43:43.724123: validation loss: 0.0467 +2022-05-14 10:43:43.724780: Average global foreground Dice: [0.9497, 0.943, 0.9204] +2022-05-14 10:43:43.724870: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 10:43:44.124091: lr: 0.000283 +2022-05-14 10:43:44.124286: saving scheduled checkpoint file... +2022-05-14 10:43:44.169140: saving checkpoint... +2022-05-14 10:43:44.967116: done, saving took 0.84 seconds +2022-05-14 10:43:44.992752: done +2022-05-14 10:43:45.032886: saving checkpoint... +2022-05-14 10:43:45.800553: done, saving took 0.81 seconds +2022-05-14 10:44:03.137526: This epoch took 482.238375 s + +2022-05-14 10:44:03.137808: +epoch: 47 +2022-05-14 10:51:21.528461: train loss : 0.0547 +2022-05-14 10:51:46.772011: validation loss: 0.0521 +2022-05-14 10:51:46.772633: Average global foreground Dice: [0.9484, 0.9409, 0.9243] +2022-05-14 10:51:46.772720: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 10:51:47.171428: lr: 0.000282 +2022-05-14 10:51:47.171625: saving scheduled checkpoint file... +2022-05-14 10:51:47.216202: saving checkpoint... +2022-05-14 10:51:48.017606: done, saving took 0.85 seconds +2022-05-14 10:51:48.041406: done +2022-05-14 10:51:48.080013: saving checkpoint... +2022-05-14 10:51:48.844414: done, saving took 0.80 seconds +2022-05-14 10:52:03.994515: This epoch took 480.856552 s + +2022-05-14 10:52:03.994787: +epoch: 48 +2022-05-14 10:59:21.726314: train loss : 0.0513 +2022-05-14 10:59:46.931412: validation loss: 0.0554 +2022-05-14 10:59:46.932050: Average global foreground Dice: [0.9456, 0.9305, 0.9266] +2022-05-14 10:59:46.932135: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 10:59:47.338636: lr: 0.000281 +2022-05-14 10:59:47.338826: saving scheduled checkpoint file... +2022-05-14 10:59:47.382642: saving checkpoint... +2022-05-14 10:59:48.103085: done, saving took 0.76 seconds +2022-05-14 10:59:48.126243: done +2022-05-14 11:00:15.039927: This epoch took 491.045034 s + +2022-05-14 11:00:15.040189: +epoch: 49 +2022-05-14 11:07:33.552930: train loss : 0.0534 +2022-05-14 11:07:58.732751: validation loss: 0.0580 +2022-05-14 11:07:58.733420: Average global foreground Dice: [0.9501, 0.9364, 0.9258] +2022-05-14 11:07:58.733524: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 11:07:59.133975: lr: 0.00028 +2022-05-14 11:07:59.134166: saving scheduled checkpoint file... +2022-05-14 11:07:59.181923: saving checkpoint... +2022-05-14 11:07:59.970505: done, saving took 0.84 seconds +2022-05-14 11:07:59.987571: done +2022-05-14 11:08:00.007185: saving checkpoint... +2022-05-14 11:08:00.762401: done, saving took 0.77 seconds +2022-05-14 11:08:18.925753: This epoch took 483.885415 s + +2022-05-14 11:08:18.926129: +epoch: 50 +2022-05-14 11:15:36.668220: train loss : 0.0524 +2022-05-14 11:16:01.927119: validation loss: 0.0510 +2022-05-14 11:16:01.927743: Average global foreground Dice: [0.9464, 0.9376, 0.9228] +2022-05-14 11:16:01.927830: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 11:16:02.328243: lr: 0.000279 +2022-05-14 11:16:02.328421: saving scheduled checkpoint file... +2022-05-14 11:16:02.372656: saving checkpoint... +2022-05-14 11:16:03.170201: done, saving took 0.84 seconds +2022-05-14 11:16:03.193112: done +2022-05-14 11:16:03.226949: saving checkpoint... +2022-05-14 11:16:03.990661: done, saving took 0.80 seconds +2022-05-14 11:16:26.134207: This epoch took 487.207960 s + +2022-05-14 11:16:26.134525: +epoch: 51 +2022-05-14 11:23:44.624410: train loss : 0.0537 +2022-05-14 11:24:09.970050: validation loss: 0.0556 +2022-05-14 11:24:09.970671: Average global foreground Dice: [0.9439, 0.9357, 0.928] +2022-05-14 11:24:09.970758: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 11:24:10.374128: lr: 0.000278 +2022-05-14 11:24:10.374318: saving scheduled checkpoint file... +2022-05-14 11:24:10.421279: saving checkpoint... +2022-05-14 11:24:11.230884: done, saving took 0.86 seconds +2022-05-14 11:24:11.260525: done +2022-05-14 11:24:11.300916: saving checkpoint... +2022-05-14 11:24:12.059625: done, saving took 0.80 seconds +2022-05-14 11:24:29.349894: This epoch took 483.215259 s + +2022-05-14 11:24:29.350171: +epoch: 52 +2022-05-14 11:31:47.511267: train loss : 0.0517 +2022-05-14 11:32:12.744855: validation loss: 0.0554 +2022-05-14 11:32:12.745541: Average global foreground Dice: [0.9469, 0.9388, 0.9224] +2022-05-14 11:32:12.745632: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 11:32:13.147106: lr: 0.000277 +2022-05-14 11:32:13.147296: saving scheduled checkpoint file... +2022-05-14 11:32:13.191963: saving checkpoint... +2022-05-14 11:32:13.971547: done, saving took 0.82 seconds +2022-05-14 11:32:13.990389: done +2022-05-14 11:32:14.013146: saving checkpoint... +2022-05-14 11:32:14.788322: done, saving took 0.80 seconds +2022-05-14 11:32:31.984244: This epoch took 482.633962 s + +2022-05-14 11:32:31.984616: +epoch: 53 +2022-05-14 11:39:50.947832: train loss : 0.0536 +2022-05-14 11:40:16.178553: validation loss: 0.0603 +2022-05-14 11:40:16.179179: Average global foreground Dice: [0.9459, 0.9297, 0.9218] +2022-05-14 11:40:16.179264: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 11:40:16.581659: lr: 0.000276 +2022-05-14 11:40:16.581842: saving scheduled checkpoint file... +2022-05-14 11:40:16.629441: saving checkpoint... +2022-05-14 11:40:17.371790: done, saving took 0.79 seconds +2022-05-14 11:40:17.390202: done +2022-05-14 11:40:35.004479: This epoch took 483.019725 s + +2022-05-14 11:40:35.004761: +epoch: 54 +2022-05-14 11:47:54.014667: train loss : 0.0512 +2022-05-14 11:48:19.295907: validation loss: 0.0547 +2022-05-14 11:48:19.296549: Average global foreground Dice: [0.9521, 0.9377, 0.9259] +2022-05-14 11:48:19.296637: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 11:48:19.702986: lr: 0.000275 +2022-05-14 11:48:19.703177: saving scheduled checkpoint file... +2022-05-14 11:48:19.748258: saving checkpoint... +2022-05-14 11:48:20.540711: done, saving took 0.84 seconds +2022-05-14 11:48:20.562914: done +2022-05-14 11:48:20.586945: saving checkpoint... +2022-05-14 11:48:21.362539: done, saving took 0.80 seconds +2022-05-14 11:48:39.828463: This epoch took 484.823590 s + +2022-05-14 11:48:39.828737: +epoch: 55 +2022-05-14 11:55:58.129328: train loss : 0.0487 +2022-05-14 11:56:23.357391: validation loss: 0.0552 +2022-05-14 11:56:23.358029: Average global foreground Dice: [0.9463, 0.9317, 0.9185] +2022-05-14 11:56:23.358118: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 11:56:23.758462: lr: 0.000274 +2022-05-14 11:56:23.758659: saving scheduled checkpoint file... +2022-05-14 11:56:23.802968: saving checkpoint... +2022-05-14 11:56:24.607701: done, saving took 0.85 seconds +2022-05-14 11:56:24.626866: done +2022-05-14 11:56:42.174639: This epoch took 482.345797 s + +2022-05-14 11:56:42.174916: +epoch: 56 +2022-05-14 12:04:01.517646: train loss : 0.0522 +2022-05-14 12:04:26.814686: validation loss: 0.0472 +2022-05-14 12:04:26.815538: Average global foreground Dice: [0.9506, 0.933, 0.92] +2022-05-14 12:04:26.815629: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 12:04:27.216088: lr: 0.000272 +2022-05-14 12:04:27.216281: saving scheduled checkpoint file... +2022-05-14 12:04:27.261223: saving checkpoint... +2022-05-14 12:04:28.053575: done, saving took 0.84 seconds +2022-05-14 12:04:28.073117: done +2022-05-14 12:04:48.476904: This epoch took 486.301876 s + +2022-05-14 12:04:48.477276: +epoch: 57 +2022-05-14 12:12:06.001076: train loss : 0.0506 +2022-05-14 12:12:31.291813: validation loss: 0.0503 +2022-05-14 12:12:31.292576: Average global foreground Dice: [0.9512, 0.9387, 0.9263] +2022-05-14 12:12:31.292664: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 12:12:31.693692: lr: 0.000271 +2022-05-14 12:12:31.693880: saving scheduled checkpoint file... +2022-05-14 12:12:31.738771: saving checkpoint... +2022-05-14 12:12:32.515743: done, saving took 0.82 seconds +2022-05-14 12:12:32.534815: done +2022-05-14 12:12:52.157543: This epoch took 483.680172 s + +2022-05-14 12:12:52.157829: +epoch: 58 +2022-05-14 12:20:09.794342: train loss : 0.0501 +2022-05-14 12:20:35.054283: validation loss: 0.0506 +2022-05-14 12:20:35.056071: Average global foreground Dice: [0.9509, 0.9453, 0.9259] +2022-05-14 12:20:35.056212: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 12:20:35.460202: lr: 0.00027 +2022-05-14 12:20:35.460411: saving scheduled checkpoint file... +2022-05-14 12:20:35.505215: saving checkpoint... +2022-05-14 12:20:36.285710: done, saving took 0.83 seconds +2022-05-14 12:20:36.304507: done +2022-05-14 12:20:36.327869: saving checkpoint... +2022-05-14 12:20:37.104348: done, saving took 0.80 seconds +2022-05-14 12:20:57.079975: This epoch took 484.922035 s + +2022-05-14 12:20:57.080286: +epoch: 59 +2022-05-14 12:28:15.357270: train loss : 0.0489 +2022-05-14 12:28:40.530340: validation loss: 0.0527 +2022-05-14 12:28:40.530983: Average global foreground Dice: [0.9489, 0.9403, 0.9276] +2022-05-14 12:28:40.531069: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 12:28:40.933505: lr: 0.000269 +2022-05-14 12:28:40.933700: saving scheduled checkpoint file... +2022-05-14 12:28:40.978001: saving checkpoint... +2022-05-14 12:28:41.783278: done, saving took 0.85 seconds +2022-05-14 12:28:41.801579: done +2022-05-14 12:28:41.824970: saving checkpoint... +2022-05-14 12:28:42.603209: done, saving took 0.80 seconds +2022-05-14 12:29:06.206064: This epoch took 489.125676 s + +2022-05-14 12:29:06.206341: +epoch: 60 +2022-05-14 12:36:24.458856: train loss : 0.0498 +2022-05-14 12:36:49.702554: validation loss: 0.0525 +2022-05-14 12:36:49.703166: Average global foreground Dice: [0.9452, 0.9327, 0.9208] +2022-05-14 12:36:49.703251: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 12:36:50.117248: lr: 0.000268 +2022-05-14 12:36:50.117431: saving scheduled checkpoint file... +2022-05-14 12:36:50.162470: saving checkpoint... +2022-05-14 12:36:50.948688: done, saving took 0.83 seconds +2022-05-14 12:36:50.967720: done +2022-05-14 12:37:09.791790: This epoch took 483.585347 s + +2022-05-14 12:37:09.792065: +epoch: 61 +2022-05-14 12:44:27.554224: train loss : 0.0518 +2022-05-14 12:44:52.717310: validation loss: 0.0573 +2022-05-14 12:44:52.717937: Average global foreground Dice: [0.9509, 0.9348, 0.9247] +2022-05-14 12:44:52.718022: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 12:44:53.118088: lr: 0.000267 +2022-05-14 12:44:53.118278: saving scheduled checkpoint file... +2022-05-14 12:44:53.162244: saving checkpoint... +2022-05-14 12:44:53.881083: done, saving took 0.76 seconds +2022-05-14 12:44:53.907626: done +2022-05-14 12:45:11.737108: This epoch took 481.944938 s + +2022-05-14 12:45:11.737383: +epoch: 62 +2022-05-14 12:52:29.811606: train loss : 0.0510 +2022-05-14 12:52:55.011421: validation loss: 0.0440 +2022-05-14 12:52:55.012091: Average global foreground Dice: [0.9467, 0.9426, 0.9249] +2022-05-14 12:52:55.012181: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 12:52:55.414840: lr: 0.000265 +2022-05-14 12:52:55.415043: saving scheduled checkpoint file... +2022-05-14 12:52:55.459493: saving checkpoint... +2022-05-14 12:52:56.160291: done, saving took 0.75 seconds +2022-05-14 12:52:56.178295: done +2022-05-14 12:53:30.631660: This epoch took 498.894150 s + +2022-05-14 12:53:30.632197: +epoch: 63 +2022-05-14 13:00:47.483860: train loss : 0.0479 +2022-05-14 13:01:12.674369: validation loss: 0.0518 +2022-05-14 13:01:12.675333: Average global foreground Dice: [0.941, 0.9411, 0.9248] +2022-05-14 13:01:12.675420: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 13:01:13.080520: lr: 0.000264 +2022-05-14 13:01:13.080757: saving scheduled checkpoint file... +2022-05-14 13:01:13.125886: saving checkpoint... +2022-05-14 13:01:13.832447: done, saving took 0.75 seconds +2022-05-14 13:01:13.848533: done +2022-05-14 13:01:46.932110: This epoch took 496.299530 s + +2022-05-14 13:01:46.932390: +epoch: 64 +2022-05-14 13:09:04.156245: train loss : 0.0505 +2022-05-14 13:09:29.419202: validation loss: 0.0472 +2022-05-14 13:09:29.419858: Average global foreground Dice: [0.9486, 0.9397, 0.925] +2022-05-14 13:09:29.419947: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 13:09:29.832150: lr: 0.000263 +2022-05-14 13:09:29.832388: saving scheduled checkpoint file... +2022-05-14 13:09:29.876276: saving checkpoint... +2022-05-14 13:09:30.647897: done, saving took 0.82 seconds +2022-05-14 13:09:30.666598: done +2022-05-14 13:09:30.689513: saving checkpoint... +2022-05-14 13:09:31.448968: done, saving took 0.78 seconds +2022-05-14 13:09:49.514214: This epoch took 482.581705 s + +2022-05-14 13:09:49.514496: +epoch: 65 +2022-05-14 13:17:06.811662: train loss : 0.0475 +2022-05-14 13:17:32.042371: validation loss: 0.0480 +2022-05-14 13:17:32.043019: Average global foreground Dice: [0.9497, 0.9394, 0.9288] +2022-05-14 13:17:32.043104: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 13:17:32.444949: lr: 0.000261 +2022-05-14 13:17:32.445140: saving scheduled checkpoint file... +2022-05-14 13:17:32.489971: saving checkpoint... +2022-05-14 13:17:33.281721: done, saving took 0.84 seconds +2022-05-14 13:17:33.300430: done +2022-05-14 13:17:33.323604: saving checkpoint... +2022-05-14 13:17:34.092861: done, saving took 0.79 seconds +2022-05-14 13:17:54.262505: This epoch took 484.747907 s + +2022-05-14 13:17:54.262842: +epoch: 66 +2022-05-14 13:25:11.596638: train loss : 0.0484 +2022-05-14 13:25:36.801767: validation loss: 0.0566 +2022-05-14 13:25:36.802415: Average global foreground Dice: [0.9482, 0.9374, 0.9266] +2022-05-14 13:25:36.802513: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 13:25:37.208703: lr: 0.00026 +2022-05-14 13:25:37.208898: saving scheduled checkpoint file... +2022-05-14 13:25:37.253707: saving checkpoint... +2022-05-14 13:25:38.056065: done, saving took 0.85 seconds +2022-05-14 13:25:38.074979: done +2022-05-14 13:25:38.097658: saving checkpoint... +2022-05-14 13:25:38.870293: done, saving took 0.80 seconds +2022-05-14 13:25:56.140386: This epoch took 481.877429 s + +2022-05-14 13:25:56.140911: +epoch: 67 +2022-05-14 13:33:14.291871: train loss : 0.0474 +2022-05-14 13:33:39.616506: validation loss: 0.0549 +2022-05-14 13:33:39.617210: Average global foreground Dice: [0.9507, 0.9484, 0.9245] +2022-05-14 13:33:39.617296: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 13:33:40.071348: lr: 0.000259 +2022-05-14 13:33:40.071574: saving scheduled checkpoint file... +2022-05-14 13:33:40.120025: saving checkpoint... +2022-05-14 13:33:40.979794: done, saving took 0.91 seconds +2022-05-14 13:33:40.999245: done +2022-05-14 13:33:41.020881: saving checkpoint... +2022-05-14 13:33:41.875319: done, saving took 0.88 seconds +2022-05-14 13:34:00.496866: This epoch took 484.355845 s + +2022-05-14 13:34:00.497151: +epoch: 68 +2022-05-14 13:41:18.476709: train loss : 0.0481 +2022-05-14 13:41:43.776295: validation loss: 0.0556 +2022-05-14 13:41:43.776914: Average global foreground Dice: [0.949, 0.9373, 0.9226] +2022-05-14 13:41:43.776998: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 13:41:44.194309: lr: 0.000257 +2022-05-14 13:41:44.194539: saving scheduled checkpoint file... +2022-05-14 13:41:44.239452: saving checkpoint... +2022-05-14 13:41:45.027510: done, saving took 0.83 seconds +2022-05-14 13:41:45.046316: done +2022-05-14 13:42:09.592672: This epoch took 489.095395 s + +2022-05-14 13:42:09.593039: +epoch: 69 +2022-05-14 13:49:27.281625: train loss : 0.0492 +2022-05-14 13:49:52.641599: validation loss: 0.0512 +2022-05-14 13:49:52.642268: Average global foreground Dice: [0.9454, 0.9347, 0.9205] +2022-05-14 13:49:52.642353: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 13:49:53.048889: lr: 0.000256 +2022-05-14 13:49:53.049075: saving scheduled checkpoint file... +2022-05-14 13:49:53.093341: saving checkpoint... +2022-05-14 13:49:53.852886: done, saving took 0.80 seconds +2022-05-14 13:49:53.870167: done +2022-05-14 13:50:10.494742: This epoch took 480.901597 s + +2022-05-14 13:50:10.495051: +epoch: 70 +2022-05-14 13:57:28.364502: train loss : 0.0503 +2022-05-14 13:57:53.715277: validation loss: 0.0532 +2022-05-14 13:57:53.715974: Average global foreground Dice: [0.9486, 0.9404, 0.9237] +2022-05-14 13:57:53.716060: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 13:57:54.120847: lr: 0.000255 +2022-05-14 13:57:54.121029: saving scheduled checkpoint file... +2022-05-14 13:57:54.165783: saving checkpoint... +2022-05-14 13:57:54.954188: done, saving took 0.83 seconds +2022-05-14 13:57:54.972812: done +2022-05-14 13:58:11.379540: This epoch took 480.884390 s + +2022-05-14 13:58:11.379834: +epoch: 71 +2022-05-14 14:05:29.499403: train loss : 0.0492 +2022-05-14 14:05:54.838500: validation loss: 0.0511 +2022-05-14 14:05:54.839128: Average global foreground Dice: [0.9469, 0.9384, 0.9252] +2022-05-14 14:05:54.839214: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 14:05:55.248928: lr: 0.000253 +2022-05-14 14:05:55.249123: saving scheduled checkpoint file... +2022-05-14 14:05:55.294607: saving checkpoint... +2022-05-14 14:05:56.081523: done, saving took 0.83 seconds +2022-05-14 14:05:56.100324: done +2022-05-14 14:06:12.900952: This epoch took 481.521014 s + +2022-05-14 14:06:12.901221: +epoch: 72 +2022-05-14 14:13:30.277156: train loss : 0.0485 +2022-05-14 14:13:55.663442: validation loss: 0.0589 +2022-05-14 14:13:55.664095: Average global foreground Dice: [0.9458, 0.9392, 0.9196] +2022-05-14 14:13:55.664212: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 14:13:56.071952: lr: 0.000252 +2022-05-14 14:13:56.072136: saving scheduled checkpoint file... +2022-05-14 14:13:56.116692: saving checkpoint... +2022-05-14 14:13:56.916140: done, saving took 0.84 seconds +2022-05-14 14:13:56.933015: done +2022-05-14 14:14:20.912784: This epoch took 488.011461 s + +2022-05-14 14:14:20.913057: +epoch: 73 +2022-05-14 14:21:38.922081: train loss : 0.0479 +2022-05-14 14:22:04.265679: validation loss: 0.0448 +2022-05-14 14:22:04.266385: Average global foreground Dice: [0.9453, 0.9482, 0.9229] +2022-05-14 14:22:04.266485: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 14:22:04.670367: lr: 0.00025 +2022-05-14 14:22:04.670569: saving scheduled checkpoint file... +2022-05-14 14:22:04.715499: saving checkpoint... +2022-05-14 14:22:05.504102: done, saving took 0.83 seconds +2022-05-14 14:22:05.524160: done +2022-05-14 14:22:24.186530: This epoch took 483.273369 s + +2022-05-14 14:22:24.186930: +epoch: 74 +2022-05-14 14:29:42.161145: train loss : 0.0447 +2022-05-14 14:30:07.429797: validation loss: 0.0533 +2022-05-14 14:30:07.430421: Average global foreground Dice: [0.9476, 0.9375, 0.9234] +2022-05-14 14:30:07.430519: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 14:30:07.837401: lr: 0.000249 +2022-05-14 14:30:07.837604: saving scheduled checkpoint file... +2022-05-14 14:30:07.882108: saving checkpoint... +2022-05-14 14:30:08.662837: done, saving took 0.83 seconds +2022-05-14 14:30:08.681772: done +2022-05-14 14:30:27.674571: This epoch took 483.487516 s + +2022-05-14 14:30:27.675080: +epoch: 75 +2022-05-14 14:37:45.450285: train loss : 0.0451 +2022-05-14 14:38:10.780571: validation loss: 0.0608 +2022-05-14 14:38:10.781181: Average global foreground Dice: [0.9473, 0.9413, 0.9257] +2022-05-14 14:38:10.781270: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 14:38:11.186302: lr: 0.000247 +2022-05-14 14:38:11.186512: saving scheduled checkpoint file... +2022-05-14 14:38:11.230502: saving checkpoint... +2022-05-14 14:38:12.011765: done, saving took 0.83 seconds +2022-05-14 14:38:12.030285: done +2022-05-14 14:38:34.755928: This epoch took 487.080477 s + +2022-05-14 14:38:34.756269: +epoch: 76 +2022-05-14 14:45:52.732468: train loss : 0.0481 +2022-05-14 14:46:18.072399: validation loss: 0.0520 +2022-05-14 14:46:18.073114: Average global foreground Dice: [0.9491, 0.9388, 0.9263] +2022-05-14 14:46:18.073200: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 14:46:18.534068: lr: 0.000246 +2022-05-14 14:46:18.534254: saving scheduled checkpoint file... +2022-05-14 14:46:18.581801: saving checkpoint... +2022-05-14 14:46:19.406785: done, saving took 0.87 seconds +2022-05-14 14:46:19.424914: done +2022-05-14 14:46:45.078075: This epoch took 490.321680 s + +2022-05-14 14:46:45.078401: +epoch: 77 +2022-05-14 14:54:03.125848: train loss : 0.0457 +2022-05-14 14:54:28.435014: validation loss: 0.0550 +2022-05-14 14:54:28.435667: Average global foreground Dice: [0.9467, 0.9349, 0.925] +2022-05-14 14:54:28.435755: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 14:54:28.847014: lr: 0.000244 +2022-05-14 14:54:28.847203: saving scheduled checkpoint file... +2022-05-14 14:54:28.891958: saving checkpoint... +2022-05-14 14:54:29.691770: done, saving took 0.84 seconds +2022-05-14 14:54:29.710021: done +2022-05-14 14:54:54.404334: This epoch took 489.325789 s + +2022-05-14 14:54:54.404749: +epoch: 78 +2022-05-14 15:02:12.621842: train loss : 0.0474 +2022-05-14 15:02:37.962281: validation loss: 0.0496 +2022-05-14 15:02:37.962893: Average global foreground Dice: [0.9523, 0.9411, 0.9264] +2022-05-14 15:02:37.962979: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 15:02:38.369806: lr: 0.000243 +2022-05-14 15:02:38.369998: saving scheduled checkpoint file... +2022-05-14 15:02:38.414427: saving checkpoint... +2022-05-14 15:02:39.189204: done, saving took 0.82 seconds +2022-05-14 15:02:39.207608: done +2022-05-14 15:02:39.231178: saving checkpoint... +2022-05-14 15:02:40.008740: done, saving took 0.80 seconds +2022-05-14 15:03:01.943239: This epoch took 487.538387 s + +2022-05-14 15:03:01.943512: +epoch: 79 +2022-05-14 15:10:20.266391: train loss : 0.0440 +2022-05-14 15:10:45.577593: validation loss: 0.0510 +2022-05-14 15:10:45.578237: Average global foreground Dice: [0.9526, 0.9422, 0.9281] +2022-05-14 15:10:45.578321: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 15:10:45.990547: lr: 0.000241 +2022-05-14 15:10:45.990738: saving scheduled checkpoint file... +2022-05-14 15:10:46.034934: saving checkpoint... +2022-05-14 15:10:46.807249: done, saving took 0.82 seconds +2022-05-14 15:10:46.825348: done +2022-05-14 15:10:46.848043: saving checkpoint... +2022-05-14 15:10:47.622047: done, saving took 0.80 seconds +2022-05-14 15:11:05.692602: This epoch took 483.748961 s + +2022-05-14 15:11:05.692964: +epoch: 80 +2022-05-14 15:18:23.439691: train loss : 0.0477 +2022-05-14 15:18:48.759563: validation loss: 0.0486 +2022-05-14 15:18:48.760196: Average global foreground Dice: [0.9493, 0.9388, 0.9235] +2022-05-14 15:18:48.760283: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 15:18:49.173684: lr: 0.00024 +2022-05-14 15:18:49.173869: saving scheduled checkpoint file... +2022-05-14 15:18:49.218045: saving checkpoint... +2022-05-14 15:18:50.008193: done, saving took 0.83 seconds +2022-05-14 15:18:50.026592: done +2022-05-14 15:19:19.139895: This epoch took 493.446825 s + +2022-05-14 15:19:19.140188: +epoch: 81 +2022-05-14 15:26:36.527579: train loss : 0.0444 +2022-05-14 15:27:01.837282: validation loss: 0.0521 +2022-05-14 15:27:01.837920: Average global foreground Dice: [0.9474, 0.925, 0.9185] +2022-05-14 15:27:01.838008: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 15:27:02.247623: lr: 0.000238 +2022-05-14 15:27:02.247812: saving scheduled checkpoint file... +2022-05-14 15:27:02.293123: saving checkpoint... +2022-05-14 15:27:02.970904: done, saving took 0.72 seconds +2022-05-14 15:27:02.989018: done +2022-05-14 15:27:28.399117: This epoch took 489.258790 s + +2022-05-14 15:27:28.399493: +epoch: 82 +2022-05-14 15:34:46.392688: train loss : 0.0469 +2022-05-14 15:35:11.710382: validation loss: 0.0515 +2022-05-14 15:35:11.711037: Average global foreground Dice: [0.9485, 0.938, 0.9187] +2022-05-14 15:35:11.711122: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 15:35:12.119572: lr: 0.000237 +2022-05-14 15:35:12.119810: saving scheduled checkpoint file... +2022-05-14 15:35:12.168202: saving checkpoint... +2022-05-14 15:35:12.888892: done, saving took 0.77 seconds +2022-05-14 15:35:12.906056: done +2022-05-14 15:35:34.119326: This epoch took 485.719711 s + +2022-05-14 15:35:34.119613: +epoch: 83 +2022-05-14 15:42:51.718667: train loss : 0.0483 +2022-05-14 15:43:17.029248: validation loss: 0.0488 +2022-05-14 15:43:17.029946: Average global foreground Dice: [0.9514, 0.9359, 0.9277] +2022-05-14 15:43:17.030031: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 15:43:17.428736: lr: 0.000235 +2022-05-14 15:43:17.428926: saving scheduled checkpoint file... +2022-05-14 15:43:17.473879: saving checkpoint... +2022-05-14 15:43:18.188965: done, saving took 0.76 seconds +2022-05-14 15:43:18.206176: done +2022-05-14 15:43:37.134306: This epoch took 483.014590 s + +2022-05-14 15:43:37.134676: +epoch: 84 +2022-05-14 15:50:55.354745: train loss : 0.0436 +2022-05-14 15:51:20.687874: validation loss: 0.0473 +2022-05-14 15:51:20.688544: Average global foreground Dice: [0.947, 0.9444, 0.9279] +2022-05-14 15:51:20.688633: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 15:51:21.091664: lr: 0.000233 +2022-05-14 15:51:21.091851: saving scheduled checkpoint file... +2022-05-14 15:51:21.137151: saving checkpoint... +2022-05-14 15:51:21.880023: done, saving took 0.79 seconds +2022-05-14 15:51:21.897403: done +2022-05-14 15:51:42.617088: This epoch took 485.482298 s + +2022-05-14 15:51:42.617351: +epoch: 85 +2022-05-14 15:59:00.169896: train loss : 0.0426 +2022-05-14 15:59:25.491172: validation loss: 0.0613 +2022-05-14 15:59:25.491782: Average global foreground Dice: [0.9496, 0.9355, 0.9242] +2022-05-14 15:59:25.491868: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 15:59:25.894988: lr: 0.000232 +2022-05-14 15:59:25.895177: saving scheduled checkpoint file... +2022-05-14 15:59:25.939127: saving checkpoint... +2022-05-14 15:59:26.713120: done, saving took 0.82 seconds +2022-05-14 15:59:26.733673: done +2022-05-14 15:59:47.214938: This epoch took 484.597466 s + +2022-05-14 15:59:47.215310: +epoch: 86 +2022-05-14 16:07:04.468102: train loss : 0.0444 +2022-05-14 16:07:29.672667: validation loss: 0.0535 +2022-05-14 16:07:29.673405: Average global foreground Dice: [0.9487, 0.9439, 0.9296] +2022-05-14 16:07:29.673501: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 16:07:30.076298: lr: 0.00023 +2022-05-14 16:07:30.076493: saving scheduled checkpoint file... +2022-05-14 16:07:30.120780: saving checkpoint... +2022-05-14 16:07:30.908005: done, saving took 0.83 seconds +2022-05-14 16:07:30.927108: done +2022-05-14 16:07:52.396394: This epoch took 485.180981 s + +2022-05-14 16:07:52.396770: +epoch: 87 +2022-05-14 16:15:10.540262: train loss : 0.0470 +2022-05-14 16:15:35.837021: validation loss: 0.0562 +2022-05-14 16:15:35.837910: Average global foreground Dice: [0.95, 0.9334, 0.922] +2022-05-14 16:15:35.838042: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 16:15:36.259682: lr: 0.000228 +2022-05-14 16:15:36.259900: saving scheduled checkpoint file... +2022-05-14 16:15:36.305806: saving checkpoint... +2022-05-14 16:15:37.094831: done, saving took 0.83 seconds +2022-05-14 16:15:37.114544: done +2022-05-14 16:15:56.726271: This epoch took 484.329386 s + +2022-05-14 16:15:56.726626: +epoch: 88 +2022-05-14 16:23:14.665350: train loss : 0.0434 +2022-05-14 16:23:39.932530: validation loss: 0.0528 +2022-05-14 16:23:39.933191: Average global foreground Dice: [0.9507, 0.945, 0.9246] +2022-05-14 16:23:39.933280: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 16:23:40.333688: lr: 0.000227 +2022-05-14 16:23:40.333876: saving scheduled checkpoint file... +2022-05-14 16:23:40.378015: saving checkpoint... +2022-05-14 16:23:41.165080: done, saving took 0.83 seconds +2022-05-14 16:23:41.197351: done +2022-05-14 16:24:01.070414: This epoch took 484.343661 s + +2022-05-14 16:24:01.070816: +epoch: 89 +2022-05-14 16:31:18.276108: train loss : 0.0426 +2022-05-14 16:31:43.654245: validation loss: 0.0452 +2022-05-14 16:31:43.654840: Average global foreground Dice: [0.9513, 0.9409, 0.9275] +2022-05-14 16:31:43.654927: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 16:31:44.057087: lr: 0.000225 +2022-05-14 16:31:44.057271: saving scheduled checkpoint file... +2022-05-14 16:31:44.101837: saving checkpoint... +2022-05-14 16:31:44.896642: done, saving took 0.84 seconds +2022-05-14 16:31:44.913326: done +2022-05-14 16:31:44.933650: saving checkpoint... +2022-05-14 16:31:45.687619: done, saving took 0.77 seconds +2022-05-14 16:32:18.743294: This epoch took 497.672374 s + +2022-05-14 16:32:18.743660: +epoch: 90 +2022-05-14 16:39:36.605557: train loss : 0.0445 +2022-05-14 16:40:01.940727: validation loss: 0.0507 +2022-05-14 16:40:01.941675: Average global foreground Dice: [0.9525, 0.9433, 0.9321] +2022-05-14 16:40:01.941768: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 16:40:02.346711: lr: 0.000223 +2022-05-14 16:40:02.346933: saving scheduled checkpoint file... +2022-05-14 16:40:02.391659: saving checkpoint... +2022-05-14 16:40:03.172996: done, saving took 0.83 seconds +2022-05-14 16:40:03.191568: done +2022-05-14 16:40:03.214582: saving checkpoint... +2022-05-14 16:40:03.983820: done, saving took 0.79 seconds +2022-05-14 16:40:21.481714: This epoch took 482.737941 s + +2022-05-14 16:40:21.482015: +epoch: 91 +2022-05-14 16:47:39.527232: train loss : 0.0451 +2022-05-14 16:48:04.790200: validation loss: 0.0540 +2022-05-14 16:48:04.790874: Average global foreground Dice: [0.9487, 0.9385, 0.9238] +2022-05-14 16:48:04.790961: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 16:48:05.193815: lr: 0.000222 +2022-05-14 16:48:05.193995: saving scheduled checkpoint file... +2022-05-14 16:48:05.238801: saving checkpoint... +2022-05-14 16:48:06.021431: done, saving took 0.83 seconds +2022-05-14 16:48:06.040272: done +2022-05-14 16:48:28.071070: This epoch took 486.588957 s + +2022-05-14 16:48:28.071349: +epoch: 92 +2022-05-14 16:55:45.689987: train loss : 0.0436 +2022-05-14 16:56:11.003138: validation loss: 0.0479 +2022-05-14 16:56:11.003863: Average global foreground Dice: [0.9467, 0.9394, 0.9248] +2022-05-14 16:56:11.003953: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 16:56:11.404103: lr: 0.00022 +2022-05-14 16:56:11.404291: saving scheduled checkpoint file... +2022-05-14 16:56:11.451859: saving checkpoint... +2022-05-14 16:56:12.237915: done, saving took 0.83 seconds +2022-05-14 16:56:12.254511: done +2022-05-14 16:56:30.375086: This epoch took 482.303637 s + +2022-05-14 16:56:30.375365: +epoch: 93 +2022-05-14 17:03:48.366053: train loss : 0.0428 +2022-05-14 17:04:13.662735: validation loss: 0.0526 +2022-05-14 17:04:13.663405: Average global foreground Dice: [0.9517, 0.9442, 0.9246] +2022-05-14 17:04:13.663504: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 17:04:14.069619: lr: 0.000218 +2022-05-14 17:04:14.069927: saving scheduled checkpoint file... +2022-05-14 17:04:14.115640: saving checkpoint... +2022-05-14 17:04:14.909959: done, saving took 0.84 seconds +2022-05-14 17:04:14.929092: done +2022-05-14 17:04:33.019050: This epoch took 482.643588 s + +2022-05-14 17:04:33.019319: +epoch: 94 +2022-05-14 17:11:51.379601: train loss : 0.0447 +2022-05-14 17:12:16.786338: validation loss: 0.0511 +2022-05-14 17:12:16.787003: Average global foreground Dice: [0.9512, 0.9431, 0.9332] +2022-05-14 17:12:16.787099: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 17:12:17.189811: lr: 0.000216 +2022-05-14 17:12:17.190008: saving scheduled checkpoint file... +2022-05-14 17:12:17.234393: saving checkpoint... +2022-05-14 17:12:17.980851: done, saving took 0.79 seconds +2022-05-14 17:12:17.996695: done +2022-05-14 17:12:18.018176: saving checkpoint... +2022-05-14 17:12:18.800117: done, saving took 0.80 seconds +2022-05-14 17:12:35.431433: This epoch took 482.411992 s + +2022-05-14 17:12:35.431821: +epoch: 95 +2022-05-14 17:19:53.179739: train loss : 0.0438 +2022-05-14 17:20:18.453821: validation loss: 0.0624 +2022-05-14 17:20:18.454561: Average global foreground Dice: [0.9487, 0.9359, 0.9219] +2022-05-14 17:20:18.454650: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 17:20:18.855147: lr: 0.000215 +2022-05-14 17:20:18.855338: saving scheduled checkpoint file... +2022-05-14 17:20:18.899127: saving checkpoint... +2022-05-14 17:20:19.684546: done, saving took 0.83 seconds +2022-05-14 17:20:19.703338: done +2022-05-14 17:20:37.591533: This epoch took 482.159612 s + +2022-05-14 17:20:37.591791: +epoch: 96 +2022-05-14 17:27:54.717610: train loss : 0.0453 +2022-05-14 17:28:19.969751: validation loss: 0.0449 +2022-05-14 17:28:19.970414: Average global foreground Dice: [0.9445, 0.9436, 0.9305] +2022-05-14 17:28:19.970508: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 17:28:20.396883: lr: 0.000213 +2022-05-14 17:28:20.397078: saving scheduled checkpoint file... +2022-05-14 17:28:20.441147: saving checkpoint... +2022-05-14 17:28:21.223048: done, saving took 0.83 seconds +2022-05-14 17:28:21.241613: done +2022-05-14 17:28:39.899147: This epoch took 482.307241 s + +2022-05-14 17:28:39.899421: +epoch: 97 +2022-05-14 17:35:58.063870: train loss : 0.0446 +2022-05-14 17:36:23.433297: validation loss: 0.0634 +2022-05-14 17:36:23.434034: Average global foreground Dice: [0.949, 0.9434, 0.9254] +2022-05-14 17:36:23.434121: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 17:36:23.850872: lr: 0.000211 +2022-05-14 17:36:23.851062: saving scheduled checkpoint file... +2022-05-14 17:36:23.896290: saving checkpoint... +2022-05-14 17:36:24.684930: done, saving took 0.83 seconds +2022-05-14 17:36:24.703685: done +2022-05-14 17:36:44.125146: This epoch took 484.225595 s + +2022-05-14 17:36:44.125430: +epoch: 98 +2022-05-14 17:44:02.238503: train loss : 0.0435 +2022-05-14 17:44:27.588190: validation loss: 0.0595 +2022-05-14 17:44:27.588959: Average global foreground Dice: [0.9514, 0.9403, 0.9292] +2022-05-14 17:44:27.589048: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 17:44:27.994549: lr: 0.000209 +2022-05-14 17:44:27.994736: saving scheduled checkpoint file... +2022-05-14 17:44:28.039507: saving checkpoint... +2022-05-14 17:44:28.832627: done, saving took 0.84 seconds +2022-05-14 17:44:28.851250: done +2022-05-14 17:44:28.874142: saving checkpoint... +2022-05-14 17:44:29.653532: done, saving took 0.80 seconds +2022-05-14 17:44:50.722816: This epoch took 486.597116 s + +2022-05-14 17:44:50.723195: +epoch: 99 +2022-05-14 17:52:08.451626: train loss : 0.0440 +2022-05-14 17:52:33.776423: validation loss: 0.0532 +2022-05-14 17:52:33.777064: Average global foreground Dice: [0.9534, 0.9381, 0.9302] +2022-05-14 17:52:33.777151: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 17:52:34.179143: lr: 0.000207 +2022-05-14 17:52:34.179325: saving scheduled checkpoint file... +2022-05-14 17:52:34.224656: saving checkpoint... +2022-05-14 17:52:35.017083: done, saving took 0.84 seconds +2022-05-14 17:52:35.035763: done +2022-05-14 17:52:35.058663: saving checkpoint... +2022-05-14 17:52:35.833864: done, saving took 0.80 seconds +2022-05-14 17:52:51.246891: This epoch took 480.523567 s + +2022-05-14 17:52:51.247270: +epoch: 100 +2022-05-14 18:00:08.952713: train loss : 0.0458 +2022-05-14 18:00:34.337044: validation loss: 0.0505 +2022-05-14 18:00:34.337702: Average global foreground Dice: [0.9511, 0.9412, 0.9331] +2022-05-14 18:00:34.337790: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 18:00:34.749357: lr: 0.000206 +2022-05-14 18:00:34.749554: saving scheduled checkpoint file... +2022-05-14 18:00:34.794214: saving checkpoint... +2022-05-14 18:00:35.588921: done, saving took 0.84 seconds +2022-05-14 18:00:35.608606: done +2022-05-14 18:00:35.631358: saving checkpoint... +2022-05-14 18:00:36.401681: done, saving took 0.79 seconds +2022-05-14 18:00:54.512241: This epoch took 483.264872 s + +2022-05-14 18:00:54.512520: +epoch: 101 +2022-05-14 18:08:12.561750: train loss : 0.0440 +2022-05-14 18:08:37.910504: validation loss: 0.0557 +2022-05-14 18:08:37.911175: Average global foreground Dice: [0.9493, 0.9439, 0.9332] +2022-05-14 18:08:37.911263: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 18:08:38.329933: lr: 0.000204 +2022-05-14 18:08:38.330132: saving scheduled checkpoint file... +2022-05-14 18:08:38.374497: saving checkpoint... +2022-05-14 18:08:39.159160: done, saving took 0.83 seconds +2022-05-14 18:08:39.177647: done +2022-05-14 18:08:39.200571: saving checkpoint... +2022-05-14 18:08:39.975773: done, saving took 0.80 seconds +2022-05-14 18:08:57.169542: This epoch took 482.656914 s + +2022-05-14 18:08:57.169814: +epoch: 102 +2022-05-14 18:16:14.661502: train loss : 0.0437 +2022-05-14 18:16:39.930254: validation loss: 0.0564 +2022-05-14 18:16:39.930859: Average global foreground Dice: [0.9503, 0.9386, 0.9321] +2022-05-14 18:16:39.930944: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 18:16:40.336823: lr: 0.000202 +2022-05-14 18:16:40.337020: saving scheduled checkpoint file... +2022-05-14 18:16:40.381876: saving checkpoint... +2022-05-14 18:16:41.187754: done, saving took 0.85 seconds +2022-05-14 18:16:41.206670: done +2022-05-14 18:16:41.230157: saving checkpoint... +2022-05-14 18:16:41.998846: done, saving took 0.79 seconds +2022-05-14 18:16:59.713448: This epoch took 482.543510 s + +2022-05-14 18:16:59.713711: +epoch: 103 +2022-05-14 18:24:17.086981: train loss : 0.0408 +2022-05-14 18:24:42.383466: validation loss: 0.0533 +2022-05-14 18:24:42.384188: Average global foreground Dice: [0.9504, 0.9369, 0.9286] +2022-05-14 18:24:42.384275: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 18:24:42.792812: lr: 0.0002 +2022-05-14 18:24:42.793000: saving scheduled checkpoint file... +2022-05-14 18:24:42.838056: saving checkpoint... +2022-05-14 18:24:43.593779: done, saving took 0.80 seconds +2022-05-14 18:24:43.610859: done +2022-05-14 18:25:01.617647: This epoch took 481.903814 s + +2022-05-14 18:25:01.618001: +epoch: 104 +2022-05-14 18:32:19.446862: train loss : 0.0470 +2022-05-14 18:32:44.757066: validation loss: 0.0570 +2022-05-14 18:32:44.757712: Average global foreground Dice: [0.9477, 0.938, 0.9255] +2022-05-14 18:32:44.757869: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 18:32:45.166428: lr: 0.000198 +2022-05-14 18:32:45.166626: saving scheduled checkpoint file... +2022-05-14 18:32:45.211769: saving checkpoint... +2022-05-14 18:32:46.003121: done, saving took 0.84 seconds +2022-05-14 18:32:46.022593: done +2022-05-14 18:33:05.350712: This epoch took 483.732595 s + +2022-05-14 18:33:05.350996: +epoch: 105 +2022-05-14 18:40:22.550561: train loss : 0.0428 +2022-05-14 18:40:47.939624: validation loss: 0.0446 +2022-05-14 18:40:47.940200: Average global foreground Dice: [0.9487, 0.9438, 0.9305] +2022-05-14 18:40:47.940288: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 18:40:48.345346: lr: 0.000196 +2022-05-14 18:40:48.345546: saving scheduled checkpoint file... +2022-05-14 18:40:48.389688: saving checkpoint... +2022-05-14 18:40:49.131420: done, saving took 0.79 seconds +2022-05-14 18:40:49.149231: done +2022-05-14 18:41:07.849770: This epoch took 482.498669 s + +2022-05-14 18:41:07.850033: +epoch: 106 +2022-05-14 18:48:26.630122: train loss : 0.0433 +2022-05-14 18:48:52.026627: validation loss: 0.0537 +2022-05-14 18:48:52.027239: Average global foreground Dice: [0.9512, 0.942, 0.9295] +2022-05-14 18:48:52.027326: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 18:48:52.432362: lr: 0.000194 +2022-05-14 18:48:52.432565: saving scheduled checkpoint file... +2022-05-14 18:48:52.477188: saving checkpoint... +2022-05-14 18:48:53.264404: done, saving took 0.83 seconds +2022-05-14 18:48:53.282901: done +2022-05-14 18:49:12.192074: This epoch took 484.341933 s + +2022-05-14 18:49:12.192347: +epoch: 107 +2022-05-14 18:56:29.985817: train loss : 0.0443 +2022-05-14 18:56:55.382640: validation loss: 0.0487 +2022-05-14 18:56:55.383269: Average global foreground Dice: [0.9522, 0.9362, 0.9287] +2022-05-14 18:56:55.383353: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 18:56:55.788121: lr: 0.000193 +2022-05-14 18:56:55.788337: saving scheduled checkpoint file... +2022-05-14 18:56:55.833908: saving checkpoint... +2022-05-14 18:56:56.541380: done, saving took 0.75 seconds +2022-05-14 18:56:56.558420: done +2022-05-14 18:57:12.073995: This epoch took 479.881547 s + +2022-05-14 18:57:12.074275: +epoch: 108 +2022-05-14 19:04:29.595893: train loss : 0.0443 +2022-05-14 19:04:54.969508: validation loss: 0.0478 +2022-05-14 19:04:54.970199: Average global foreground Dice: [0.9505, 0.9421, 0.9268] +2022-05-14 19:04:54.970290: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 19:04:55.385762: lr: 0.000191 +2022-05-14 19:04:55.385946: saving scheduled checkpoint file... +2022-05-14 19:04:55.430453: saving checkpoint... +2022-05-14 19:04:56.155942: done, saving took 0.77 seconds +2022-05-14 19:04:56.172378: done +2022-05-14 19:05:23.640240: This epoch took 491.565864 s + +2022-05-14 19:05:23.640701: +epoch: 109 +2022-05-14 19:12:40.816517: train loss : 0.0444 +2022-05-14 19:13:06.180064: validation loss: 0.0365 +2022-05-14 19:13:06.180698: Average global foreground Dice: [0.9503, 0.953, 0.9334] +2022-05-14 19:13:06.180784: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 19:13:06.598911: lr: 0.000189 +2022-05-14 19:13:06.599105: saving scheduled checkpoint file... +2022-05-14 19:13:06.644890: saving checkpoint... +2022-05-14 19:13:07.410319: done, saving took 0.81 seconds +2022-05-14 19:13:07.438734: done +2022-05-14 19:13:07.479485: saving checkpoint... +2022-05-14 19:13:08.181024: done, saving took 0.74 seconds +2022-05-14 19:13:31.483203: This epoch took 487.842378 s + +2022-05-14 19:13:31.483492: +epoch: 110 +2022-05-14 19:20:49.345474: train loss : 0.0410 +2022-05-14 19:21:14.649899: validation loss: 0.0452 +2022-05-14 19:21:14.650515: Average global foreground Dice: [0.9498, 0.9433, 0.9233] +2022-05-14 19:21:14.650603: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 19:21:15.056645: lr: 0.000187 +2022-05-14 19:21:15.056834: saving scheduled checkpoint file... +2022-05-14 19:21:15.100686: saving checkpoint... +2022-05-14 19:21:15.829588: done, saving took 0.77 seconds +2022-05-14 19:21:15.854679: done +2022-05-14 19:21:34.847736: This epoch took 483.364139 s + +2022-05-14 19:21:34.848022: +epoch: 111 +2022-05-14 19:28:52.537192: train loss : 0.0405 +2022-05-14 19:29:17.877825: validation loss: 0.0513 +2022-05-14 19:29:17.878487: Average global foreground Dice: [0.9519, 0.9381, 0.9257] +2022-05-14 19:29:17.878574: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 19:29:18.284158: lr: 0.000185 +2022-05-14 19:29:18.284346: saving scheduled checkpoint file... +2022-05-14 19:29:18.328271: saving checkpoint... +2022-05-14 19:29:19.047858: done, saving took 0.76 seconds +2022-05-14 19:29:19.067645: done +2022-05-14 19:29:39.835734: This epoch took 484.987609 s + +2022-05-14 19:29:39.836005: +epoch: 112 +2022-05-14 19:36:56.688901: train loss : 0.0403 +2022-05-14 19:37:22.019269: validation loss: 0.0504 +2022-05-14 19:37:22.019986: Average global foreground Dice: [0.9487, 0.9368, 0.9279] +2022-05-14 19:37:22.020080: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 19:37:22.426728: lr: 0.000183 +2022-05-14 19:37:22.426931: saving scheduled checkpoint file... +2022-05-14 19:37:22.471257: saving checkpoint... +2022-05-14 19:37:23.252512: done, saving took 0.83 seconds +2022-05-14 19:37:23.273909: done +2022-05-14 19:37:39.466250: This epoch took 479.630128 s + +2022-05-14 19:37:39.466565: +epoch: 113 +2022-05-14 19:44:56.207059: train loss : 0.0426 +2022-05-14 19:45:21.490927: validation loss: 0.0429 +2022-05-14 19:45:21.491595: Average global foreground Dice: [0.9502, 0.9482, 0.9267] +2022-05-14 19:45:21.491683: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 19:45:21.895045: lr: 0.000181 +2022-05-14 19:45:21.895247: saving scheduled checkpoint file... +2022-05-14 19:45:21.940486: saving checkpoint... +2022-05-14 19:45:22.673149: done, saving took 0.78 seconds +2022-05-14 19:45:22.693222: done +2022-05-14 19:45:45.101275: This epoch took 485.634601 s + +2022-05-14 19:45:45.101617: +epoch: 114 +2022-05-14 19:53:02.833991: train loss : 0.0412 +2022-05-14 19:53:28.171635: validation loss: 0.0455 +2022-05-14 19:53:28.172325: Average global foreground Dice: [0.9539, 0.9498, 0.9296] +2022-05-14 19:53:28.172412: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 19:53:28.574672: lr: 0.000179 +2022-05-14 19:53:28.574875: saving scheduled checkpoint file... +2022-05-14 19:53:28.618596: saving checkpoint... +2022-05-14 19:53:29.397664: done, saving took 0.82 seconds +2022-05-14 19:53:29.417808: done +2022-05-14 19:53:29.440236: saving checkpoint... +2022-05-14 19:53:30.214152: done, saving took 0.80 seconds +2022-05-14 19:53:47.462290: This epoch took 482.360572 s + +2022-05-14 19:53:47.462571: +epoch: 115 +2022-05-14 20:01:04.837460: train loss : 0.0448 +2022-05-14 20:01:30.170500: validation loss: 0.0489 +2022-05-14 20:01:30.171127: Average global foreground Dice: [0.9492, 0.9434, 0.9286] +2022-05-14 20:01:30.171212: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 20:01:30.590081: lr: 0.000177 +2022-05-14 20:01:30.590274: saving scheduled checkpoint file... +2022-05-14 20:01:30.635188: saving checkpoint... +2022-05-14 20:01:31.414712: done, saving took 0.82 seconds +2022-05-14 20:01:31.433762: done +2022-05-14 20:01:31.457579: saving checkpoint... +2022-05-14 20:01:32.249051: done, saving took 0.81 seconds +2022-05-14 20:01:50.393062: This epoch took 482.930385 s + +2022-05-14 20:01:50.393371: +epoch: 116 +2022-05-14 20:09:07.959917: train loss : 0.0425 +2022-05-14 20:09:33.345048: validation loss: 0.0438 +2022-05-14 20:09:33.345676: Average global foreground Dice: [0.9487, 0.9402, 0.924] +2022-05-14 20:09:33.345766: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 20:09:33.754177: lr: 0.000175 +2022-05-14 20:09:33.754373: saving scheduled checkpoint file... +2022-05-14 20:09:33.798792: saving checkpoint... +2022-05-14 20:09:34.589102: done, saving took 0.83 seconds +2022-05-14 20:09:34.607966: done +2022-05-14 20:09:52.545753: This epoch took 482.152277 s + +2022-05-14 20:09:52.546020: +epoch: 117 +2022-05-14 20:17:10.195303: train loss : 0.0442 +2022-05-14 20:17:35.521090: validation loss: 0.0454 +2022-05-14 20:17:35.521734: Average global foreground Dice: [0.9472, 0.9392, 0.9269] +2022-05-14 20:17:35.521825: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 20:17:35.927339: lr: 0.000173 +2022-05-14 20:17:35.927523: saving scheduled checkpoint file... +2022-05-14 20:17:35.971287: saving checkpoint... +2022-05-14 20:17:36.752460: done, saving took 0.82 seconds +2022-05-14 20:17:36.771019: done +2022-05-14 20:18:00.083695: This epoch took 487.537576 s + +2022-05-14 20:18:00.083963: +epoch: 118 +2022-05-14 20:25:18.027964: train loss : 0.0419 +2022-05-14 20:25:43.319504: validation loss: 0.0493 +2022-05-14 20:25:43.320170: Average global foreground Dice: [0.9501, 0.9385, 0.9249] +2022-05-14 20:25:43.320257: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 20:25:43.731709: lr: 0.000172 +2022-05-14 20:25:43.731902: saving scheduled checkpoint file... +2022-05-14 20:25:43.775956: saving checkpoint... +2022-05-14 20:25:44.519362: done, saving took 0.79 seconds +2022-05-14 20:25:44.535509: done +2022-05-14 20:26:01.247957: This epoch took 481.163893 s + +2022-05-14 20:26:01.248233: +epoch: 119 +2022-05-14 20:33:18.556699: train loss : 0.0408 +2022-05-14 20:33:43.943699: validation loss: 0.0549 +2022-05-14 20:33:43.944341: Average global foreground Dice: [0.9505, 0.9411, 0.9286] +2022-05-14 20:33:43.944429: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 20:33:44.353205: lr: 0.00017 +2022-05-14 20:33:44.353411: saving scheduled checkpoint file... +2022-05-14 20:33:44.397669: saving checkpoint... +2022-05-14 20:33:45.193272: done, saving took 0.84 seconds +2022-05-14 20:33:45.212544: done +2022-05-14 20:34:05.015612: This epoch took 483.767277 s + +2022-05-14 20:34:05.015886: +epoch: 120 +2022-05-14 20:41:22.299595: train loss : 0.0402 +2022-05-14 20:41:47.648621: validation loss: 0.0540 +2022-05-14 20:41:47.649250: Average global foreground Dice: [0.9497, 0.9404, 0.9272] +2022-05-14 20:41:47.649335: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 20:41:48.073295: lr: 0.000168 +2022-05-14 20:41:48.073497: saving scheduled checkpoint file... +2022-05-14 20:41:48.117190: saving checkpoint... +2022-05-14 20:41:48.901789: done, saving took 0.83 seconds +2022-05-14 20:41:48.920543: done +2022-05-14 20:42:11.101526: This epoch took 486.085507 s + +2022-05-14 20:42:11.101999: +epoch: 121 +2022-05-14 20:49:28.492283: train loss : 0.0401 +2022-05-14 20:49:53.880024: validation loss: 0.0606 +2022-05-14 20:49:53.880676: Average global foreground Dice: [0.9466, 0.9367, 0.9223] +2022-05-14 20:49:53.880769: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 20:49:54.290401: lr: 0.000166 +2022-05-14 20:49:54.290606: saving scheduled checkpoint file... +2022-05-14 20:49:54.335202: saving checkpoint... +2022-05-14 20:49:55.123247: done, saving took 0.83 seconds +2022-05-14 20:49:55.142275: done +2022-05-14 20:50:14.214323: This epoch took 483.112214 s + +2022-05-14 20:50:14.214655: +epoch: 122 +2022-05-14 20:57:30.829515: train loss : 0.0440 +2022-05-14 20:57:56.259802: validation loss: 0.0543 +2022-05-14 20:57:56.260529: Average global foreground Dice: [0.9479, 0.9343, 0.9213] +2022-05-14 20:57:56.260618: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 20:57:56.669525: lr: 0.000164 +2022-05-14 20:57:56.669717: saving scheduled checkpoint file... +2022-05-14 20:57:56.715182: saving checkpoint... +2022-05-14 20:57:57.517555: done, saving took 0.85 seconds +2022-05-14 20:57:57.534122: done +2022-05-14 20:58:13.733253: This epoch took 479.518497 s + +2022-05-14 20:58:13.733557: +epoch: 123 +2022-05-14 21:05:31.062656: train loss : 0.0415 +2022-05-14 21:05:56.344294: validation loss: 0.0471 +2022-05-14 21:05:56.344970: Average global foreground Dice: [0.9476, 0.9281, 0.9274] +2022-05-14 21:05:56.345058: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 21:05:56.753529: lr: 0.000162 +2022-05-14 21:05:56.753777: saving scheduled checkpoint file... +2022-05-14 21:05:56.797625: saving checkpoint... +2022-05-14 21:05:57.589088: done, saving took 0.84 seconds +2022-05-14 21:05:57.607453: done +2022-05-14 21:06:19.079616: This epoch took 485.345949 s + +2022-05-14 21:06:19.079884: +epoch: 124 +2022-05-14 21:13:36.309296: train loss : 0.0412 +2022-05-14 21:14:01.743688: validation loss: 0.0507 +2022-05-14 21:14:01.744364: Average global foreground Dice: [0.9508, 0.9427, 0.9286] +2022-05-14 21:14:01.744463: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 21:14:02.150629: lr: 0.00016 +2022-05-14 21:14:02.150817: saving scheduled checkpoint file... +2022-05-14 21:14:02.195562: saving checkpoint... +2022-05-14 21:14:02.981205: done, saving took 0.83 seconds +2022-05-14 21:14:02.999924: done +2022-05-14 21:14:23.761022: This epoch took 484.681036 s + +2022-05-14 21:14:23.761302: +epoch: 125 +2022-05-14 21:21:40.672034: train loss : 0.0392 +2022-05-14 21:22:06.108757: validation loss: 0.0455 +2022-05-14 21:22:06.109385: Average global foreground Dice: [0.9493, 0.9342, 0.9231] +2022-05-14 21:22:06.109482: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 21:22:06.518692: lr: 0.000158 +2022-05-14 21:22:06.518893: saving scheduled checkpoint file... +2022-05-14 21:22:06.563766: saving checkpoint... +2022-05-14 21:22:07.364356: done, saving took 0.85 seconds +2022-05-14 21:22:07.382982: done +2022-05-14 21:22:23.623019: This epoch took 479.861617 s + +2022-05-14 21:22:23.623291: +epoch: 126 +2022-05-14 21:29:41.357157: train loss : 0.0416 +2022-05-14 21:30:06.746773: validation loss: 0.0410 +2022-05-14 21:30:06.747412: Average global foreground Dice: [0.9507, 0.9467, 0.9321] +2022-05-14 21:30:06.747508: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 21:30:07.151374: lr: 0.000156 +2022-05-14 21:30:07.151583: saving scheduled checkpoint file... +2022-05-14 21:30:07.195484: saving checkpoint... +2022-05-14 21:30:07.970893: done, saving took 0.82 seconds +2022-05-14 21:30:07.994120: done +2022-05-14 21:30:23.340000: This epoch took 479.716604 s + +2022-05-14 21:30:23.340298: +epoch: 127 +2022-05-14 21:37:40.672477: train loss : 0.0402 +2022-05-14 21:38:06.031253: validation loss: 0.0562 +2022-05-14 21:38:06.031974: Average global foreground Dice: [0.9496, 0.9453, 0.9234] +2022-05-14 21:38:06.032062: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 21:38:06.503195: lr: 0.000154 +2022-05-14 21:38:06.503386: saving scheduled checkpoint file... +2022-05-14 21:38:06.551742: saving checkpoint... +2022-05-14 21:38:07.398662: done, saving took 0.90 seconds +2022-05-14 21:38:07.418699: done +2022-05-14 21:38:51.071268: This epoch took 507.730835 s + +2022-05-14 21:38:51.071683: +epoch: 128 +2022-05-14 21:46:07.765512: train loss : 0.0406 +2022-05-14 21:46:33.093122: validation loss: 0.0524 +2022-05-14 21:46:33.093743: Average global foreground Dice: [0.949, 0.9452, 0.9297] +2022-05-14 21:46:33.093829: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 21:46:33.498965: lr: 0.000152 +2022-05-14 21:46:33.499147: saving scheduled checkpoint file... +2022-05-14 21:46:33.543236: saving checkpoint... +2022-05-14 21:46:34.335975: done, saving took 0.84 seconds +2022-05-14 21:46:34.354420: done +2022-05-14 21:46:50.888563: This epoch took 479.816754 s + +2022-05-14 21:46:50.888848: +epoch: 129 +2022-05-14 21:54:08.307096: train loss : 0.0398 +2022-05-14 21:54:33.750595: validation loss: 0.0501 +2022-05-14 21:54:33.751273: Average global foreground Dice: [0.9515, 0.9354, 0.927] +2022-05-14 21:54:33.751359: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 21:54:34.158402: lr: 0.00015 +2022-05-14 21:54:34.158598: saving scheduled checkpoint file... +2022-05-14 21:54:34.203126: saving checkpoint... +2022-05-14 21:54:34.990723: done, saving took 0.83 seconds +2022-05-14 21:54:35.009037: done +2022-05-14 21:54:54.143471: This epoch took 483.254514 s + +2022-05-14 21:54:54.143739: +epoch: 130 +2022-05-14 22:02:10.569392: train loss : 0.0393 +2022-05-14 22:02:35.974606: validation loss: 0.0509 +2022-05-14 22:02:35.975334: Average global foreground Dice: [0.9507, 0.9416, 0.9323] +2022-05-14 22:02:35.975424: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 22:02:36.387579: lr: 0.000148 +2022-05-14 22:02:36.387781: saving scheduled checkpoint file... +2022-05-14 22:02:36.432494: saving checkpoint... +2022-05-14 22:02:37.216626: done, saving took 0.83 seconds +2022-05-14 22:02:37.236840: done +2022-05-14 22:02:55.094742: This epoch took 480.950899 s + +2022-05-14 22:02:55.095017: +epoch: 131 +2022-05-14 22:10:10.954128: train loss : 0.0407 +2022-05-14 22:10:36.258927: validation loss: 0.0446 +2022-05-14 22:10:36.259572: Average global foreground Dice: [0.9506, 0.9329, 0.9243] +2022-05-14 22:10:36.259662: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 22:10:36.666077: lr: 0.000146 +2022-05-14 22:10:36.666272: saving scheduled checkpoint file... +2022-05-14 22:10:36.711235: saving checkpoint... +2022-05-14 22:10:37.522667: done, saving took 0.86 seconds +2022-05-14 22:10:37.541363: done +2022-05-14 22:10:58.160858: This epoch took 483.065738 s + +2022-05-14 22:10:58.161137: +epoch: 132 +2022-05-14 22:18:15.706262: train loss : 0.0421 +2022-05-14 22:18:41.078923: validation loss: 0.0561 +2022-05-14 22:18:41.079591: Average global foreground Dice: [0.9506, 0.9422, 0.9273] +2022-05-14 22:18:41.079683: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 22:18:41.489013: lr: 0.000144 +2022-05-14 22:18:41.489208: saving scheduled checkpoint file... +2022-05-14 22:18:41.533607: saving checkpoint... +2022-05-14 22:18:42.326282: done, saving took 0.84 seconds +2022-05-14 22:18:42.345022: done +2022-05-14 22:19:01.947840: This epoch took 483.786596 s + +2022-05-14 22:19:01.948117: +epoch: 133 +2022-05-14 22:26:19.261096: train loss : 0.0423 +2022-05-14 22:26:44.638458: validation loss: 0.0561 +2022-05-14 22:26:44.639106: Average global foreground Dice: [0.9473, 0.9426, 0.9272] +2022-05-14 22:26:44.639192: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 22:26:45.046021: lr: 0.000142 +2022-05-14 22:26:45.046209: saving scheduled checkpoint file... +2022-05-14 22:26:45.091363: saving checkpoint... +2022-05-14 22:26:45.885221: done, saving took 0.84 seconds +2022-05-14 22:26:45.905345: done +2022-05-14 22:27:01.134072: This epoch took 479.185852 s + +2022-05-14 22:27:01.134362: +epoch: 134 +2022-05-14 22:34:17.969689: train loss : 0.0431 +2022-05-14 22:34:43.374451: validation loss: 0.0460 +2022-05-14 22:34:43.375248: Average global foreground Dice: [0.9524, 0.9478, 0.9319] +2022-05-14 22:34:43.375334: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 22:34:43.788860: lr: 0.00014 +2022-05-14 22:34:43.789053: saving scheduled checkpoint file... +2022-05-14 22:34:43.833744: saving checkpoint... +2022-05-14 22:34:44.627617: done, saving took 0.84 seconds +2022-05-14 22:34:44.646016: done +2022-05-14 22:35:02.266201: This epoch took 481.131735 s + +2022-05-14 22:35:02.266479: +epoch: 135 +2022-05-14 22:42:20.179601: train loss : 0.0415 +2022-05-14 22:42:45.449569: validation loss: 0.0467 +2022-05-14 22:42:45.450321: Average global foreground Dice: [0.9522, 0.9422, 0.9298] +2022-05-14 22:42:45.450411: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 22:42:45.859643: lr: 0.000138 +2022-05-14 22:42:45.859831: saving scheduled checkpoint file... +2022-05-14 22:42:45.904354: saving checkpoint... +2022-05-14 22:42:46.693745: done, saving took 0.83 seconds +2022-05-14 22:42:46.712786: done +2022-05-14 22:43:04.498917: This epoch took 482.232311 s + +2022-05-14 22:43:04.499368: +epoch: 136 +2022-05-14 22:50:21.950510: train loss : 0.0400 +2022-05-14 22:50:47.281422: validation loss: 0.0487 +2022-05-14 22:50:47.282167: Average global foreground Dice: [0.95, 0.9453, 0.9275] +2022-05-14 22:50:47.282253: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 22:50:47.710514: lr: 0.000136 +2022-05-14 22:50:47.710693: saving scheduled checkpoint file... +2022-05-14 22:50:47.755569: saving checkpoint... +2022-05-14 22:50:48.547561: done, saving took 0.84 seconds +2022-05-14 22:50:48.564529: done +2022-05-14 22:51:07.854475: This epoch took 483.354968 s + +2022-05-14 22:51:07.854842: +epoch: 137 +2022-05-14 22:58:25.595122: train loss : 0.0407 +2022-05-14 22:58:50.953539: validation loss: 0.0621 +2022-05-14 22:58:50.954202: Average global foreground Dice: [0.9453, 0.9207, 0.916] +2022-05-14 22:58:50.954287: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 22:58:51.366586: lr: 0.000134 +2022-05-14 22:58:51.366769: saving scheduled checkpoint file... +2022-05-14 22:58:51.411783: saving checkpoint... +2022-05-14 22:58:52.207563: done, saving took 0.84 seconds +2022-05-14 22:58:52.231258: done +2022-05-14 22:59:06.648940: This epoch took 478.793991 s + +2022-05-14 22:59:06.649228: +epoch: 138 +2022-05-14 23:06:22.492086: train loss : 0.0400 +2022-05-14 23:06:47.900572: validation loss: 0.0445 +2022-05-14 23:06:47.901244: Average global foreground Dice: [0.9521, 0.9471, 0.9292] +2022-05-14 23:06:47.901331: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 23:06:48.324791: lr: 0.000132 +2022-05-14 23:06:48.324970: saving scheduled checkpoint file... +2022-05-14 23:06:48.368941: saving checkpoint... +2022-05-14 23:06:49.156374: done, saving took 0.83 seconds +2022-05-14 23:06:49.175225: done +2022-05-14 23:07:04.917987: This epoch took 478.268655 s + +2022-05-14 23:07:04.918264: +epoch: 139 +2022-05-14 23:14:22.168486: train loss : 0.0414 +2022-05-14 23:14:47.493834: validation loss: 0.0566 +2022-05-14 23:14:47.494524: Average global foreground Dice: [0.9485, 0.9392, 0.9242] +2022-05-14 23:14:47.494613: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 23:14:47.909601: lr: 0.00013 +2022-05-14 23:14:47.909794: saving scheduled checkpoint file... +2022-05-14 23:14:47.954415: saving checkpoint... +2022-05-14 23:14:48.748014: done, saving took 0.84 seconds +2022-05-14 23:14:48.767409: done +2022-05-14 23:15:05.685289: This epoch took 480.766927 s + +2022-05-14 23:15:05.685599: +epoch: 140 +2022-05-14 23:22:23.318525: train loss : 0.0403 +2022-05-14 23:22:48.865201: validation loss: 0.0491 +2022-05-14 23:22:48.865830: Average global foreground Dice: [0.9497, 0.9467, 0.9287] +2022-05-14 23:22:48.865915: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 23:22:49.275163: lr: 0.000128 +2022-05-14 23:22:49.275352: saving scheduled checkpoint file... +2022-05-14 23:22:49.321059: saving checkpoint... +2022-05-14 23:22:50.123903: done, saving took 0.85 seconds +2022-05-14 23:22:50.142274: done +2022-05-14 23:23:08.243169: This epoch took 482.557466 s + +2022-05-14 23:23:08.243460: +epoch: 141 +2022-05-14 23:30:25.234710: train loss : 0.0420 +2022-05-14 23:30:50.663962: validation loss: 0.0402 +2022-05-14 23:30:50.664611: Average global foreground Dice: [0.951, 0.9443, 0.9273] +2022-05-14 23:30:50.664698: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 23:30:51.073689: lr: 0.000127 +2022-05-14 23:30:51.073874: saving scheduled checkpoint file... +2022-05-14 23:30:51.119247: saving checkpoint... +2022-05-14 23:30:51.749266: done, saving took 0.68 seconds +2022-05-14 23:30:51.765307: done +2022-05-14 23:31:12.065432: This epoch took 483.821865 s + +2022-05-14 23:31:12.065728: +epoch: 142 +2022-05-14 23:38:29.198951: train loss : 0.0421 +2022-05-14 23:38:54.548790: validation loss: 0.0470 +2022-05-14 23:38:54.549447: Average global foreground Dice: [0.953, 0.9393, 0.9251] +2022-05-14 23:38:54.549536: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 23:38:54.959526: lr: 0.000125 +2022-05-14 23:38:54.959726: saving scheduled checkpoint file... +2022-05-14 23:38:55.004905: saving checkpoint... +2022-05-14 23:38:55.732080: done, saving took 0.77 seconds +2022-05-14 23:38:55.747998: done +2022-05-14 23:39:10.892483: This epoch took 478.826624 s + +2022-05-14 23:39:10.892878: +epoch: 143 +2022-05-14 23:46:27.983149: train loss : 0.0398 +2022-05-14 23:46:53.304445: validation loss: 0.0514 +2022-05-14 23:46:53.305042: Average global foreground Dice: [0.9543, 0.943, 0.9255] +2022-05-14 23:46:53.305128: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 23:46:53.720562: lr: 0.000123 +2022-05-14 23:46:53.720760: saving scheduled checkpoint file... +2022-05-14 23:46:53.765592: saving checkpoint... +2022-05-14 23:46:54.523563: done, saving took 0.80 seconds +2022-05-14 23:46:54.539515: done +2022-05-14 23:47:11.149003: This epoch took 480.256021 s + +2022-05-14 23:47:11.149283: +epoch: 144 +2022-05-14 23:54:28.717656: train loss : 0.0387 +2022-05-14 23:54:54.061384: validation loss: 0.0533 +2022-05-14 23:54:54.062104: Average global foreground Dice: [0.9517, 0.9444, 0.9243] +2022-05-14 23:54:54.062192: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-14 23:54:54.524286: lr: 0.000121 +2022-05-14 23:54:54.524504: saving scheduled checkpoint file... +2022-05-14 23:54:54.572919: saving checkpoint... +2022-05-14 23:54:55.413326: done, saving took 0.89 seconds +2022-05-14 23:54:55.431092: done +2022-05-14 23:55:13.047709: This epoch took 481.898322 s + +2022-05-14 23:55:13.047964: +epoch: 145 +2022-05-15 00:02:29.386811: train loss : 0.0379 +2022-05-15 00:02:54.763837: validation loss: 0.0395 +2022-05-15 00:02:54.764519: Average global foreground Dice: [0.9528, 0.9451, 0.9285] +2022-05-15 00:02:54.764607: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 00:02:55.175569: lr: 0.000119 +2022-05-15 00:02:55.175764: saving scheduled checkpoint file... +2022-05-15 00:02:55.220684: saving checkpoint... +2022-05-15 00:02:55.919666: done, saving took 0.74 seconds +2022-05-15 00:02:55.938727: done +2022-05-15 00:03:13.580787: This epoch took 480.532716 s + +2022-05-15 00:03:13.581095: +epoch: 146 +2022-05-15 00:10:31.028904: train loss : 0.0406 +2022-05-15 00:10:56.357912: validation loss: 0.0489 +2022-05-15 00:10:56.358542: Average global foreground Dice: [0.952, 0.9415, 0.9299] +2022-05-15 00:10:56.358632: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 00:10:56.776198: lr: 0.000117 +2022-05-15 00:10:56.776400: saving scheduled checkpoint file... +2022-05-15 00:10:56.823507: saving checkpoint... +2022-05-15 00:10:57.552324: done, saving took 0.78 seconds +2022-05-15 00:10:57.569053: done +2022-05-15 00:11:16.740656: This epoch took 483.159459 s + +2022-05-15 00:11:16.740925: +epoch: 147 +2022-05-15 00:18:34.417730: train loss : 0.0404 +2022-05-15 00:18:59.755044: validation loss: 0.0598 +2022-05-15 00:18:59.755814: Average global foreground Dice: [0.9513, 0.9426, 0.9236] +2022-05-15 00:18:59.755906: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 00:19:00.176612: lr: 0.000115 +2022-05-15 00:19:00.176814: saving scheduled checkpoint file... +2022-05-15 00:19:00.221192: saving checkpoint... +2022-05-15 00:19:00.947084: done, saving took 0.77 seconds +2022-05-15 00:19:00.963468: done +2022-05-15 00:19:17.019054: This epoch took 480.278026 s + +2022-05-15 00:19:17.019342: +epoch: 148 +2022-05-15 00:26:34.594958: train loss : 0.0396 +2022-05-15 00:26:59.928472: validation loss: 0.0498 +2022-05-15 00:26:59.929194: Average global foreground Dice: [0.9511, 0.9489, 0.9315] +2022-05-15 00:26:59.929282: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 00:27:00.341415: lr: 0.000113 +2022-05-15 00:27:00.341834: saving scheduled checkpoint file... +2022-05-15 00:27:00.386741: saving checkpoint... +2022-05-15 00:27:01.146455: done, saving took 0.80 seconds +2022-05-15 00:27:01.162296: done +2022-05-15 00:27:17.582306: This epoch took 480.562862 s + +2022-05-15 00:27:17.582602: +epoch: 149 +2022-05-15 00:34:35.051989: train loss : 0.0384 +2022-05-15 00:35:00.546415: validation loss: 0.0484 +2022-05-15 00:35:00.547136: Average global foreground Dice: [0.9523, 0.9418, 0.9246] +2022-05-15 00:35:00.547226: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 00:35:00.967039: lr: 0.000111 +2022-05-15 00:35:00.967238: saving scheduled checkpoint file... +2022-05-15 00:35:01.011837: saving checkpoint... +2022-05-15 00:35:01.742153: done, saving took 0.77 seconds +2022-05-15 00:35:01.757920: done +2022-05-15 00:35:21.710173: This epoch took 484.127468 s + +2022-05-15 00:35:21.710489: +epoch: 150 +2022-05-15 00:42:39.422789: train loss : 0.0372 +2022-05-15 00:43:04.771317: validation loss: 0.0446 +2022-05-15 00:43:04.772042: Average global foreground Dice: [0.9515, 0.9292, 0.925] +2022-05-15 00:43:04.772132: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 00:43:05.183948: lr: 0.000109 +2022-05-15 00:43:05.184127: saving scheduled checkpoint file... +2022-05-15 00:43:05.228245: saving checkpoint... +2022-05-15 00:43:05.959625: done, saving took 0.78 seconds +2022-05-15 00:43:05.976161: done +2022-05-15 00:43:22.167524: This epoch took 480.456876 s + +2022-05-15 00:43:22.167830: +epoch: 151 +2022-05-15 00:50:40.330260: train loss : 0.0393 +2022-05-15 00:51:05.672220: validation loss: 0.0433 +2022-05-15 00:51:05.672889: Average global foreground Dice: [0.9505, 0.939, 0.925] +2022-05-15 00:51:05.672975: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 00:51:06.089572: lr: 0.000107 +2022-05-15 00:51:06.089772: saving scheduled checkpoint file... +2022-05-15 00:51:06.136512: saving checkpoint... +2022-05-15 00:51:06.866854: done, saving took 0.78 seconds +2022-05-15 00:51:06.884244: done +2022-05-15 00:51:25.820093: This epoch took 483.652156 s + +2022-05-15 00:51:25.820390: +epoch: 152 +2022-05-15 00:58:43.554311: train loss : 0.0395 +2022-05-15 00:59:09.059259: validation loss: 0.0532 +2022-05-15 00:59:09.059869: Average global foreground Dice: [0.9505, 0.938, 0.9257] +2022-05-15 00:59:09.059955: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 00:59:09.470340: lr: 0.000106 +2022-05-15 00:59:09.470534: saving scheduled checkpoint file... +2022-05-15 00:59:09.515800: saving checkpoint... +2022-05-15 00:59:10.249107: done, saving took 0.78 seconds +2022-05-15 00:59:10.269454: done +2022-05-15 00:59:31.976011: This epoch took 486.155456 s + +2022-05-15 00:59:31.976444: +epoch: 153 +2022-05-15 01:06:49.359771: train loss : 0.0400 +2022-05-15 01:07:14.724766: validation loss: 0.0429 +2022-05-15 01:07:14.725502: Average global foreground Dice: [0.9497, 0.9439, 0.9245] +2022-05-15 01:07:14.725589: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 01:07:15.145225: lr: 0.000104 +2022-05-15 01:07:15.145411: saving scheduled checkpoint file... +2022-05-15 01:07:15.189541: saving checkpoint... +2022-05-15 01:07:15.880829: done, saving took 0.74 seconds +2022-05-15 01:07:15.898482: done +2022-05-15 01:07:37.668615: This epoch took 485.692045 s + +2022-05-15 01:07:37.668891: +epoch: 154 +2022-05-15 01:14:54.656522: train loss : 0.0411 +2022-05-15 01:15:20.062136: validation loss: 0.0514 +2022-05-15 01:15:20.062852: Average global foreground Dice: [0.9518, 0.9428, 0.9293] +2022-05-15 01:15:20.062942: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 01:15:20.476422: lr: 0.000102 +2022-05-15 01:15:20.476627: saving scheduled checkpoint file... +2022-05-15 01:15:20.521205: saving checkpoint... +2022-05-15 01:15:21.193850: done, saving took 0.72 seconds +2022-05-15 01:15:21.212294: done +2022-05-15 01:15:37.259659: This epoch took 479.590666 s + +2022-05-15 01:15:37.259944: +epoch: 155 +2022-05-15 01:22:53.926517: train loss : 0.0364 +2022-05-15 01:23:19.248577: validation loss: 0.0416 +2022-05-15 01:23:19.249188: Average global foreground Dice: [0.9534, 0.9333, 0.9305] +2022-05-15 01:23:19.249276: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 01:23:19.663813: lr: 0.0001 +2022-05-15 01:23:19.664006: saving scheduled checkpoint file... +2022-05-15 01:23:19.709162: saving checkpoint... +2022-05-15 01:23:20.475352: done, saving took 0.81 seconds +2022-05-15 01:23:20.492061: done +2022-05-15 01:23:56.674862: This epoch took 499.414816 s + +2022-05-15 01:23:56.675157: +epoch: 156 +2022-05-15 01:31:13.822490: train loss : 0.0410 +2022-05-15 01:31:39.143807: validation loss: 0.0512 +2022-05-15 01:31:39.144483: Average global foreground Dice: [0.9531, 0.9468, 0.9311] +2022-05-15 01:31:39.144571: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 01:31:39.599934: lr: 9.8e-05 +2022-05-15 01:31:39.600214: saving scheduled checkpoint file... +2022-05-15 01:31:39.645017: saving checkpoint... +2022-05-15 01:31:40.438207: done, saving took 0.84 seconds +2022-05-15 01:31:40.456203: done +2022-05-15 01:32:00.219991: This epoch took 483.544732 s + +2022-05-15 01:32:00.220289: +epoch: 157 +2022-05-15 01:39:17.325376: train loss : 0.0385 +2022-05-15 01:39:42.630761: validation loss: 0.0455 +2022-05-15 01:39:42.631401: Average global foreground Dice: [0.9511, 0.9302, 0.9262] +2022-05-15 01:39:42.631500: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 01:39:43.046404: lr: 9.6e-05 +2022-05-15 01:39:43.046603: saving scheduled checkpoint file... +2022-05-15 01:39:43.091504: saving checkpoint... +2022-05-15 01:39:43.880895: done, saving took 0.83 seconds +2022-05-15 01:39:43.899771: done +2022-05-15 01:40:04.079781: This epoch took 483.859390 s + +2022-05-15 01:40:04.080072: +epoch: 158 +2022-05-15 01:47:20.940129: train loss : 0.0388 +2022-05-15 01:47:46.326748: validation loss: 0.0485 +2022-05-15 01:47:46.327373: Average global foreground Dice: [0.9491, 0.9438, 0.9281] +2022-05-15 01:47:46.327541: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 01:47:46.737555: lr: 9.4e-05 +2022-05-15 01:47:46.737762: saving scheduled checkpoint file... +2022-05-15 01:47:46.781883: saving checkpoint... +2022-05-15 01:47:47.573657: done, saving took 0.84 seconds +2022-05-15 01:47:47.591877: done +2022-05-15 01:48:20.265345: This epoch took 496.185174 s + +2022-05-15 01:48:20.265637: +epoch: 159 +2022-05-15 01:55:37.797010: train loss : 0.0387 +2022-05-15 01:56:03.147345: validation loss: 0.0428 +2022-05-15 01:56:03.147963: Average global foreground Dice: [0.9506, 0.9484, 0.9298] +2022-05-15 01:56:03.148051: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 01:56:03.583106: lr: 9.3e-05 +2022-05-15 01:56:03.583282: saving scheduled checkpoint file... +2022-05-15 01:56:03.627223: saving checkpoint... +2022-05-15 01:56:04.407308: done, saving took 0.82 seconds +2022-05-15 01:56:04.426264: done +2022-05-15 01:56:27.238227: This epoch took 486.972489 s + +2022-05-15 01:56:27.238506: +epoch: 160 +2022-05-15 02:03:44.197894: train loss : 0.0401 +2022-05-15 02:04:09.617128: validation loss: 0.0378 +2022-05-15 02:04:09.617849: Average global foreground Dice: [0.951, 0.9458, 0.9306] +2022-05-15 02:04:09.617936: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 02:04:10.033498: lr: 9.1e-05 +2022-05-15 02:04:10.033838: saving scheduled checkpoint file... +2022-05-15 02:04:10.079110: saving checkpoint... +2022-05-15 02:04:10.879273: done, saving took 0.85 seconds +2022-05-15 02:04:10.898414: done +2022-05-15 02:04:32.001520: This epoch took 484.762913 s + +2022-05-15 02:04:32.001788: +epoch: 161 +2022-05-15 02:11:49.457512: train loss : 0.0379 +2022-05-15 02:12:14.789896: validation loss: 0.0441 +2022-05-15 02:12:14.790564: Average global foreground Dice: [0.9502, 0.9342, 0.9228] +2022-05-15 02:12:14.790652: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 02:12:15.201532: lr: 8.9e-05 +2022-05-15 02:12:15.201719: saving scheduled checkpoint file... +2022-05-15 02:12:15.246343: saving checkpoint... +2022-05-15 02:12:16.040495: done, saving took 0.84 seconds +2022-05-15 02:12:16.061559: done +2022-05-15 02:12:35.300920: This epoch took 483.299035 s + +2022-05-15 02:12:35.301172: +epoch: 162 +2022-05-15 02:19:52.091334: train loss : 0.0382 +2022-05-15 02:20:17.389841: validation loss: 0.0580 +2022-05-15 02:20:17.390481: Average global foreground Dice: [0.9516, 0.9276, 0.9254] +2022-05-15 02:20:17.390569: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 02:20:17.805863: lr: 8.7e-05 +2022-05-15 02:20:17.806047: saving scheduled checkpoint file... +2022-05-15 02:20:17.850403: saving checkpoint... +2022-05-15 02:20:18.645761: done, saving took 0.84 seconds +2022-05-15 02:20:18.664181: done +2022-05-15 02:20:38.683990: This epoch took 483.382716 s + +2022-05-15 02:20:38.684281: +epoch: 163 +2022-05-15 02:27:55.867573: train loss : 0.0405 +2022-05-15 02:28:21.207276: validation loss: 0.0465 +2022-05-15 02:28:21.207935: Average global foreground Dice: [0.9507, 0.9525, 0.935] +2022-05-15 02:28:21.208023: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 02:28:21.624197: lr: 8.5e-05 +2022-05-15 02:28:21.624403: saving scheduled checkpoint file... +2022-05-15 02:28:21.669064: saving checkpoint... +2022-05-15 02:28:22.461140: done, saving took 0.84 seconds +2022-05-15 02:28:22.479713: done +2022-05-15 02:28:44.946235: This epoch took 486.261832 s + +2022-05-15 02:28:44.946618: +epoch: 164 +2022-05-15 02:36:02.768543: train loss : 0.0373 +2022-05-15 02:36:28.097182: validation loss: 0.0528 +2022-05-15 02:36:28.097827: Average global foreground Dice: [0.9497, 0.9385, 0.9245] +2022-05-15 02:36:28.097913: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 02:36:28.507497: lr: 8.4e-05 +2022-05-15 02:36:28.507696: saving scheduled checkpoint file... +2022-05-15 02:36:28.552024: saving checkpoint... +2022-05-15 02:36:29.352798: done, saving took 0.85 seconds +2022-05-15 02:36:29.371191: done +2022-05-15 02:36:51.524266: This epoch took 486.577545 s + +2022-05-15 02:36:51.524570: +epoch: 165 +2022-05-15 02:44:09.076771: train loss : 0.0356 +2022-05-15 02:44:34.387867: validation loss: 0.0480 +2022-05-15 02:44:34.388521: Average global foreground Dice: [0.9537, 0.936, 0.9276] +2022-05-15 02:44:34.388613: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 02:44:34.800318: lr: 8.2e-05 +2022-05-15 02:44:34.800545: saving scheduled checkpoint file... +2022-05-15 02:44:34.845241: saving checkpoint... +2022-05-15 02:44:35.613070: done, saving took 0.81 seconds +2022-05-15 02:44:35.628644: done +2022-05-15 02:44:58.598347: This epoch took 487.073675 s + +2022-05-15 02:44:58.598714: +epoch: 166 +2022-05-15 02:52:16.348307: train loss : 0.0386 +2022-05-15 02:52:41.674074: validation loss: 0.0432 +2022-05-15 02:52:41.674767: Average global foreground Dice: [0.9526, 0.938, 0.9265] +2022-05-15 02:52:41.674859: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 02:52:42.084785: lr: 8e-05 +2022-05-15 02:52:42.084981: saving scheduled checkpoint file... +2022-05-15 02:52:42.130178: saving checkpoint... +2022-05-15 02:52:42.933966: done, saving took 0.85 seconds +2022-05-15 02:52:42.952556: done +2022-05-15 02:53:01.324301: This epoch took 482.725487 s + +2022-05-15 02:53:01.324589: +epoch: 167 +2022-05-15 03:00:18.489975: train loss : 0.0369 +2022-05-15 03:00:43.754015: validation loss: 0.0429 +2022-05-15 03:00:43.754745: Average global foreground Dice: [0.9512, 0.9421, 0.9291] +2022-05-15 03:00:43.754832: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 03:00:44.166379: lr: 7.8e-05 +2022-05-15 03:00:44.166576: saving scheduled checkpoint file... +2022-05-15 03:00:44.210932: saving checkpoint... +2022-05-15 03:00:45.011197: done, saving took 0.84 seconds +2022-05-15 03:00:45.031068: done +2022-05-15 03:01:02.279155: This epoch took 480.954453 s + +2022-05-15 03:01:02.279493: +epoch: 168 +2022-05-15 03:08:19.850794: train loss : 0.0373 +2022-05-15 03:08:45.159428: validation loss: 0.0588 +2022-05-15 03:08:45.160091: Average global foreground Dice: [0.9534, 0.9465, 0.9291] +2022-05-15 03:08:45.160178: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 03:08:45.572984: lr: 7.7e-05 +2022-05-15 03:08:45.573177: saving scheduled checkpoint file... +2022-05-15 03:08:45.617115: saving checkpoint... +2022-05-15 03:08:46.403411: done, saving took 0.83 seconds +2022-05-15 03:08:46.422155: done +2022-05-15 03:09:15.782784: This epoch took 493.503187 s + +2022-05-15 03:09:15.783058: +epoch: 169 +2022-05-15 03:16:33.504078: train loss : 0.0362 +2022-05-15 03:16:58.906091: validation loss: 0.0460 +2022-05-15 03:16:58.906804: Average global foreground Dice: [0.9518, 0.937, 0.9305] +2022-05-15 03:16:58.906905: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 03:16:59.321841: lr: 7.5e-05 +2022-05-15 03:16:59.322026: saving scheduled checkpoint file... +2022-05-15 03:16:59.367281: saving checkpoint... +2022-05-15 03:17:00.132458: done, saving took 0.81 seconds +2022-05-15 03:17:00.148224: done +2022-05-15 03:17:20.041109: This epoch took 484.257952 s + +2022-05-15 03:17:20.041373: +epoch: 170 +2022-05-15 03:24:37.704458: train loss : 0.0376 +2022-05-15 03:25:03.010310: validation loss: 0.0490 +2022-05-15 03:25:03.010935: Average global foreground Dice: [0.9512, 0.9424, 0.9261] +2022-05-15 03:25:03.011023: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 03:25:03.423578: lr: 7.3e-05 +2022-05-15 03:25:03.423779: saving scheduled checkpoint file... +2022-05-15 03:25:03.468816: saving checkpoint... +2022-05-15 03:25:04.198016: done, saving took 0.77 seconds +2022-05-15 03:25:04.216657: done +2022-05-15 03:25:22.318065: This epoch took 482.276577 s + +2022-05-15 03:25:22.318357: +epoch: 171 +2022-05-15 03:32:39.370467: train loss : 0.0372 +2022-05-15 03:33:04.690789: validation loss: 0.0466 +2022-05-15 03:33:04.691411: Average global foreground Dice: [0.9513, 0.945, 0.93] +2022-05-15 03:33:04.691504: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 03:33:05.102087: lr: 7.2e-05 +2022-05-15 03:33:05.102282: saving scheduled checkpoint file... +2022-05-15 03:33:05.147559: saving checkpoint... +2022-05-15 03:33:05.982866: done, saving took 0.88 seconds +2022-05-15 03:33:06.001469: done +2022-05-15 03:33:21.014888: This epoch took 478.696426 s + +2022-05-15 03:33:21.015154: +epoch: 172 +2022-05-15 03:40:38.635301: train loss : 0.0372 +2022-05-15 03:41:03.929124: validation loss: 0.0518 +2022-05-15 03:41:03.929814: Average global foreground Dice: [0.9544, 0.949, 0.9326] +2022-05-15 03:41:03.929905: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 03:41:04.355929: lr: 7e-05 +2022-05-15 03:41:04.356129: saving scheduled checkpoint file... +2022-05-15 03:41:04.400992: saving checkpoint... +2022-05-15 03:41:05.084708: done, saving took 0.73 seconds +2022-05-15 03:41:05.100178: done +2022-05-15 03:41:05.121080: saving checkpoint... +2022-05-15 03:41:05.868191: done, saving took 0.77 seconds +2022-05-15 03:41:22.970470: This epoch took 481.955204 s + +2022-05-15 03:41:22.970776: +epoch: 173 +2022-05-15 03:48:40.635943: train loss : 0.0379 +2022-05-15 03:49:06.023926: validation loss: 0.0557 +2022-05-15 03:49:06.024590: Average global foreground Dice: [0.9481, 0.932, 0.9216] +2022-05-15 03:49:06.024678: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 03:49:06.434716: lr: 6.8e-05 +2022-05-15 03:49:06.434899: saving scheduled checkpoint file... +2022-05-15 03:49:06.480131: saving checkpoint... +2022-05-15 03:49:07.210133: done, saving took 0.78 seconds +2022-05-15 03:49:07.232690: done +2022-05-15 03:49:24.484947: This epoch took 481.514060 s + +2022-05-15 03:49:24.485248: +epoch: 174 +2022-05-15 03:56:41.423748: train loss : 0.0373 +2022-05-15 03:57:06.773922: validation loss: 0.0457 +2022-05-15 03:57:06.774544: Average global foreground Dice: [0.9515, 0.9447, 0.93] +2022-05-15 03:57:06.774632: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 03:57:07.186705: lr: 6.7e-05 +2022-05-15 03:57:07.186964: saving scheduled checkpoint file... +2022-05-15 03:57:07.232303: saving checkpoint... +2022-05-15 03:57:07.958702: done, saving took 0.77 seconds +2022-05-15 03:57:07.974927: done +2022-05-15 03:57:32.053902: This epoch took 487.568557 s + +2022-05-15 03:57:32.054199: +epoch: 175 +2022-05-15 04:04:49.048925: train loss : 0.0365 +2022-05-15 04:05:14.366164: validation loss: 0.0469 +2022-05-15 04:05:14.366855: Average global foreground Dice: [0.9545, 0.9462, 0.9271] +2022-05-15 04:05:14.366946: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 04:05:14.786281: lr: 6.5e-05 +2022-05-15 04:05:14.786491: saving scheduled checkpoint file... +2022-05-15 04:05:14.831949: saving checkpoint... +2022-05-15 04:05:15.538654: done, saving took 0.75 seconds +2022-05-15 04:05:15.554591: done +2022-05-15 04:05:32.995406: This epoch took 480.941107 s + +2022-05-15 04:05:32.995755: +epoch: 176 +2022-05-15 04:12:50.991315: train loss : 0.0367 +2022-05-15 04:13:16.373954: validation loss: 0.0485 +2022-05-15 04:13:16.374644: Average global foreground Dice: [0.9535, 0.9275, 0.9256] +2022-05-15 04:13:16.374736: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 04:13:16.789996: lr: 6.3e-05 +2022-05-15 04:13:16.790212: saving scheduled checkpoint file... +2022-05-15 04:13:16.834828: saving checkpoint... +2022-05-15 04:13:17.554563: done, saving took 0.76 seconds +2022-05-15 04:13:17.572383: done +2022-05-15 04:13:34.813462: This epoch took 481.817562 s + +2022-05-15 04:13:34.813731: +epoch: 177 +2022-05-15 04:20:52.443381: train loss : 0.0374 +2022-05-15 04:21:17.798238: validation loss: 0.0446 +2022-05-15 04:21:17.798850: Average global foreground Dice: [0.9517, 0.9462, 0.9308] +2022-05-15 04:21:17.798958: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 04:21:18.212877: lr: 6.2e-05 +2022-05-15 04:21:18.213076: saving scheduled checkpoint file... +2022-05-15 04:21:18.257792: saving checkpoint... +2022-05-15 04:21:18.975058: done, saving took 0.76 seconds +2022-05-15 04:21:18.992856: done +2022-05-15 04:21:36.911788: This epoch took 482.097949 s + +2022-05-15 04:21:36.912090: +epoch: 178 +2022-05-15 04:28:54.230991: train loss : 0.0386 +2022-05-15 04:29:19.588117: validation loss: 0.0423 +2022-05-15 04:29:19.588813: Average global foreground Dice: [0.9511, 0.9432, 0.9335] +2022-05-15 04:29:19.588906: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 04:29:20.017175: lr: 6e-05 +2022-05-15 04:29:20.017369: saving scheduled checkpoint file... +2022-05-15 04:29:20.062849: saving checkpoint... +2022-05-15 04:29:20.785990: done, saving took 0.77 seconds +2022-05-15 04:29:20.809339: done +2022-05-15 04:30:00.328359: This epoch took 503.416166 s + +2022-05-15 04:30:00.328748: +epoch: 179 +2022-05-15 04:37:17.969886: train loss : 0.0354 +2022-05-15 04:37:43.320636: validation loss: 0.0406 +2022-05-15 04:37:43.321369: Average global foreground Dice: [0.9524, 0.9419, 0.9309] +2022-05-15 04:37:43.321469: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 04:37:43.735868: lr: 5.9e-05 +2022-05-15 04:37:43.736063: saving scheduled checkpoint file... +2022-05-15 04:37:43.780523: saving checkpoint... +2022-05-15 04:37:44.577983: done, saving took 0.84 seconds +2022-05-15 04:37:44.596549: done +2022-05-15 04:38:07.939255: This epoch took 487.610405 s + +2022-05-15 04:38:07.939541: +epoch: 180 +2022-05-15 04:45:25.448707: train loss : 0.0371 +2022-05-15 04:45:50.875297: validation loss: 0.0438 +2022-05-15 04:45:50.875938: Average global foreground Dice: [0.9511, 0.9435, 0.9288] +2022-05-15 04:45:50.876026: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 04:45:51.296266: lr: 5.7e-05 +2022-05-15 04:45:51.296468: saving scheduled checkpoint file... +2022-05-15 04:45:51.341775: saving checkpoint... +2022-05-15 04:45:52.015704: done, saving took 0.72 seconds +2022-05-15 04:45:52.033414: done +2022-05-15 04:46:08.089788: This epoch took 480.150124 s + +2022-05-15 04:46:08.090172: +epoch: 181 +2022-05-15 04:53:25.543382: train loss : 0.0356 +2022-05-15 04:53:51.000055: validation loss: 0.0434 +2022-05-15 04:53:51.000720: Average global foreground Dice: [0.9517, 0.9343, 0.9239] +2022-05-15 04:53:51.000821: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 04:53:51.413094: lr: 5.6e-05 +2022-05-15 04:53:51.413283: saving scheduled checkpoint file... +2022-05-15 04:53:51.458311: saving checkpoint... +2022-05-15 04:53:52.258579: done, saving took 0.85 seconds +2022-05-15 04:53:52.277605: done +2022-05-15 04:54:12.980501: This epoch took 484.890195 s + +2022-05-15 04:54:12.980804: +epoch: 182 +2022-05-15 05:01:31.385115: train loss : 0.0373 +2022-05-15 05:01:56.740463: validation loss: 0.0432 +2022-05-15 05:01:56.741149: Average global foreground Dice: [0.953, 0.9449, 0.9302] +2022-05-15 05:01:56.741236: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 05:01:57.157481: lr: 5.4e-05 +2022-05-15 05:01:57.157668: saving scheduled checkpoint file... +2022-05-15 05:01:57.201925: saving checkpoint... +2022-05-15 05:01:57.999680: done, saving took 0.84 seconds +2022-05-15 05:01:58.020886: done +2022-05-15 05:02:13.520235: This epoch took 480.539331 s + +2022-05-15 05:02:13.520553: +epoch: 183 +2022-05-15 05:09:31.173949: train loss : 0.0372 +2022-05-15 05:09:56.461777: validation loss: 0.0487 +2022-05-15 05:09:56.462426: Average global foreground Dice: [0.948, 0.9357, 0.9263] +2022-05-15 05:09:56.462527: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 05:09:56.873747: lr: 5.3e-05 +2022-05-15 05:09:56.873933: saving scheduled checkpoint file... +2022-05-15 05:09:56.917722: saving checkpoint... +2022-05-15 05:09:57.710525: done, saving took 0.84 seconds +2022-05-15 05:09:57.730430: done +2022-05-15 05:10:12.687864: This epoch took 479.167207 s + +2022-05-15 05:10:12.688147: +epoch: 184 +2022-05-15 05:17:29.621799: train loss : 0.0367 +2022-05-15 05:17:55.047341: validation loss: 0.0407 +2022-05-15 05:17:55.047935: Average global foreground Dice: [0.9539, 0.9431, 0.9297] +2022-05-15 05:17:55.048022: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 05:17:55.460532: lr: 5.1e-05 +2022-05-15 05:17:55.460730: saving scheduled checkpoint file... +2022-05-15 05:17:55.506262: saving checkpoint... +2022-05-15 05:17:56.304428: done, saving took 0.84 seconds +2022-05-15 05:17:56.324275: done +2022-05-15 05:18:14.425185: This epoch took 481.736942 s + +2022-05-15 05:18:14.425453: +epoch: 185 +2022-05-15 05:25:31.762872: train loss : 0.0363 +2022-05-15 05:25:57.189681: validation loss: 0.0451 +2022-05-15 05:25:57.190311: Average global foreground Dice: [0.951, 0.9371, 0.9278] +2022-05-15 05:25:57.190395: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 05:25:57.605024: lr: 5e-05 +2022-05-15 05:25:57.605321: saving scheduled checkpoint file... +2022-05-15 05:25:57.650309: saving checkpoint... +2022-05-15 05:25:58.446172: done, saving took 0.84 seconds +2022-05-15 05:25:58.465132: done +2022-05-15 05:26:16.761407: This epoch took 482.335850 s + +2022-05-15 05:26:16.761753: +epoch: 186 +2022-05-15 05:33:34.552267: train loss : 0.0380 +2022-05-15 05:33:59.867816: validation loss: 0.0486 +2022-05-15 05:33:59.868505: Average global foreground Dice: [0.9503, 0.9407, 0.9268] +2022-05-15 05:33:59.868597: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 05:34:00.279394: lr: 4.8e-05 +2022-05-15 05:34:00.279589: saving scheduled checkpoint file... +2022-05-15 05:34:00.323714: saving checkpoint... +2022-05-15 05:34:01.108646: done, saving took 0.83 seconds +2022-05-15 05:34:01.126729: done +2022-05-15 05:34:20.693810: This epoch took 483.931956 s + +2022-05-15 05:34:20.694068: +epoch: 187 +2022-05-15 05:41:37.774614: train loss : 0.0366 +2022-05-15 05:42:03.066570: validation loss: 0.0403 +2022-05-15 05:42:03.067219: Average global foreground Dice: [0.9529, 0.9432, 0.9296] +2022-05-15 05:42:03.067307: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 05:42:03.482014: lr: 4.7e-05 +2022-05-15 05:42:03.482207: saving scheduled checkpoint file... +2022-05-15 05:42:03.527195: saving checkpoint... +2022-05-15 05:42:04.325276: done, saving took 0.84 seconds +2022-05-15 05:42:04.344497: done +2022-05-15 05:42:21.553538: This epoch took 480.859365 s + +2022-05-15 05:42:21.553812: +epoch: 188 +2022-05-15 05:49:38.926920: train loss : 0.0388 +2022-05-15 05:50:04.225767: validation loss: 0.0574 +2022-05-15 05:50:04.226475: Average global foreground Dice: [0.9507, 0.9382, 0.9242] +2022-05-15 05:50:04.226562: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 05:50:04.645892: lr: 4.5e-05 +2022-05-15 05:50:04.646086: saving scheduled checkpoint file... +2022-05-15 05:50:04.691088: saving checkpoint... +2022-05-15 05:50:05.513307: done, saving took 0.87 seconds +2022-05-15 05:50:05.533722: done +2022-05-15 05:50:25.856806: This epoch took 484.302891 s + +2022-05-15 05:50:25.857095: +epoch: 189 +2022-05-15 05:57:43.413883: train loss : 0.0346 +2022-05-15 05:58:08.723830: validation loss: 0.0509 +2022-05-15 05:58:08.724517: Average global foreground Dice: [0.9493, 0.9458, 0.9269] +2022-05-15 05:58:08.724606: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 05:58:09.137488: lr: 4.4e-05 +2022-05-15 05:58:09.137672: saving scheduled checkpoint file... +2022-05-15 05:58:09.181484: saving checkpoint... +2022-05-15 05:58:09.962805: done, saving took 0.83 seconds +2022-05-15 05:58:09.981456: done +2022-05-15 05:58:29.775266: This epoch took 483.918056 s + +2022-05-15 05:58:29.775568: +epoch: 190 +2022-05-15 06:05:47.637381: train loss : 0.0369 +2022-05-15 06:06:13.058076: validation loss: 0.0475 +2022-05-15 06:06:13.058787: Average global foreground Dice: [0.9518, 0.9359, 0.9271] +2022-05-15 06:06:13.058875: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 06:06:13.492530: lr: 4.3e-05 +2022-05-15 06:06:13.492710: saving scheduled checkpoint file... +2022-05-15 06:06:13.537691: saving checkpoint... +2022-05-15 06:06:14.315952: done, saving took 0.82 seconds +2022-05-15 06:06:14.334571: done +2022-05-15 06:06:31.068904: This epoch took 481.293216 s + +2022-05-15 06:06:31.069288: +epoch: 191 +2022-05-15 06:13:50.171517: train loss : 0.0377 +2022-05-15 06:14:15.578760: validation loss: 0.0516 +2022-05-15 06:14:15.579467: Average global foreground Dice: [0.9524, 0.9434, 0.9288] +2022-05-15 06:14:15.579557: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 06:14:15.997166: lr: 4.1e-05 +2022-05-15 06:14:15.997357: saving scheduled checkpoint file... +2022-05-15 06:14:16.044222: saving checkpoint... +2022-05-15 06:14:16.838391: done, saving took 0.84 seconds +2022-05-15 06:14:16.856734: done +2022-05-15 06:14:33.957353: This epoch took 482.887960 s + +2022-05-15 06:14:33.957639: +epoch: 192 +2022-05-15 06:21:51.447709: train loss : 0.0362 +2022-05-15 06:22:16.853889: validation loss: 0.0562 +2022-05-15 06:22:16.854539: Average global foreground Dice: [0.9514, 0.9267, 0.9273] +2022-05-15 06:22:16.854630: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 06:22:17.268584: lr: 4e-05 +2022-05-15 06:22:17.268761: saving scheduled checkpoint file... +2022-05-15 06:22:17.313160: saving checkpoint... +2022-05-15 06:22:18.104355: done, saving took 0.84 seconds +2022-05-15 06:22:18.123273: done +2022-05-15 06:22:35.456342: This epoch took 481.498563 s + +2022-05-15 06:22:35.456741: +epoch: 193 +2022-05-15 06:29:53.043469: train loss : 0.0373 +2022-05-15 06:30:18.379163: validation loss: 0.0444 +2022-05-15 06:30:18.379881: Average global foreground Dice: [0.9508, 0.9331, 0.9287] +2022-05-15 06:30:18.379968: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 06:30:18.797065: lr: 3.9e-05 +2022-05-15 06:30:18.797253: saving scheduled checkpoint file... +2022-05-15 06:30:18.841855: saving checkpoint... +2022-05-15 06:30:19.638227: done, saving took 0.84 seconds +2022-05-15 06:30:19.654488: done +2022-05-15 06:30:41.863166: This epoch took 486.406304 s + +2022-05-15 06:30:41.863555: +epoch: 194 +2022-05-15 06:37:59.422635: train loss : 0.0380 +2022-05-15 06:38:24.793253: validation loss: 0.0435 +2022-05-15 06:38:24.793988: Average global foreground Dice: [0.9544, 0.9312, 0.9303] +2022-05-15 06:38:24.794075: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 06:38:25.264468: lr: 3.7e-05 +2022-05-15 06:38:25.264704: saving scheduled checkpoint file... +2022-05-15 06:38:25.312902: saving checkpoint... +2022-05-15 06:38:26.181984: done, saving took 0.92 seconds +2022-05-15 06:38:26.201956: done +2022-05-15 06:38:42.791793: This epoch took 480.928134 s + +2022-05-15 06:38:42.792076: +epoch: 195 +2022-05-15 06:46:00.358541: train loss : 0.0363 +2022-05-15 06:46:25.730424: validation loss: 0.0445 +2022-05-15 06:46:25.731084: Average global foreground Dice: [0.9517, 0.9372, 0.9267] +2022-05-15 06:46:25.731172: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 06:46:26.144606: lr: 3.6e-05 +2022-05-15 06:46:26.144803: saving scheduled checkpoint file... +2022-05-15 06:46:26.188745: saving checkpoint... +2022-05-15 06:46:26.979408: done, saving took 0.83 seconds +2022-05-15 06:46:26.998453: done +2022-05-15 06:46:44.554831: This epoch took 481.762658 s + +2022-05-15 06:46:44.555098: +epoch: 196 +2022-05-15 06:54:02.182379: train loss : 0.0364 +2022-05-15 06:54:27.542491: validation loss: 0.0529 +2022-05-15 06:54:27.543163: Average global foreground Dice: [0.9506, 0.9262, 0.923] +2022-05-15 06:54:27.543256: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 06:54:27.967449: lr: 3.5e-05 +2022-05-15 06:54:27.967649: saving scheduled checkpoint file... +2022-05-15 06:54:28.013092: saving checkpoint... +2022-05-15 06:54:28.805033: done, saving took 0.84 seconds +2022-05-15 06:54:28.824172: done +2022-05-15 06:54:44.691252: This epoch took 480.136033 s + +2022-05-15 06:54:44.691621: +epoch: 197 +2022-05-15 07:02:01.770674: train loss : 0.0388 +2022-05-15 07:02:27.130256: validation loss: 0.0478 +2022-05-15 07:02:27.130888: Average global foreground Dice: [0.9529, 0.9419, 0.9322] +2022-05-15 07:02:27.130974: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 07:02:27.548015: lr: 3.3e-05 +2022-05-15 07:02:27.548195: saving scheduled checkpoint file... +2022-05-15 07:02:27.592906: saving checkpoint... +2022-05-15 07:02:28.397169: done, saving took 0.85 seconds +2022-05-15 07:02:28.416131: done +2022-05-15 07:02:50.156940: This epoch took 485.465214 s + +2022-05-15 07:02:50.157211: +epoch: 198 +2022-05-15 07:10:07.943305: train loss : 0.0370 +2022-05-15 07:10:33.303983: validation loss: 0.0500 +2022-05-15 07:10:33.304705: Average global foreground Dice: [0.9516, 0.9474, 0.9318] +2022-05-15 07:10:33.304794: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 07:10:33.719533: lr: 3.2e-05 +2022-05-15 07:10:33.719728: saving scheduled checkpoint file... +2022-05-15 07:10:33.766672: saving checkpoint... +2022-05-15 07:10:34.568694: done, saving took 0.85 seconds +2022-05-15 07:10:34.585683: done +2022-05-15 07:10:59.063332: This epoch took 488.906019 s + +2022-05-15 07:10:59.063661: +epoch: 199 +2022-05-15 07:18:16.476429: train loss : 0.0363 +2022-05-15 07:18:41.841617: validation loss: 0.0537 +2022-05-15 07:18:41.842297: Average global foreground Dice: [0.9517, 0.9406, 0.9279] +2022-05-15 07:18:41.842383: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 07:18:42.258700: lr: 3.1e-05 +2022-05-15 07:18:42.258956: saving scheduled checkpoint file... +2022-05-15 07:18:42.303855: saving checkpoint... +2022-05-15 07:18:43.106532: done, saving took 0.85 seconds +2022-05-15 07:18:43.125390: done +2022-05-15 07:19:05.573673: This epoch took 486.509909 s + +2022-05-15 07:19:05.573947: +epoch: 200 +2022-05-15 07:26:22.464160: train loss : 0.0378 +2022-05-15 07:26:47.856486: validation loss: 0.0548 +2022-05-15 07:26:47.857124: Average global foreground Dice: [0.9524, 0.9438, 0.9266] +2022-05-15 07:26:47.857223: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 07:26:48.274826: lr: 3e-05 +2022-05-15 07:26:48.275047: saving scheduled checkpoint file... +2022-05-15 07:26:48.320526: saving checkpoint... +2022-05-15 07:26:49.109829: done, saving took 0.83 seconds +2022-05-15 07:26:49.128271: done +2022-05-15 07:27:20.916981: This epoch took 495.342913 s + +2022-05-15 07:27:20.917345: +epoch: 201 +2022-05-15 07:34:38.018184: train loss : 0.0381 +2022-05-15 07:35:03.368197: validation loss: 0.0442 +2022-05-15 07:35:03.368859: Average global foreground Dice: [0.9518, 0.9353, 0.9245] +2022-05-15 07:35:03.368951: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 07:35:03.783710: lr: 2.9e-05 +2022-05-15 07:35:03.783912: saving scheduled checkpoint file... +2022-05-15 07:35:03.828915: saving checkpoint... +2022-05-15 07:35:04.547869: done, saving took 0.76 seconds +2022-05-15 07:35:04.563586: done +2022-05-15 07:35:24.517107: This epoch took 483.599660 s + +2022-05-15 07:35:24.517400: +epoch: 202 +2022-05-15 07:42:41.879703: train loss : 0.0372 +2022-05-15 07:43:07.251829: validation loss: 0.0525 +2022-05-15 07:43:07.252508: Average global foreground Dice: [0.9518, 0.9385, 0.9323] +2022-05-15 07:43:07.252596: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 07:43:07.670818: lr: 2.8e-05 +2022-05-15 07:43:07.670999: saving scheduled checkpoint file... +2022-05-15 07:43:07.715818: saving checkpoint... +2022-05-15 07:43:08.446615: done, saving took 0.78 seconds +2022-05-15 07:43:08.463272: done +2022-05-15 07:43:26.783737: This epoch took 482.266183 s + +2022-05-15 07:43:26.784114: +epoch: 203 +2022-05-15 07:50:44.865912: train loss : 0.0371 +2022-05-15 07:51:10.222037: validation loss: 0.0497 +2022-05-15 07:51:10.222684: Average global foreground Dice: [0.95, 0.9282, 0.9271] +2022-05-15 07:51:10.222772: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 07:51:10.638060: lr: 2.6e-05 +2022-05-15 07:51:10.638246: saving scheduled checkpoint file... +2022-05-15 07:51:10.682980: saving checkpoint... +2022-05-15 07:51:11.405482: done, saving took 0.77 seconds +2022-05-15 07:51:11.434248: done +2022-05-15 07:51:38.107661: This epoch took 491.323443 s + +2022-05-15 07:51:38.107950: +epoch: 204 +2022-05-15 07:58:55.372003: train loss : 0.0364 +2022-05-15 07:59:20.704562: validation loss: 0.0464 +2022-05-15 07:59:20.705169: Average global foreground Dice: [0.9521, 0.9417, 0.9281] +2022-05-15 07:59:20.705253: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 07:59:21.118610: lr: 2.5e-05 +2022-05-15 07:59:21.118797: saving scheduled checkpoint file... +2022-05-15 07:59:21.164025: saving checkpoint... +2022-05-15 07:59:21.880337: done, saving took 0.76 seconds +2022-05-15 07:59:21.897363: done +2022-05-15 07:59:40.281075: This epoch took 482.173020 s + +2022-05-15 07:59:40.281371: +epoch: 205 +2022-05-15 08:06:57.565980: train loss : 0.0366 +2022-05-15 08:07:22.953394: validation loss: 0.0522 +2022-05-15 08:07:22.954084: Average global foreground Dice: [0.9541, 0.9354, 0.9281] +2022-05-15 08:07:22.954172: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 08:07:23.378526: lr: 2.4e-05 +2022-05-15 08:07:23.378711: saving scheduled checkpoint file... +2022-05-15 08:07:23.424487: saving checkpoint... +2022-05-15 08:07:24.153196: done, saving took 0.77 seconds +2022-05-15 08:07:24.170478: done +2022-05-15 08:07:46.716335: This epoch took 486.434851 s + +2022-05-15 08:07:46.716655: +epoch: 206 +2022-05-15 08:15:04.270532: train loss : 0.0362 +2022-05-15 08:15:29.639334: validation loss: 0.0544 +2022-05-15 08:15:29.640115: Average global foreground Dice: [0.9497, 0.9447, 0.9297] +2022-05-15 08:15:29.640205: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 08:15:30.056316: lr: 2.3e-05 +2022-05-15 08:15:30.056518: saving scheduled checkpoint file... +2022-05-15 08:15:30.101331: saving checkpoint... +2022-05-15 08:15:30.910455: done, saving took 0.85 seconds +2022-05-15 08:15:30.928927: done +2022-05-15 08:16:04.302241: This epoch took 497.585463 s + +2022-05-15 08:16:04.302603: +epoch: 207 +2022-05-15 08:23:21.510001: train loss : 0.0364 +2022-05-15 08:23:46.818569: validation loss: 0.0525 +2022-05-15 08:23:46.819342: Average global foreground Dice: [0.9497, 0.9309, 0.9275] +2022-05-15 08:23:46.819431: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 08:23:47.225128: lr: 2.2e-05 +2022-05-15 08:23:47.225310: saving scheduled checkpoint file... +2022-05-15 08:23:47.269713: saving checkpoint... +2022-05-15 08:23:48.046082: done, saving took 0.82 seconds +2022-05-15 08:23:48.064933: done +2022-05-15 08:24:17.525273: This epoch took 493.222557 s + +2022-05-15 08:24:17.525620: +epoch: 208 +2022-05-15 08:31:35.049602: train loss : 0.0383 +2022-05-15 08:32:00.408500: validation loss: 0.0492 +2022-05-15 08:32:00.409148: Average global foreground Dice: [0.9506, 0.9413, 0.9234] +2022-05-15 08:32:00.409234: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 08:32:00.816043: lr: 2.1e-05 +2022-05-15 08:32:00.816219: saving scheduled checkpoint file... +2022-05-15 08:32:00.861778: saving checkpoint... +2022-05-15 08:32:01.649201: done, saving took 0.83 seconds +2022-05-15 08:32:01.668445: done +2022-05-15 08:32:29.089719: This epoch took 491.563984 s + +2022-05-15 08:32:29.090011: +epoch: 209 +2022-05-15 08:39:46.646959: train loss : 0.0376 +2022-05-15 08:40:11.968551: validation loss: 0.0519 +2022-05-15 08:40:11.969246: Average global foreground Dice: [0.9533, 0.9377, 0.9295] +2022-05-15 08:40:11.969332: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 08:40:12.389928: lr: 2e-05 +2022-05-15 08:40:12.390120: saving scheduled checkpoint file... +2022-05-15 08:40:12.435742: saving checkpoint... +2022-05-15 08:40:13.238899: done, saving took 0.85 seconds +2022-05-15 08:40:13.255273: done +2022-05-15 08:40:32.165082: This epoch took 483.074970 s + +2022-05-15 08:40:32.165343: +epoch: 210 +2022-05-15 08:47:49.888720: train loss : 0.0363 +2022-05-15 08:48:15.209677: validation loss: 0.0523 +2022-05-15 08:48:15.210343: Average global foreground Dice: [0.9519, 0.9346, 0.9264] +2022-05-15 08:48:15.210431: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 08:48:15.626482: lr: 1.9e-05 +2022-05-15 08:48:15.626699: saving scheduled checkpoint file... +2022-05-15 08:48:15.672685: saving checkpoint... +2022-05-15 08:48:16.388490: done, saving took 0.76 seconds +2022-05-15 08:48:16.406976: done +2022-05-15 08:48:41.290300: This epoch took 489.124858 s + +2022-05-15 08:48:41.290588: +epoch: 211 +2022-05-15 08:55:59.128819: train loss : 0.0379 +2022-05-15 08:56:24.459966: validation loss: 0.0516 +2022-05-15 08:56:24.460599: Average global foreground Dice: [0.952, 0.9283, 0.9316] +2022-05-15 08:56:24.460688: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 08:56:24.871518: lr: 1.8e-05 +2022-05-15 08:56:24.871713: saving scheduled checkpoint file... +2022-05-15 08:56:24.916681: saving checkpoint... +2022-05-15 08:56:25.717298: done, saving took 0.85 seconds +2022-05-15 08:56:25.754956: done +2022-05-15 08:56:42.692757: This epoch took 481.402069 s + +2022-05-15 08:56:42.693037: +epoch: 212 +2022-05-15 09:04:00.371079: train loss : 0.0352 +2022-05-15 09:04:25.706390: validation loss: 0.0535 +2022-05-15 09:04:25.707321: Average global foreground Dice: [0.9536, 0.9434, 0.9293] +2022-05-15 09:04:25.707408: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 09:04:26.115426: lr: 1.7e-05 +2022-05-15 09:04:26.115655: saving scheduled checkpoint file... +2022-05-15 09:04:26.159718: saving checkpoint... +2022-05-15 09:04:26.957842: done, saving took 0.84 seconds +2022-05-15 09:04:26.976424: done +2022-05-15 09:04:46.135147: This epoch took 483.442012 s + +2022-05-15 09:04:46.135430: +epoch: 213 +2022-05-15 09:12:03.365504: train loss : 0.0352 +2022-05-15 09:12:28.669634: validation loss: 0.0452 +2022-05-15 09:12:28.670271: Average global foreground Dice: [0.95, 0.9387, 0.924] +2022-05-15 09:12:28.670356: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 09:12:29.077193: lr: 1.6e-05 +2022-05-15 09:12:29.077464: saving scheduled checkpoint file... +2022-05-15 09:12:29.124534: saving checkpoint... +2022-05-15 09:12:29.910878: done, saving took 0.83 seconds +2022-05-15 09:12:29.929099: done +2022-05-15 09:12:46.890254: This epoch took 480.754708 s + +2022-05-15 09:12:46.890537: +epoch: 214 +2022-05-15 09:20:04.743057: train loss : 0.0359 +2022-05-15 09:20:30.078493: validation loss: 0.0515 +2022-05-15 09:20:30.079124: Average global foreground Dice: [0.9511, 0.9368, 0.9271] +2022-05-15 09:20:30.079212: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 09:20:30.501337: lr: 1.5e-05 +2022-05-15 09:20:30.501523: saving scheduled checkpoint file... +2022-05-15 09:20:30.548864: saving checkpoint... +2022-05-15 09:20:31.334645: done, saving took 0.83 seconds +2022-05-15 09:20:31.353808: done +2022-05-15 09:20:48.699139: This epoch took 481.808496 s + +2022-05-15 09:20:48.699427: +epoch: 215 +2022-05-15 09:28:06.459966: train loss : 0.0359 +2022-05-15 09:28:31.863258: validation loss: 0.0397 +2022-05-15 09:28:31.864054: Average global foreground Dice: [0.9531, 0.9451, 0.9327] +2022-05-15 09:28:31.864173: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 09:28:32.272585: lr: 1.5e-05 +2022-05-15 09:28:32.272776: saving scheduled checkpoint file... +2022-05-15 09:28:32.317677: saving checkpoint... +2022-05-15 09:28:33.118428: done, saving took 0.85 seconds +2022-05-15 09:28:33.137017: done +2022-05-15 09:28:59.062343: This epoch took 490.362797 s + +2022-05-15 09:28:59.062634: +epoch: 216 +2022-05-15 09:36:16.229347: train loss : 0.0371 +2022-05-15 09:36:41.492398: validation loss: 0.0516 +2022-05-15 09:36:41.493104: Average global foreground Dice: [0.9512, 0.9327, 0.9266] +2022-05-15 09:36:41.493192: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 09:36:41.904919: lr: 1.4e-05 +2022-05-15 09:36:41.905111: saving scheduled checkpoint file... +2022-05-15 09:36:41.950198: saving checkpoint... +2022-05-15 09:36:42.737374: done, saving took 0.83 seconds +2022-05-15 09:36:42.755614: done +2022-05-15 09:37:03.639362: This epoch took 484.576627 s + +2022-05-15 09:37:03.639625: +epoch: 217 +2022-05-15 09:44:20.756472: train loss : 0.0361 +2022-05-15 09:44:46.068009: validation loss: 0.0387 +2022-05-15 09:44:46.068629: Average global foreground Dice: [0.9509, 0.9327, 0.9265] +2022-05-15 09:44:46.068716: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 09:44:46.477305: lr: 1.3e-05 +2022-05-15 09:44:46.477511: saving scheduled checkpoint file... +2022-05-15 09:44:46.522349: saving checkpoint... +2022-05-15 09:44:47.323359: done, saving took 0.85 seconds +2022-05-15 09:44:47.342079: done +2022-05-15 09:45:06.464087: This epoch took 482.824358 s + +2022-05-15 09:45:06.464345: +epoch: 218 +2022-05-15 09:52:24.045805: train loss : 0.0360 +2022-05-15 09:52:49.343460: validation loss: 0.0555 +2022-05-15 09:52:49.344064: Average global foreground Dice: [0.9518, 0.9427, 0.9298] +2022-05-15 09:52:49.344149: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 09:52:49.753805: lr: 1.2e-05 +2022-05-15 09:52:49.754000: saving scheduled checkpoint file... +2022-05-15 09:52:49.799418: saving checkpoint... +2022-05-15 09:52:50.596085: done, saving took 0.84 seconds +2022-05-15 09:52:50.614492: done +2022-05-15 09:53:06.039683: This epoch took 479.575108 s + +2022-05-15 09:53:06.040113: +epoch: 219 +2022-05-15 10:00:23.710042: train loss : 0.0381 +2022-05-15 10:00:49.004876: validation loss: 0.0476 +2022-05-15 10:00:49.005503: Average global foreground Dice: [0.9513, 0.9402, 0.9282] +2022-05-15 10:00:49.005602: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 10:00:49.412755: lr: 1.1e-05 +2022-05-15 10:00:49.412942: saving scheduled checkpoint file... +2022-05-15 10:00:49.460233: saving checkpoint... +2022-05-15 10:00:50.243901: done, saving took 0.83 seconds +2022-05-15 10:00:50.262430: done +2022-05-15 10:01:07.308883: This epoch took 481.268651 s + +2022-05-15 10:01:07.309161: +epoch: 220 +2022-05-15 10:08:24.812502: train loss : 0.0362 +2022-05-15 10:08:50.051689: validation loss: 0.0489 +2022-05-15 10:08:50.052624: Average global foreground Dice: [0.951, 0.9474, 0.9293] +2022-05-15 10:08:50.052712: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 10:08:50.466086: lr: 1.1e-05 +2022-05-15 10:08:50.466279: saving scheduled checkpoint file... +2022-05-15 10:08:50.510713: saving checkpoint... +2022-05-15 10:08:51.293948: done, saving took 0.83 seconds +2022-05-15 10:08:51.330282: done +2022-05-15 10:09:09.845510: This epoch took 482.536242 s + +2022-05-15 10:09:09.845808: +epoch: 221 +2022-05-15 10:16:27.035375: train loss : 0.0358 +2022-05-15 10:16:52.285462: validation loss: 0.0473 +2022-05-15 10:16:52.286141: Average global foreground Dice: [0.9536, 0.9385, 0.9347] +2022-05-15 10:16:52.286232: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 10:16:52.692976: lr: 1e-05 +2022-05-15 10:16:52.693167: saving scheduled checkpoint file... +2022-05-15 10:16:52.738409: saving checkpoint... +2022-05-15 10:16:53.444271: done, saving took 0.75 seconds +2022-05-15 10:16:53.460302: done +2022-05-15 10:17:16.570850: This epoch took 486.724936 s + +2022-05-15 10:17:16.571140: +epoch: 222 +2022-05-15 10:24:34.694588: train loss : 0.0359 +2022-05-15 10:24:59.973226: validation loss: 0.0581 +2022-05-15 10:24:59.973872: Average global foreground Dice: [0.9517, 0.9334, 0.9276] +2022-05-15 10:24:59.973959: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 10:25:00.385039: lr: 9e-06 +2022-05-15 10:25:00.385276: saving scheduled checkpoint file... +2022-05-15 10:25:00.429020: saving checkpoint... +2022-05-15 10:25:01.129515: done, saving took 0.74 seconds +2022-05-15 10:25:01.145308: done +2022-05-15 10:25:17.379961: This epoch took 480.808691 s + +2022-05-15 10:25:17.380267: +epoch: 223 +2022-05-15 10:32:34.703795: train loss : 0.0358 +2022-05-15 10:32:59.946692: validation loss: 0.0587 +2022-05-15 10:32:59.947367: Average global foreground Dice: [0.9508, 0.9357, 0.9303] +2022-05-15 10:32:59.947474: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 10:33:00.358248: lr: 9e-06 +2022-05-15 10:33:00.358454: saving scheduled checkpoint file... +2022-05-15 10:33:00.403581: saving checkpoint... +2022-05-15 10:33:01.101611: done, saving took 0.74 seconds +2022-05-15 10:33:01.120287: done +2022-05-15 10:33:15.920209: This epoch took 478.539834 s + +2022-05-15 10:33:15.920499: +epoch: 224 +2022-05-15 10:40:33.405024: train loss : 0.0384 +2022-05-15 10:40:58.647384: validation loss: 0.0558 +2022-05-15 10:40:58.648034: Average global foreground Dice: [0.9513, 0.9413, 0.9277] +2022-05-15 10:40:58.648121: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 10:40:59.068089: lr: 8e-06 +2022-05-15 10:40:59.068295: saving scheduled checkpoint file... +2022-05-15 10:40:59.113684: saving checkpoint... +2022-05-15 10:40:59.831529: done, saving took 0.76 seconds +2022-05-15 10:40:59.849652: done +2022-05-15 10:41:22.323860: This epoch took 486.403261 s + +2022-05-15 10:41:22.324129: +epoch: 225 +2022-05-15 10:48:39.034390: train loss : 0.0345 +2022-05-15 10:49:04.270866: validation loss: 0.0526 +2022-05-15 10:49:04.271581: Average global foreground Dice: [0.9523, 0.9519, 0.9306] +2022-05-15 10:49:04.271682: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 10:49:04.703997: lr: 7e-06 +2022-05-15 10:49:04.704294: saving scheduled checkpoint file... +2022-05-15 10:49:04.756320: saving checkpoint... +2022-05-15 10:49:05.497576: done, saving took 0.79 seconds +2022-05-15 10:49:05.516688: done +2022-05-15 10:49:20.855691: This epoch took 478.531462 s + +2022-05-15 10:49:20.855959: +epoch: 226 +2022-05-15 10:56:38.344662: train loss : 0.0356 +2022-05-15 10:57:03.547633: validation loss: 0.0400 +2022-05-15 10:57:03.548399: Average global foreground Dice: [0.9521, 0.942, 0.931] +2022-05-15 10:57:03.548514: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 10:57:03.964187: lr: 7e-06 +2022-05-15 10:57:03.964386: saving scheduled checkpoint file... +2022-05-15 10:57:04.009267: saving checkpoint... +2022-05-15 10:57:04.746128: done, saving took 0.78 seconds +2022-05-15 10:57:04.769080: done +2022-05-15 10:57:56.836707: This epoch took 515.980643 s + +2022-05-15 10:57:56.836986: +epoch: 227 +2022-05-15 11:05:14.728245: train loss : 0.0360 +2022-05-15 11:05:39.909145: validation loss: 0.0501 +2022-05-15 11:05:39.909881: Average global foreground Dice: [0.951, 0.9338, 0.9323] +2022-05-15 11:05:39.909969: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 11:05:40.319730: lr: 6e-06 +2022-05-15 11:05:40.319931: saving scheduled checkpoint file... +2022-05-15 11:05:40.364647: saving checkpoint... +2022-05-15 11:05:41.168764: done, saving took 0.85 seconds +2022-05-15 11:05:41.193075: done +2022-05-15 11:05:58.595576: This epoch took 481.758494 s + +2022-05-15 11:05:58.595840: +epoch: 228 +2022-05-15 11:13:15.939415: train loss : 0.0364 +2022-05-15 11:13:41.216269: validation loss: 0.0495 +2022-05-15 11:13:41.216952: Average global foreground Dice: [0.9505, 0.9448, 0.9327] +2022-05-15 11:13:41.217041: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 11:13:41.629727: lr: 6e-06 +2022-05-15 11:13:41.629942: saving scheduled checkpoint file... +2022-05-15 11:13:41.675137: saving checkpoint... +2022-05-15 11:13:42.488585: done, saving took 0.86 seconds +2022-05-15 11:13:42.513883: done +2022-05-15 11:14:18.994735: This epoch took 500.398777 s + +2022-05-15 11:14:18.995148: +epoch: 229 +2022-05-15 11:21:35.907960: train loss : 0.0346 +2022-05-15 11:22:01.157757: validation loss: 0.0465 +2022-05-15 11:22:01.158475: Average global foreground Dice: [0.9532, 0.9449, 0.9299] +2022-05-15 11:22:01.158563: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 11:22:01.570647: lr: 5e-06 +2022-05-15 11:22:01.570844: saving scheduled checkpoint file... +2022-05-15 11:22:01.615331: saving checkpoint... +2022-05-15 11:22:02.437357: done, saving took 0.87 seconds +2022-05-15 11:22:02.463004: done +2022-05-15 11:22:46.396992: This epoch took 507.401737 s + +2022-05-15 11:22:46.397272: +epoch: 230 +2022-05-15 11:30:04.013948: train loss : 0.0363 +2022-05-15 11:30:29.207950: validation loss: 0.0478 +2022-05-15 11:30:29.208712: Average global foreground Dice: [0.949, 0.9423, 0.9299] +2022-05-15 11:30:29.208800: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 11:30:29.635484: lr: 5e-06 +2022-05-15 11:30:29.635674: saving scheduled checkpoint file... +2022-05-15 11:30:29.688822: saving checkpoint... +2022-05-15 11:30:30.488358: done, saving took 0.85 seconds +2022-05-15 11:30:30.506895: done +2022-05-15 11:30:47.078007: This epoch took 480.680629 s + +2022-05-15 11:30:47.078293: +epoch: 231 +2022-05-15 11:38:05.316136: train loss : 0.0371 +2022-05-15 11:38:30.491748: validation loss: 0.0477 +2022-05-15 11:38:30.492384: Average global foreground Dice: [0.9522, 0.9394, 0.9314] +2022-05-15 11:38:30.492484: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 11:38:30.898485: lr: 4e-06 +2022-05-15 11:38:30.898849: saving scheduled checkpoint file... +2022-05-15 11:38:30.943602: saving checkpoint... +2022-05-15 11:38:31.738473: done, saving took 0.84 seconds +2022-05-15 11:38:31.761986: done +2022-05-15 11:38:59.241058: This epoch took 492.162660 s + +2022-05-15 11:38:59.241326: +epoch: 232 +2022-05-15 11:46:16.787780: train loss : 0.0356 +2022-05-15 11:46:42.018455: validation loss: 0.0470 +2022-05-15 11:46:42.019267: Average global foreground Dice: [0.9529, 0.9386, 0.9301] +2022-05-15 11:46:42.019359: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 11:46:42.451734: lr: 4e-06 +2022-05-15 11:46:42.451933: saving scheduled checkpoint file... +2022-05-15 11:46:42.496598: saving checkpoint... +2022-05-15 11:46:43.307504: done, saving took 0.86 seconds +2022-05-15 11:46:43.331875: done +2022-05-15 11:47:01.647579: This epoch took 482.406142 s + +2022-05-15 11:47:01.647867: +epoch: 233 +2022-05-15 11:54:18.982992: train loss : 0.0381 +2022-05-15 11:54:44.233804: validation loss: 0.0510 +2022-05-15 11:54:44.234501: Average global foreground Dice: [0.952, 0.9363, 0.9284] +2022-05-15 11:54:44.234590: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 11:54:44.643032: lr: 3e-06 +2022-05-15 11:54:44.643224: saving scheduled checkpoint file... +2022-05-15 11:54:44.687598: saving checkpoint... +2022-05-15 11:54:45.485765: done, saving took 0.84 seconds +2022-05-15 11:54:45.503895: done +2022-05-15 11:55:08.450267: This epoch took 486.802289 s + +2022-05-15 11:55:08.450617: +epoch: 234 +2022-05-15 12:02:26.259476: train loss : 0.0355 +2022-05-15 12:02:51.519206: validation loss: 0.0485 +2022-05-15 12:02:51.519817: Average global foreground Dice: [0.9504, 0.9295, 0.9293] +2022-05-15 12:02:51.519906: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 12:02:51.933079: lr: 3e-06 +2022-05-15 12:02:51.933268: saving scheduled checkpoint file... +2022-05-15 12:02:51.978068: saving checkpoint... +2022-05-15 12:02:52.712739: done, saving took 0.78 seconds +2022-05-15 12:02:52.729103: done +2022-05-15 12:03:11.851177: This epoch took 483.400453 s + +2022-05-15 12:03:11.851510: +epoch: 235 +2022-05-15 12:10:29.811094: train loss : 0.0340 +2022-05-15 12:10:55.102836: validation loss: 0.0411 +2022-05-15 12:10:55.103486: Average global foreground Dice: [0.9538, 0.9437, 0.9308] +2022-05-15 12:10:55.103579: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 12:10:55.513961: lr: 3e-06 +2022-05-15 12:10:55.514165: saving scheduled checkpoint file... +2022-05-15 12:10:55.558958: saving checkpoint... +2022-05-15 12:10:56.314641: done, saving took 0.80 seconds +2022-05-15 12:10:56.331981: done +2022-05-15 12:11:25.771367: This epoch took 493.919750 s + +2022-05-15 12:11:25.771743: +epoch: 236 +2022-05-15 12:18:43.543045: train loss : 0.0370 +2022-05-15 12:19:08.810826: validation loss: 0.0582 +2022-05-15 12:19:08.811471: Average global foreground Dice: [0.9506, 0.9385, 0.9246] +2022-05-15 12:19:08.811564: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 12:19:09.229853: lr: 2e-06 +2022-05-15 12:19:09.230037: saving scheduled checkpoint file... +2022-05-15 12:19:09.274372: saving checkpoint... +2022-05-15 12:19:09.983398: done, saving took 0.75 seconds +2022-05-15 12:19:10.001350: done +2022-05-15 12:19:31.912294: This epoch took 486.140449 s + +2022-05-15 12:19:31.912582: +epoch: 237 +2022-05-15 12:26:49.222585: train loss : 0.0371 +2022-05-15 12:27:14.473120: validation loss: 0.0514 +2022-05-15 12:27:14.473811: Average global foreground Dice: [0.9499, 0.9427, 0.9266] +2022-05-15 12:27:14.473899: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 12:27:14.948941: lr: 2e-06 +2022-05-15 12:27:14.949135: saving scheduled checkpoint file... +2022-05-15 12:27:14.997178: saving checkpoint... +2022-05-15 12:27:15.830325: done, saving took 0.88 seconds +2022-05-15 12:27:15.849069: done +2022-05-15 12:27:32.244787: This epoch took 480.332103 s + +2022-05-15 12:27:32.245064: +epoch: 238 +2022-05-15 12:34:49.715222: train loss : 0.0359 +2022-05-15 12:35:14.970410: validation loss: 0.0480 +2022-05-15 12:35:14.971089: Average global foreground Dice: [0.9496, 0.944, 0.9288] +2022-05-15 12:35:14.971175: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 12:35:15.388947: lr: 2e-06 +2022-05-15 12:35:15.389143: saving scheduled checkpoint file... +2022-05-15 12:35:15.434321: saving checkpoint... +2022-05-15 12:35:16.133755: done, saving took 0.74 seconds +2022-05-15 12:35:16.152841: done +2022-05-15 12:35:33.115909: This epoch took 480.870718 s + +2022-05-15 12:35:33.116421: +epoch: 239 +2022-05-15 12:42:49.673317: train loss : 0.0347 +2022-05-15 12:43:14.890048: validation loss: 0.0502 +2022-05-15 12:43:14.890675: Average global foreground Dice: [0.9517, 0.9257, 0.929] +2022-05-15 12:43:14.890761: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 12:43:15.300465: lr: 1e-06 +2022-05-15 12:43:15.300648: saving scheduled checkpoint file... +2022-05-15 12:43:15.345171: saving checkpoint... +2022-05-15 12:43:16.128954: done, saving took 0.83 seconds +2022-05-15 12:43:16.144213: done +2022-05-15 12:43:42.949049: This epoch took 489.832232 s + +2022-05-15 12:43:42.949333: +epoch: 240 +2022-05-15 12:51:00.735076: train loss : 0.0346 +2022-05-15 12:51:25.982047: validation loss: 0.0505 +2022-05-15 12:51:25.982673: Average global foreground Dice: [0.9503, 0.9436, 0.9366] +2022-05-15 12:51:25.982760: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 12:51:26.399032: lr: 1e-06 +2022-05-15 12:51:26.399216: saving scheduled checkpoint file... +2022-05-15 12:51:26.444163: saving checkpoint... +2022-05-15 12:51:27.217997: done, saving took 0.82 seconds +2022-05-15 12:51:27.236396: done +2022-05-15 12:51:44.349782: This epoch took 481.400349 s + +2022-05-15 12:51:44.350057: +epoch: 241 +2022-05-15 12:59:01.796574: train loss : 0.0362 +2022-05-15 12:59:27.070106: validation loss: 0.0484 +2022-05-15 12:59:27.070763: Average global foreground Dice: [0.9505, 0.94, 0.9283] +2022-05-15 12:59:27.070851: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 12:59:27.485440: lr: 1e-06 +2022-05-15 12:59:27.485661: saving scheduled checkpoint file... +2022-05-15 12:59:27.530771: saving checkpoint... +2022-05-15 12:59:28.329867: done, saving took 0.84 seconds +2022-05-15 12:59:28.348511: done +2022-05-15 12:59:50.605839: This epoch took 486.255677 s + +2022-05-15 12:59:50.606109: +epoch: 242 +2022-05-15 13:07:07.805925: train loss : 0.0357 +2022-05-15 13:07:33.104703: validation loss: 0.0344 +2022-05-15 13:07:33.105323: Average global foreground Dice: [0.9504, 0.9378, 0.9303] +2022-05-15 13:07:33.105409: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 13:07:33.515901: lr: 1e-06 +2022-05-15 13:07:33.516096: saving scheduled checkpoint file... +2022-05-15 13:07:33.562548: saving checkpoint... +2022-05-15 13:07:34.385472: done, saving took 0.87 seconds +2022-05-15 13:07:34.409771: done +2022-05-15 13:07:53.680955: This epoch took 483.074744 s + +2022-05-15 13:07:53.681248: +epoch: 243 +2022-05-15 13:15:11.112884: train loss : 0.0371 +2022-05-15 13:15:36.360023: validation loss: 0.0589 +2022-05-15 13:15:36.360742: Average global foreground Dice: [0.9518, 0.942, 0.9284] +2022-05-15 13:15:36.360831: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 13:15:36.774165: lr: 0.0 +2022-05-15 13:15:36.774359: saving scheduled checkpoint file... +2022-05-15 13:15:36.819600: saving checkpoint... +2022-05-15 13:15:37.610932: done, saving took 0.84 seconds +2022-05-15 13:15:37.629668: done +2022-05-15 13:16:10.695667: This epoch took 497.014318 s + +2022-05-15 13:16:10.695946: +epoch: 244 +2022-05-15 13:23:28.058187: train loss : 0.0366 +2022-05-15 13:23:53.384415: validation loss: 0.0472 +2022-05-15 13:23:53.385106: Average global foreground Dice: [0.9528, 0.9349, 0.9289] +2022-05-15 13:23:53.385196: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 13:23:53.797549: lr: 0.0 +2022-05-15 13:23:53.797742: saving scheduled checkpoint file... +2022-05-15 13:23:53.842757: saving checkpoint... +2022-05-15 13:23:54.546162: done, saving took 0.75 seconds +2022-05-15 13:23:54.568363: done +2022-05-15 13:24:24.480547: This epoch took 493.784468 s + +2022-05-15 13:24:24.480921: +epoch: 245 +2022-05-15 13:31:41.861784: train loss : 0.0359 +2022-05-15 13:32:07.186340: validation loss: 0.0570 +2022-05-15 13:32:07.187164: Average global foreground Dice: [0.9505, 0.9398, 0.9252] +2022-05-15 13:32:07.187254: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 13:32:07.654204: lr: 0.0 +2022-05-15 13:32:07.654388: saving scheduled checkpoint file... +2022-05-15 13:32:07.703147: saving checkpoint... +2022-05-15 13:32:08.590110: done, saving took 0.94 seconds +2022-05-15 13:32:08.612690: done +2022-05-15 13:32:27.301940: This epoch took 482.820900 s + +2022-05-15 13:32:27.302304: +epoch: 246 +2022-05-15 13:39:44.958048: train loss : 0.0369 +2022-05-15 13:40:10.307958: validation loss: 0.0426 +2022-05-15 13:40:10.308623: Average global foreground Dice: [0.9546, 0.9299, 0.9329] +2022-05-15 13:40:10.308716: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 13:40:10.725674: lr: 0.0 +2022-05-15 13:40:10.725884: saving scheduled checkpoint file... +2022-05-15 13:40:10.770552: saving checkpoint... +2022-05-15 13:40:11.576410: done, saving took 0.85 seconds +2022-05-15 13:40:11.595451: done +2022-05-15 13:40:42.657645: This epoch took 495.355220 s + +2022-05-15 13:40:42.658132: +epoch: 247 +2022-05-15 13:47:59.875669: train loss : 0.0359 +2022-05-15 13:48:25.136696: validation loss: 0.0447 +2022-05-15 13:48:25.137402: Average global foreground Dice: [0.9531, 0.9446, 0.9351] +2022-05-15 13:48:25.137504: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 13:48:25.554892: lr: 0.0 +2022-05-15 13:48:25.555063: saving scheduled checkpoint file... +2022-05-15 13:48:25.600153: saving checkpoint... +2022-05-15 13:48:26.396896: done, saving took 0.84 seconds +2022-05-15 13:48:26.415628: done +2022-05-15 13:48:43.914482: This epoch took 481.255995 s + +2022-05-15 13:48:43.914738: +epoch: 248 +2022-05-15 13:56:01.234101: train loss : 0.0357 +2022-05-15 13:56:26.576544: validation loss: 0.0442 +2022-05-15 13:56:26.577196: Average global foreground Dice: [0.9511, 0.9391, 0.9257] +2022-05-15 13:56:26.577281: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 13:56:26.991418: lr: 0.0 +2022-05-15 13:56:26.991608: saving scheduled checkpoint file... +2022-05-15 13:56:27.036570: saving checkpoint... +2022-05-15 13:56:27.827563: done, saving took 0.84 seconds +2022-05-15 13:56:27.846276: done +2022-05-15 13:56:44.825676: This epoch took 480.910833 s + +2022-05-15 13:56:44.825960: +epoch: 249 +2022-05-15 14:04:02.078629: train loss : 0.0341 +2022-05-15 14:04:27.312836: validation loss: 0.0466 +2022-05-15 14:04:27.313457: Average global foreground Dice: [0.953, 0.9413, 0.9321] +2022-05-15 14:04:27.313543: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-05-15 14:04:27.731789: lr: 0.0 +2022-05-15 14:04:27.731981: saving scheduled checkpoint file... +2022-05-15 14:04:27.777411: saving checkpoint... +2022-05-15 14:04:28.570212: done, saving took 0.84 seconds +2022-05-15 14:04:28.588943: done +2022-05-15 14:04:49.178616: This epoch took 484.352537 s + +2022-05-15 14:04:49.215878: saving checkpoint... +2022-05-15 14:04:49.782251: done, saving took 0.60 seconds diff --git a/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_3/model_best.model b/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_3/model_best.model new file mode 100644 index 0000000000000000000000000000000000000000..9096152cd01d58c05db7129de638013af697fd15 --- /dev/null +++ b/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_3/model_best.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a33940fb950c129f8ac84bf08892e397ef999cb9c170328548813e1941c42faf +size 409138657 diff --git a/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_3/model_latest.model b/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_3/model_latest.model new file mode 100644 index 0000000000000000000000000000000000000000..b7d604bd4ad7094af3ff3e389677ef4180597c54 --- /dev/null +++ b/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_3/model_latest.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9f394d53faa52cc82e9a009724b265be31e91f0f9310391a654fc635aa44a71e +size 409161377 diff --git a/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_4/model_best.model b/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_4/model_best.model new file mode 100644 index 0000000000000000000000000000000000000000..feeb223a435a8a18f6a7d310137ef10ae04a228a --- /dev/null +++ b/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_4/model_best.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:87c35837c05e7ff8f610752be5d3ad981f35cd2b2942f5e4d012fcd554895e3a +size 409149537 diff --git a/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_4/model_latest.model b/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_4/model_latest.model new file mode 100644 index 0000000000000000000000000000000000000000..521546d3f76cd5d5dd4286b2a2a54364323b6f58 --- /dev/null +++ b/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_4/model_latest.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f1ef3bc00e173bea0357d1c27fc238d23426aa1da0ec6cf53507ac653de92c98 +size 409156769 diff --git a/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/plans.pkl b/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/plans.pkl new file mode 100644 index 0000000000000000000000000000000000000000..4694a053b265f2735e6bf4464e2f92dc18acdff4 --- /dev/null +++ b/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/plans.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b57580277c4d7cda521ce4c8bf850febca88a51b2ff025b73470c14a2ebe14d3 +size 390605