diff --git "a/slurm.out" "b/slurm.out"
new file mode 100644--- /dev/null
+++ "b/slurm.out"
@@ -0,0 +1,2860 @@
+0: W1123 14:35:36.265000 1308318 torch/distributed/run.py:792] 
+0: W1123 14:35:36.265000 1308318 torch/distributed/run.py:792] *****************************************
+0: W1123 14:35:36.265000 1308318 torch/distributed/run.py:792] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+0: W1123 14:35:36.265000 1308318 torch/distributed/run.py:792] *****************************************
+3: W1123 14:35:36.265000 3160663 torch/distributed/run.py:792] 
+3: W1123 14:35:36.265000 3160663 torch/distributed/run.py:792] *****************************************
+3: W1123 14:35:36.265000 3160663 torch/distributed/run.py:792] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+3: W1123 14:35:36.265000 3160663 torch/distributed/run.py:792] *****************************************
+1: W1123 14:35:36.265000 364677 torch/distributed/run.py:792] 
+1: W1123 14:35:36.265000 364677 torch/distributed/run.py:792] *****************************************
+1: W1123 14:35:36.265000 364677 torch/distributed/run.py:792] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+1: W1123 14:35:36.265000 364677 torch/distributed/run.py:792] *****************************************
+2: W1123 14:35:36.265000 3283449 torch/distributed/run.py:792] 
+2: W1123 14:35:36.265000 3283449 torch/distributed/run.py:792] *****************************************
+2: W1123 14:35:36.265000 3283449 torch/distributed/run.py:792] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+2: W1123 14:35:36.265000 3283449 torch/distributed/run.py:792] *****************************************
+1: [2025-11-23 14:40:32,705] [INFO] [axolotl.utils.schemas.validation.check_eval_packing:119] [PID:364771] [RANK:0] explicitly setting `eval_sample_packing` to match `sample_packing`
+1: [2025-11-23 14:40:32,705] [INFO] [axolotl.utils.schemas.validation.hint_sample_packing_padding:218] [PID:364771] [RANK:0] Setting `pad_to_sequence_len: true` to prevent memory leaks when sample_packing
+2: [2025-11-23 14:40:32,705] [INFO] [axolotl.utils.schemas.validation.check_eval_packing:119] [PID:3283536] [RANK:0] explicitly setting `eval_sample_packing` to match `sample_packing`
+2: [2025-11-23 14:40:32,705] [INFO] [axolotl.utils.schemas.validation.hint_sample_packing_padding:218] [PID:3283536] [RANK:0] Setting `pad_to_sequence_len: true` to prevent memory leaks when sample_packing
+0: [2025-11-23 14:40:32,706] [INFO] [axolotl.utils.schemas.validation.check_eval_packing:119] [PID:1308399] [RANK:0] explicitly setting `eval_sample_packing` to match `sample_packing`
+0: [2025-11-23 14:40:32,706] [INFO] [axolotl.utils.schemas.validation.hint_sample_packing_padding:218] [PID:1308399] [RANK:0] Setting `pad_to_sequence_len: true` to prevent memory leaks when sample_packing
+3: [2025-11-23 14:40:32,706] [INFO] [axolotl.utils.schemas.validation.check_eval_packing:119] [PID:3160742] [RANK:0] explicitly setting `eval_sample_packing` to match `sample_packing`
+3: [2025-11-23 14:40:32,706] [INFO] [axolotl.utils.schemas.validation.hint_sample_packing_padding:218] [PID:3160742] [RANK:0] Setting `pad_to_sequence_len: true` to prevent memory leaks when sample_packing
+0: [2025-11-23 14:41:03,101] [WARNING] [axolotl.utils.config.normalize_config:139] [PID:1308399] [RANK:0] Invalid value for save_steps (1.6666666666666667) from saves_per_epoch and/or num_epochs. Saving at training end only.
+0: [2025-11-23 14:41:03,271] [INFO] [axolotl.cli.config.load_cfg:245] [PID:1308399] [RANK:0] config:
+0: {
+0:   "activation_offloading": false,
+0:   "auto_resume_from_checkpoints": true,
+0:   "axolotl_config_path": "/lustre/fswork/projects/rech/dgo/udv55np/train/tmp/1763904854545720057.yaml",
+0:   "base_model": "/lustre/fswork/projects/rech/qwv/udv55np/Gemma/base/gemma-3-4b",
+0:   "base_model_config": "/lustre/fswork/projects/rech/qwv/udv55np/Gemma/base/gemma-3-4b",
+0:   "batch_size": 16,
+0:   "bf16": true,
+0:   "capabilities": {
+0:     "bf16": true,
+0:     "compute_capability": "sm_90",
+0:     "fp8": false,
+0:     "n_gpu": 16,
+0:     "n_node": 1
+0:   },
+0:   "chat_template": "gemma3",
+0:   "context_parallel_size": 1,
+0:   "curriculum_sampling": true,
+0:   "dataloader_num_workers": 2,
+0:   "dataset_prepared_path": "/lustre/fswork/projects/rech/dgo/udv55np/dataset_gemma/Nemotron-Super-49B-v1_5/split_0.25",
+0:   "dataset_processes": 32,
+0:   "datasets": [
+0:     {
+0:       "chat_template": "tokenizer_default",
+0:       "data_files": [
+0:         "/lustre/fswork/projects/rech/qwv/udv55np/dataset/ift/Nemotron-Super-49B-v1_5/no_thinking/0014.jsonl",
+0:         "/lustre/fswork/projects/rech/qwv/udv55np/dataset/ift/Nemotron-Super-49B-v1_5/no_thinking/0010.jsonl",
+0:         "/lustre/fswork/projects/rech/qwv/udv55np/dataset/ift/Nemotron-Super-49B-v1_5/no_thinking/0012.jsonl",
+0:         "/lustre/fswork/projects/rech/qwv/udv55np/dataset/ift/Nemotron-Super-49B-v1_5/no_thinking/0008.jsonl",
+0:         "/lustre/fswork/projects/rech/qwv/udv55np/dataset/ift/Nemotron-Super-49B-v1_5/no_thinking/0001.jsonl",
+0:         "/lustre/fswork/projects/rech/qwv/udv55np/dataset/ift/Nemotron-Super-49B-v1_5/no_thinking/0002.jsonl",
+0:         "/lustre/fswork/projects/rech/qwv/udv55np/dataset/ift/Nemotron-Super-49B-v1_5/no_thinking/0013.jsonl",
+0:         "/lustre/fswork/projects/rech/qwv/udv55np/dataset/ift/Nemotron-Super-49B-v1_5/no_thinking/0015.jsonl",
+0:         "/lustre/fswork/projects/rech/qwv/udv55np/dataset/ift/Nemotron-Super-49B-v1_5/no_thinking/0004.jsonl",
+0:         "/lustre/fswork/projects/rech/qwv/udv55np/dataset/ift/Nemotron-Super-49B-v1_5/no_thinking/0011.jsonl",
+0:         "/lustre/fswork/projects/rech/qwv/udv55np/dataset/ift/Nemotron-Super-49B-v1_5/no_thinking/0000.jsonl",
+0:         "/lustre/fswork/projects/rech/qwv/udv55np/dataset/ift/Nemotron-Super-49B-v1_5/no_thinking/0003.jsonl"
+0:       ],
+0:       "ds_type": "json",
+0:       "field_messages": "conversations",
+0:       "message_property_mappings": {
+0:         "content": "content",
+0:         "role": "role"
+0:       },
+0:       "path": "/lustre/fswork/projects/rech/qwv/udv55np/dataset/ift/Nemotron-Super-49B-v1_5/no_thinking",
+0:       "trust_remote_code": false,
+0:       "type": "chat_template"
+0:     },
+0:     {
+0:       "chat_template": "tokenizer_default",
+0:       "data_files": [
+0:         "/lustre/fswork/projects/rech/qwv/udv55np/dataset/ift/Nemotron-Super-49B-v1_5/thinking/0007.jsonl",
+0:         "/lustre/fswork/projects/rech/qwv/udv55np/dataset/ift/Nemotron-Super-49B-v1_5/thinking/0009.jsonl",
+0:         "/lustre/fswork/projects/rech/qwv/udv55np/dataset/ift/Nemotron-Super-49B-v1_5/thinking/0005.jsonl",
+0:         "/lustre/fswork/projects/rech/qwv/udv55np/dataset/ift/Nemotron-Super-49B-v1_5/thinking/0006.jsonl"
+0:       ],
+0:       "ds_type": "json",
+0:       "field_messages": "conversations",
+0:       "message_property_mappings": {
+0:         "content": "content",
+0:         "role": "role"
+0:       },
+0:       "path": "/lustre/fswork/projects/rech/qwv/udv55np/dataset/ift/Nemotron-Super-49B-v1_5/thinking",
+0:       "trust_remote_code": false,
+0:       "type": "chat_template"
+0:     }
+0:   ],
+0:   "ddp": true,
+0:   "deepspeed": {
+0:     "bf16": {
+0:       "enabled": true
+0:     },
+0:     "gradient_accumulation_steps": "auto",
+0:     "gradient_clipping": "auto",
+0:     "train_batch_size": "auto",
+0:     "train_micro_batch_size_per_gpu": "auto",
+0:     "wall_clock_breakdown": false,
+0:     "zero_optimization": {
+0:       "contiguous_gradients": true,
+0:       "overlap_comm": true,
+0:       "reduce_bucket_size": "auto",
+0:       "stage": 3,
+0:       "stage3_gather_16bit_weights_on_model_save": true,
+0:       "stage3_param_persistence_threshold": "auto",
+0:       "stage3_prefetch_bucket_size": "auto",
+0:       "sub_group_size": 0
+0:     }
+0:   },
+0:   "device": "cuda:0",
+0:   "device_map": {
+0:     "": 0
+0:   },
+0:   "dion_rank_fraction": 1.0,
+0:   "dion_rank_multiple_of": 1,
+0:   "env_capabilities": {
+0:     "torch_version": "2.6.0"
+0:   },
+0:   "eot_tokens": [
+0:     "<end_of_turn>"
+0:   ],
+0:   "eval_batch_size": 1,
+0:   "eval_causal_lm_metrics": [
+0:     "sacrebleu",
+0:     "comet",
+0:     "ter",
+0:     "chrf"
+0:   ],
+0:   "eval_max_new_tokens": 128,
+0:   "eval_sample_packing": true,
+0:   "eval_table_size": 0,
+0:   "evals_per_epoch": 0,
+0:   "flash_attention": true,
+0:   "fp16": false,
+0:   "gradient_accumulation_steps": 1,
+0:   "gradient_checkpointing": true,
+0:   "gradient_checkpointing_kwargs": {
+0:     "use_reentrant": true
+0:   },
+0:   "is_multimodal": true,
+0:   "learning_rate": 5e-06,
+0:   "lisa_layers_attribute": "model.layers",
+0:   "load_best_model_at_end": false,
+0:   "load_in_4bit": false,
+0:   "load_in_8bit": false,
+0:   "local_rank": 0,
+0:   "logging_steps": 10,
+0:   "lora_dropout": 0.0,
+0:   "loraplus_lr_embedding": 1e-06,
+0:   "lr_scheduler": "warmup_stable_decay",
+0:   "lr_scheduler_kwargs": {
+0:     "min_lr_ratio": 0.1,
+0:     "num_decay_steps": 200
+0:   },
+0:   "max_prompt_len": 512,
+0:   "mean_resizing_embeddings": false,
+0:   "micro_batch_size": 1,
+0:   "model_config_type": "gemma3",
+0:   "num_epochs": 0.6,
+0:   "optimizer": "adamw_torch_fused",
+0:   "output_dir": "/lustre/fswork/projects/rech/dgo/udv55np/ift/Nemotron-Super-49B-v1_5/gemma-3-4b/0.25",
+0:   "pad_to_sequence_len": true,
+0:   "pretrain_multipack_attn": true,
+0:   "pretrain_multipack_buffer_size": 10000,
+0:   "processor_config": "/lustre/fswork/projects/rech/qwv/udv55np/Gemma/base/gemma-3-4b",
+0:   "profiler_steps_start": 0,
+0:   "qlora_sharded_model_loading": false,
+0:   "ray_num_workers": 1,
+0:   "resources_per_worker": {
+0:     "GPU": 1
+0:   },
+0:   "sample_packing": true,
+0:   "sample_packing_bin_size": 200,
+0:   "sample_packing_group_size": 100000,
+0:   "sample_packing_sequentially": true,
+0:   "save_only_model": true,
+0:   "save_safetensors": true,
+0:   "save_total_limit": 20,
+0:   "saves_per_epoch": 1,
+0:   "sequence_len": 16384,
+0:   "shuffle_before_merging_datasets": true,
+0:   "shuffle_merged_datasets": false,
+0:   "skip_prepare_dataset": false,
+0:   "strict": false,
+0:   "tensor_parallel_size": 1,
+0:   "tf32": false,
+0:   "tiled_mlp_use_original_mlp": true,
+0:   "tokenizer_config": "/lustre/fswork/projects/rech/qwv/udv55np/Gemma/base/gemma-3-27b",
+0:   "torch_dtype": "torch.bfloat16",
+0:   "train_on_inputs": false,
+0:   "trl": {
+0:     "log_completions": false,
+0:     "mask_truncated_completions": false,
+0:     "ref_model_mixup_alpha": 0.9,
+0:     "ref_model_sync_steps": 64,
+0:     "scale_rewards": true,
+0:     "sync_ref_model": false,
+0:     "use_vllm": false,
+0:     "vllm_server_host": "0.0.0.0",
+0:     "vllm_server_port": 8000
+0:   },
+0:   "use_ray": false,
+0:   "use_tensorboard": true,
+0:   "val_set_size": 0.0,
+0:   "vllm": {
+0:     "device": "auto",
+0:     "dtype": "auto",
+0:     "gpu_memory_utilization": 0.9,
+0:     "host": "0.0.0.0",
+0:     "port": 8000
+0:   },
+0:   "warmup_steps": 100,
+0:   "weight_decay": 0.0,
+0:   "world_size": 16
+0: }
+0: [2025-11-23 14:41:03,272] [INFO] [axolotl.cli.checks.check_user_token:35] [PID:1308399] [RANK:0] Skipping HuggingFace token verification because HF_HUB_OFFLINE is set to True. Only local files will be used.
+0: [2025-11-23 14:41:05,217] [INFO] [axolotl.utils.data.sft._load_raw_datasets:314] [PID:1308402] [RANK:3] Loading raw datasets...
+0: 
Generating train split: 0 examples [00:00, ? examples/s]
Generating train split: 4846 examples [00:00, 15248.57 examples/s]
Generating train split: 9519 examples [00:00, 24463.45 examples/s]
Generating train split: 18753 examples [00:00, 39086.15 examples/s]
Generating train split: 39500 examples [00:00, 59505.62 examples/s]
Generating train split: 48267 examples [00:01, 55975.58 examples/s]
Generating train split: 66919 examples [00:01, 76634.72 examples/s]
Generating train split: 78951 examples [00:01, 65800.19 examples/s]
Generating train split: 97624 examples [00:01, 82425.21 examples/s]
Generating train split: 109206 examples [00:01, 80541.54 examples/s]
Generating train split: 122934 examples [00:01, 78509.21 examples/s]
Generating train split: 143957 examples [00:02, 88800.06 examples/s]
Generating train split: 157927 examples [00:02, 86949.66 examples/s]
Generating train split: 174149 examples [00:02, 97566.29 examples/s]
Generating train split: 192711 examples [00:02, 82021.51 examples/s]
Generating
+0:  train split: 208979 examples [00:02, 92570.25 examples/s]
Generating train split: 227766 examples [00:03, 65942.46 examples/s]
Generating train split: 241828 examples [00:03, 73823.49 examples/s]
Generating train split: 253075 examples [00:03, 75034.14 examples/s]
Generating train split: 271665 examples [00:03, 88668.01 examples/s]
Generating train split: 283314 examples [00:03, 90764.99 examples/s]
Generating train split: 297311 examples [00:03, 92080.81 examples/s]
Generating train split: 318122 examples [00:04, 97581.81 examples/s]
Generating train split: 331996 examples [00:04, 89227.04 examples/s]
Generating train split: 348297 examples [00:04, 99720.42 examples/s]
Generating train split: 367237 examples [00:04, 87126.05 examples/s]
Generating train split: 387742 examples [00:04, 94473.92 examples/s]
Generating train split: 401884 examples [00:05, 89252.95 examples/s]
Generating train split: 417957 examples [00:05, 99154.28 examples/s]
Generating train split: 417957 examples [00:05, 80974.73 examples/s]
+0: 
+0: [2025-11-23 14:41:12,795] [INFO] [axolotl.utils.data.wrappers.get_dataset_wrapper:88] [PID:1308402] [RANK:3] Loading dataset: /lustre/fswork/projects/rech/qwv/udv55np/dataset/ift/Nemotron-Super-49B-v1_5/no_thinking with base_type: chat_template and prompt_style: None
+0: 
Tokenizing Prompts (num_proc=32):   0%|          | 0/417957 [00:00<?, ? examples/s]
Tokenizing Prompts (num_proc=32):   0%|          | 1000/417957 [00:03<27:39, 251.26 examples/s]
Tokenizing Prompts (num_proc=32):   0%|          | 2000/417957 [00:04<13:39, 507.33 examples/s]
Tokenizing Prompts (num_proc=32):   1%|          | 4000/417957 [00:05<06:15, 1101.32 examples/s]
Tokenizing Prompts (num_proc=32):   1%|▏         | 6000/417957 [00:05<04:15, 1614.76 examples/s]
Tokenizing Prompts (num_proc=32):   2%|▏         | 8000/417957 [00:06<03:20, 2044.72 examples/s]
Tokenizing Prompts (num_proc=32):   2%|▏         | 10000/417957 [00:06<02:49, 2408.33 examples/s]
Tokenizing Prompts (num_proc=32):   3%|▎         | 13000/417957 [00:07<01:51, 3629.88 examples/s]
Tokenizing Prompts (num_proc=32):   3%|▎         | 14000/417957 [00:07<01:53, 3554.52 examples/s]
Tokenizing Prompts (num_proc=32):   4%|▍         | 17000/417957 [00:07<01:21, 4916.54 examples/s]
Tokenizing Prompts (num_proc=32):   5%|▍         |
+0:  20000/417957 [00:08<01:06, 6027.22 examples/s]
Tokenizing Prompts (num_proc=32):   5%|▌         | 21000/417957 [00:08<01:13, 5391.94 examples/s]
Tokenizing Prompts (num_proc=32):   6%|▌         | 23000/417957 [00:08<01:09, 5703.85 examples/s]
Tokenizing Prompts (num_proc=32):   6%|▌         | 25000/417957 [00:08<01:06, 5923.00 examples/s]
Tokenizing Prompts (num_proc=32):   6%|▋         | 27000/417957 [00:09<01:04, 6075.64 examples/s]
Tokenizing Prompts (num_proc=32):   7%|▋         | 30000/417957 [00:09<00:44, 8789.81 examples/s]
Tokenizing Prompts (num_proc=32):   8%|▊         | 32000/417957 [00:09<00:40, 9637.40 examples/s]
Tokenizing Prompts (num_proc=32):   8%|▊         | 34000/417957 [00:09<00:47, 8076.23 examples/s]
Tokenizing Prompts (num_proc=32):   9%|▊         | 36000/417957 [00:10<00:48, 7904.96 examples/s]
Tokenizing Prompts (num_proc=32):   9%|▉         | 38000/417957 [00:10<00:41, 9137.01 examples/s]
Tokenizing Prompts (num_proc=32):  10%|▉         | 40000/417957 [00:10<00:
+0: 46, 8156.72 examples/s]
Tokenizing Prompts (num_proc=32):  10%|█         | 43000/417957 [00:10<00:34, 10878.06 examples/s]
Tokenizing Prompts (num_proc=32):  11%|█         | 45000/417957 [00:11<00:41, 8896.01 examples/s] 
Tokenizing Prompts (num_proc=32):  11%|█▏        | 48000/417957 [00:11<00:34, 10611.09 examples/s]
Tokenizing Prompts (num_proc=32):  12%|█▏        | 50000/417957 [00:11<00:36, 10200.75 examples/s]
Tokenizing Prompts (num_proc=32):  12%|█▏        | 52000/417957 [00:11<00:37, 9845.05 examples/s] 
Tokenizing Prompts (num_proc=32):  13%|█▎        | 55000/417957 [00:11<00:27, 13017.42 examples/s]
Tokenizing Prompts (num_proc=32):  14%|█▎        | 57000/417957 [00:12<00:30, 11717.12 examples/s]
Tokenizing Prompts (num_proc=32):  14%|█▍        | 59000/417957 [00:12<00:30, 11944.91 examples/s]
Tokenizing Prompts (num_proc=32):  15%|█▍        | 61000/417957 [00:12<00:28, 12578.39 examples/s]
Tokenizing Prompts (num_proc=32):  15%|█▌        | 63000/417957 [00:12<00
+0: :27, 12960.66 examples/s]
Tokenizing Prompts (num_proc=32):  16%|█▌        | 65000/417957 [00:12<00:39, 8950.93 examples/s] 
Tokenizing Prompts (num_proc=32):  16%|█▌        | 67000/417957 [00:12<00:34, 10162.83 examples/s]
Tokenizing Prompts (num_proc=32):  17%|█▋        | 71000/417957 [00:13<00:27, 12441.71 examples/s]
Tokenizing Prompts (num_proc=32):  17%|█▋        | 73000/417957 [00:13<00:25, 13333.33 examples/s]
Tokenizing Prompts (num_proc=32):  18%|█▊        | 75000/417957 [00:13<00:34, 9982.94 examples/s] 
Tokenizing Prompts (num_proc=32):  18%|█▊        | 77000/417957 [00:13<00:33, 10206.93 examples/s]
Tokenizing Prompts (num_proc=32):  19%|█▉        | 80000/417957 [00:13<00:25, 13466.41 examples/s]
Tokenizing Prompts (num_proc=32):  20%|█▉        | 83000/417957 [00:14<00:31, 10662.13 examples/s]
Tokenizing Prompts (num_proc=32):  20%|██        | 85000/417957 [00:14<00:32, 10269.72 examples/s]
Tokenizing Prompts (num_proc=32):  21%|██▏       | 89000/417957 [
+0: 00:14<00:23, 14254.14 examples/s]
Tokenizing Prompts (num_proc=32):  22%|██▏       | 91000/417957 [00:14<00:26, 12368.68 examples/s]
Tokenizing Prompts (num_proc=32):  22%|██▏       | 93000/417957 [00:15<00:23, 13558.34 examples/s]
Tokenizing Prompts (num_proc=32):  23%|██▎       | 95000/417957 [00:15<00:35, 9053.85 examples/s] 
Tokenizing Prompts (num_proc=32):  24%|██▎       | 99000/417957 [00:15<00:24, 13203.57 examples/s]
Tokenizing Prompts (num_proc=32):  24%|██▍       | 101000/417957 [00:15<00:24, 12761.12 examples/s]
Tokenizing Prompts (num_proc=32):  25%|██▍       | 103000/417957 [00:16<00:28, 11073.02 examples/s]
Tokenizing Prompts (num_proc=32):  25%|██▌       | 105000/417957 [00:16<00:36, 8617.85 examples/s] 
Tokenizing Prompts (num_proc=32):  26%|██▌       | 107000/417957 [00:16<00:30, 10037.40 examples/s]
Tokenizing Prompts (num_proc=32):  26%|██▌       | 109000/417957 [00:16<00:30, 10298.25 examples/s]
Tokenizing Prompts (num_proc=32):  27%|�
+0: ��█▋       | 112000/417957 [00:16<00:27, 11264.05 examples/s]
Tokenizing Prompts (num_proc=32):  28%|██▊       | 115000/417957 [00:17<00:21, 13857.95 examples/s]
Tokenizing Prompts (num_proc=32):  28%|██▊       | 117000/417957 [00:17<00:28, 10588.87 examples/s]
Tokenizing Prompts (num_proc=32):  29%|██▊       | 120000/417957 [00:17<00:23, 12729.68 examples/s]
Tokenizing Prompts (num_proc=32):  29%|██▉       | 122000/417957 [00:17<00:22, 13204.02 examples/s]
Tokenizing Prompts (num_proc=32):  30%|██▉       | 124000/417957 [00:17<00:24, 11878.16 examples/s]
Tokenizing Prompts (num_proc=32):  30%|███       | 126000/417957 [00:18<00:30, 9470.36 examples/s] 
Tokenizing Prompts (num_proc=32):  31%|███       | 129000/417957 [00:18<00:23, 12346.92 examples/s]
Tokenizing Prompts (num_proc=32):  31%|███▏      | 131000/417957 [00:18<00:22, 12744.09 examples/s]
Tokenizing Prompts (num_proc=32):  32%|███▏      | 135000/417957 [00:18<00:26, 10519.83 examples/s]
+0: Tokenizing Prompts (num_proc=32):  33%|███▎      | 137000/417957 [00:19<00:27, 10270.90 examples/s]
Tokenizing Prompts (num_proc=32):  33%|███▎      | 140000/417957 [00:19<00:24, 11239.78 examples/s]
Tokenizing Prompts (num_proc=32):  34%|███▍      | 142000/417957 [00:19<00:27, 9943.38 examples/s] 
Tokenizing Prompts (num_proc=32):  35%|███▍      | 145000/417957 [00:19<00:22, 12179.94 examples/s]
Tokenizing Prompts (num_proc=32):  35%|███▌      | 147000/417957 [00:19<00:22, 12038.15 examples/s]
Tokenizing Prompts (num_proc=32):  36%|███▌      | 149000/417957 [00:20<00:23, 11504.34 examples/s]
Tokenizing Prompts (num_proc=32):  36%|███▌      | 151000/417957 [00:20<00:21, 12674.34 examples/s]
Tokenizing Prompts (num_proc=32):  37%|███▋      | 153000/417957 [00:20<00:19, 13773.95 examples/s]
Tokenizing Prompts (num_proc=32):  37%|███▋      | 155000/417957 [00:20<00:19, 13651.31 examples/s]
Tokenizing Prompts (num_proc=32):  38%|███▊ 
+0:      | 157000/417957 [00:20<00:29, 8930.98 examples/s] 
Tokenizing Prompts (num_proc=32):  39%|███▊      | 161000/417957 [00:21<00:20, 12323.61 examples/s]
Tokenizing Prompts (num_proc=32):  39%|███▉      | 164000/417957 [00:21<00:19, 13260.67 examples/s]
Tokenizing Prompts (num_proc=32):  40%|███▉      | 167000/417957 [00:21<00:25, 9744.47 examples/s] 
Tokenizing Prompts (num_proc=32):  40%|████      | 169000/417957 [00:21<00:26, 9400.42 examples/s]
Tokenizing Prompts (num_proc=32):  41%|████▏     | 173000/417957 [00:22<00:19, 12767.57 examples/s]
Tokenizing Prompts (num_proc=32):  42%|████▏     | 175000/417957 [00:22<00:20, 11655.97 examples/s]
Tokenizing Prompts (num_proc=32):  42%|████▏     | 177000/417957 [00:22<00:24, 9772.20 examples/s] 
Tokenizing Prompts (num_proc=32):  43%|████▎     | 179000/417957 [00:22<00:21, 11015.77 examples/s]
Tokenizing Prompts (num_proc=32):  43%|████▎     | 181000/417957 [00:22<00:19, 11866.55
+0:  examples/s]
Tokenizing Prompts (num_proc=32):  44%|████▍     | 183000/417957 [00:23<00:19, 12257.06 examples/s]
Tokenizing Prompts (num_proc=32):  44%|████▍     | 185000/417957 [00:23<00:20, 11099.60 examples/s]
Tokenizing Prompts (num_proc=32):  45%|████▍     | 187000/417957 [00:23<00:21, 10997.66 examples/s]
Tokenizing Prompts (num_proc=32):  45%|████▌     | 189000/417957 [00:23<00:18, 12606.99 examples/s]
Tokenizing Prompts (num_proc=32):  46%|████▌     | 193000/417957 [00:23<00:13, 16106.63 examples/s]
Tokenizing Prompts (num_proc=32):  47%|████▋     | 195000/417957 [00:24<00:19, 11521.27 examples/s]
Tokenizing Prompts (num_proc=32):  47%|████▋     | 197000/417957 [00:24<00:20, 10837.44 examples/s]
Tokenizing Prompts (num_proc=32):  48%|████▊     | 199000/417957 [00:24<00:23, 9171.22 examples/s] 
Tokenizing Prompts (num_proc=32):  48%|████▊     | 201000/417957 [00:24<00:22, 9603.09 examples/s]
Tokenizing Prompts (nu
+0: m_proc=32):  49%|████▊     | 203000/417957 [00:24<00:21, 9966.46 examples/s]
Tokenizing Prompts (num_proc=32):  49%|████▉     | 205000/417957 [00:25<00:21, 9863.58 examples/s]
Tokenizing Prompts (num_proc=32):  50%|████▉     | 207000/417957 [00:25<00:22, 9563.06 examples/s]
Tokenizing Prompts (num_proc=32):  50%|█████     | 209000/417957 [00:25<00:25, 8144.97 examples/s]
Tokenizing Prompts (num_proc=32):  51%|█████     | 212000/417957 [00:25<00:20, 10035.32 examples/s]
Tokenizing Prompts (num_proc=32):  51%|█████     | 214000/417957 [00:26<00:18, 11264.64 examples/s]
Tokenizing Prompts (num_proc=32):  52%|█████▏    | 216000/417957 [00:26<00:18, 10772.13 examples/s]
Tokenizing Prompts (num_proc=32):  52%|█████▏    | 219000/417957 [00:26<00:14, 13440.93 examples/s]
Tokenizing Prompts (num_proc=32):  53%|█████▎    | 222000/417957 [00:26<00:12, 15416.63 examples/s]
Tokenizing Prompts (num_proc=32):  54%|█████
+0: ▎    | 224000/417957 [00:26<00:13, 14374.21 examples/s]
Tokenizing Prompts (num_proc=32):  55%|█████▍    | 228000/417957 [00:27<00:19, 9878.74 examples/s] 
Tokenizing Prompts (num_proc=32):  55%|█████▌    | 230000/417957 [00:27<00:17, 10938.11 examples/s]
Tokenizing Prompts (num_proc=32):  56%|█████▌    | 232000/417957 [00:27<00:17, 10549.96 examples/s]
Tokenizing Prompts (num_proc=32):  56%|█████▌    | 234000/417957 [00:27<00:18, 9783.91 examples/s] 
Tokenizing Prompts (num_proc=32):  57%|█████▋    | 237000/417957 [00:28<00:19, 9328.73 examples/s]
Tokenizing Prompts (num_proc=32):  57%|█████▋    | 239000/417957 [00:28<00:17, 9942.73 examples/s]
Tokenizing Prompts (num_proc=32):  58%|█████▊    | 241000/417957 [00:28<00:17, 9875.55 examples/s]
Tokenizing Prompts (num_proc=32):  58%|█████▊    | 243000/417957 [00:28<00:16, 10481.33 examples/s]
Tokenizing Prompts (num_proc=32):  59%|█████▊    | 245000/417
+0: 957 [00:28<00:15, 11297.03 examples/s]
Tokenizing Prompts (num_proc=32):  59%|█████▉    | 247000/417957 [00:29<00:17, 9673.81 examples/s] 
Tokenizing Prompts (num_proc=32):  60%|█████▉    | 249000/417957 [00:29<00:15, 11114.61 examples/s]
Tokenizing Prompts (num_proc=32):  60%|██████    | 252000/417957 [00:29<00:11, 14508.15 examples/s]
Tokenizing Prompts (num_proc=32):  61%|██████    | 255000/417957 [00:29<00:10, 15576.11 examples/s]
Tokenizing Prompts (num_proc=32):  62%|██████▏   | 258000/417957 [00:29<00:10, 15437.77 examples/s]
Tokenizing Prompts (num_proc=32):  62%|██████▏   | 260000/417957 [00:30<00:19, 7902.26 examples/s] 
Tokenizing Prompts (num_proc=32):  63%|██████▎   | 264000/417957 [00:30<00:13, 11069.78 examples/s]
Tokenizing Prompts (num_proc=32):  64%|██████▎   | 266000/417957 [00:30<00:12, 11928.18 examples/s]
Tokenizing Prompts (num_proc=32):  64%|██████▍   | 268000/417957 [0
+0: 0:30<00:13, 11045.11 examples/s]
Tokenizing Prompts (num_proc=32):  65%|██████▍   | 270000/417957 [00:31<00:15, 9742.51 examples/s] 
Tokenizing Prompts (num_proc=32):  65%|██████▌   | 272000/417957 [00:31<00:13, 10524.88 examples/s]
Tokenizing Prompts (num_proc=32):  66%|██████▌   | 274000/417957 [00:31<00:15, 9514.70 examples/s] 
Tokenizing Prompts (num_proc=32):  66%|██████▌   | 276000/417957 [00:31<00:17, 8183.97 examples/s]
Tokenizing Prompts (num_proc=32):  67%|██████▋   | 279000/417957 [00:31<00:12, 11133.20 examples/s]
Tokenizing Prompts (num_proc=32):  67%|██████▋   | 282000/417957 [00:32<00:11, 12013.35 examples/s]
Tokenizing Prompts (num_proc=32):  68%|██████▊   | 284000/417957 [00:32<00:10, 12995.92 examples/s]
Tokenizing Prompts (num_proc=32):  68%|██████▊   | 286000/417957 [00:32<00:11, 11580.78 examples/s]
Tokenizing Prompts (num_proc=32):  69%|██████▉   | 288000/417957 [
+0: 00:32<00:11, 11668.33 examples/s]
Tokenizing Prompts (num_proc=32):  70%|██████▉   | 291000/417957 [00:32<00:08, 14380.52 examples/s]
Tokenizing Prompts (num_proc=32):  70%|███████   | 293000/417957 [00:33<00:11, 10819.59 examples/s]
Tokenizing Prompts (num_proc=32):  71%|███████   | 295000/417957 [00:33<00:11, 11033.69 examples/s]
Tokenizing Prompts (num_proc=32):  71%|███████▏  | 298000/417957 [00:33<00:10, 11529.64 examples/s]
Tokenizing Prompts (num_proc=32):  72%|███████▏  | 300000/417957 [00:33<00:09, 12045.78 examples/s]
Tokenizing Prompts (num_proc=32):  72%|███████▏  | 302000/417957 [00:33<00:09, 12536.40 examples/s]
Tokenizing Prompts (num_proc=32):  73%|███████▎  | 304000/417957 [00:34<00:10, 10992.14 examples/s]
Tokenizing Prompts (num_proc=32):  73%|███████▎  | 306000/417957 [00:34<00:11, 9490.95 examples/s] 
Tokenizing Prompts (num_proc=32):  74%|███████▎  | 3
+0: 08000/417957 [00:34<00:10, 10465.30 examples/s]
Tokenizing Prompts (num_proc=32):  74%|███████▍  | 310000/417957 [00:34<00:12, 8871.42 examples/s] 
Tokenizing Prompts (num_proc=32):  75%|███████▍  | 313000/417957 [00:35<00:09, 10497.36 examples/s]
Tokenizing Prompts (num_proc=32):  75%|███████▌  | 315000/417957 [00:35<00:08, 11774.36 examples/s]
Tokenizing Prompts (num_proc=32):  76%|███████▌  | 317000/417957 [00:35<00:09, 10906.10 examples/s]
Tokenizing Prompts (num_proc=32):  76%|███████▋  | 319000/417957 [00:35<00:08, 12351.42 examples/s]
Tokenizing Prompts (num_proc=32):  77%|███████▋  | 321000/417957 [00:35<00:07, 13820.14 examples/s]
Tokenizing Prompts (num_proc=32):  78%|███████▊  | 325000/417957 [00:35<00:06, 13389.19 examples/s]
Tokenizing Prompts (num_proc=32):  78%|███████▊  | 327000/417957 [00:36<00:07, 11482.92 examples/s]
Tokenizing Prompts (num_proc=32):  79%|███
+0: ████▉  | 330000/417957 [00:36<00:06, 13253.00 examples/s]
Tokenizing Prompts (num_proc=32):  79%|███████▉  | 332062/417957 [00:36<00:06, 12951.82 examples/s]
Tokenizing Prompts (num_proc=32):  80%|███████▉  | 334062/417957 [00:36<00:08, 10006.40 examples/s]
Tokenizing Prompts (num_proc=32):  81%|████████  | 337124/417957 [00:36<00:07, 11423.22 examples/s]
Tokenizing Prompts (num_proc=32):  81%|████████▏ | 340124/417957 [00:37<00:06, 12098.20 examples/s]
Tokenizing Prompts (num_proc=32):  82%|████████▏ | 342124/417957 [00:37<00:08, 9204.52 examples/s] 
Tokenizing Prompts (num_proc=32):  82%|████████▏ | 344186/417957 [00:37<00:07, 9555.02 examples/s]
Tokenizing Prompts (num_proc=32):  83%|████████▎ | 346186/417957 [00:37<00:07, 9628.29 examples/s]
Tokenizing Prompts (num_proc=32):  84%|████████▍ | 350186/417957 [00:38<00:05, 12663.61 examples/s]
Tokenizing Prompts (
+0: num_proc=32):  84%|████████▍ | 352186/417957 [00:38<00:05, 12907.62 examples/s]
Tokenizing Prompts (num_proc=32):  85%|████████▍ | 354186/417957 [00:38<00:04, 13200.65 examples/s]
Tokenizing Prompts (num_proc=32):  85%|████████▌ | 357186/417957 [00:38<00:04, 13445.07 examples/s]
Tokenizing Prompts (num_proc=32):  86%|████████▌ | 359186/417957 [00:38<00:04, 12988.21 examples/s]
Tokenizing Prompts (num_proc=32):  86%|████████▋ | 361247/417957 [00:38<00:04, 13526.82 examples/s]
Tokenizing Prompts (num_proc=32):  87%|████████▋ | 363247/417957 [00:39<00:05, 10860.00 examples/s]
Tokenizing Prompts (num_proc=32):  87%|████████▋ | 365309/417957 [00:39<00:05, 9148.20 examples/s] 
Tokenizing Prompts (num_proc=32):  88%|████████▊ | 367370/417957 [00:39<00:05, 10017.93 examples/s]
Tokenizing Prompts (num_proc=32):  88%|████████▊ | 369431/417957 [00:39<00:04, 115
+0: 15.00 examples/s]
Tokenizing Prompts (num_proc=32):  89%|████████▉ | 371492/417957 [00:40<00:05, 8670.33 examples/s] 
Tokenizing Prompts (num_proc=32):  89%|████████▉ | 373492/417957 [00:40<00:04, 9188.56 examples/s]
Tokenizing Prompts (num_proc=32):  90%|████████▉ | 375553/417957 [00:40<00:05, 8071.44 examples/s]
Tokenizing Prompts (num_proc=32):  91%|█████████ | 378553/417957 [00:40<00:03, 10996.28 examples/s]
Tokenizing Prompts (num_proc=32):  91%|█████████ | 380615/417957 [00:41<00:03, 11117.03 examples/s]
Tokenizing Prompts (num_proc=32):  92%|█████████▏| 382615/417957 [00:41<00:03, 11111.04 examples/s]
Tokenizing Prompts (num_proc=32):  92%|█████████▏| 384615/417957 [00:41<00:02, 11537.76 examples/s]
Tokenizing Prompts (num_proc=32):  93%|█████████▎| 386676/417957 [00:41<00:02, 11972.76 examples/s]
Tokenizing Prompts (num_proc=32):  93%|██████�
+0: ��██▎| 389737/417957 [00:41<00:02, 13871.11 examples/s]
Tokenizing Prompts (num_proc=32):  94%|█████████▍| 391859/417957 [00:42<00:02, 9074.52 examples/s] 
Tokenizing Prompts (num_proc=32):  94%|█████████▍| 393920/417957 [00:42<00:03, 6552.31 examples/s]
Tokenizing Prompts (num_proc=32):  95%|█████████▍| 395920/417957 [00:42<00:03, 7240.81 examples/s]
Tokenizing Prompts (num_proc=32):  95%|█████████▍| 397042/417957 [00:43<00:03, 6089.03 examples/s]
Tokenizing Prompts (num_proc=32):  95%|█████████▌| 398042/417957 [00:43<00:03, 5745.56 examples/s]
Tokenizing Prompts (num_proc=32):  95%|█████████▌| 399042/417957 [00:43<00:03, 6043.25 examples/s]
Tokenizing Prompts (num_proc=32):  96%|█████████▌| 402103/417957 [00:43<00:01, 9212.10 examples/s]
Tokenizing Prompts (num_proc=32):  97%|█████████▋| 404225/417957 [00:43<00:01, 8247.22 examples/s]
Tokenizin
+0: g Prompts (num_proc=32):  97%|█████████▋| 405286/417957 [00:44<00:01, 7963.28 examples/s]
Tokenizing Prompts (num_proc=32):  97%|█████████▋| 406347/417957 [00:44<00:01, 7982.50 examples/s]
Tokenizing Prompts (num_proc=32):  97%|█████████▋| 407408/417957 [00:44<00:01, 7596.12 examples/s]
Tokenizing Prompts (num_proc=32):  98%|█████████▊| 409408/417957 [00:44<00:00, 8566.20 examples/s]
Tokenizing Prompts (num_proc=32):  98%|█████████▊| 410530/417957 [00:44<00:00, 7846.13 examples/s]
Tokenizing Prompts (num_proc=32):  98%|█████████▊| 411591/417957 [00:45<00:01, 3736.62 examples/s]
Tokenizing Prompts (num_proc=32):  99%|█████████▊| 412591/417957 [00:45<00:01, 3452.42 examples/s]
Tokenizing Prompts (num_proc=32):  99%|█████████▉| 413652/417957 [00:46<00:01, 3232.86 examples/s]
Tokenizing Prompts (num_proc=32):  99%|█████████▉| 414713/417
+0: 957 [00:46<00:00, 3609.57 examples/s]
Tokenizing Prompts (num_proc=32):  99%|█████████▉| 415774/417957 [00:46<00:00, 3747.26 examples/s]
Tokenizing Prompts (num_proc=32): 100%|█████████▉| 416774/417957 [00:46<00:00, 4040.58 examples/s]
Tokenizing Prompts (num_proc=32): 100%|█████████▉| 417896/417957 [00:48<00:00, 1867.65 examples/s]
Tokenizing Prompts (num_proc=32): 100%|██████████| 417957/417957 [00:48<00:00, 8594.40 examples/s]
+0: 
Generating train split: 0 examples [00:00, ? examples/s]
Generating train split: 2058 examples [00:00, 12685.95 examples/s]
Generating train split: 6090 examples [00:00, 20000.52 examples/s]
Generating train split: 10251 examples [00:00, 23470.71 examples/s]
Generating train split: 14481 examples [00:00, 26928.86 examples/s]
Generating train split: 18527 examples [00:00, 28584.50 examples/s]
Generating train split: 22610 examples [00:00, 29680.76 examples/s]
Generating train split: 26777 examples [00:00, 30683.36 examples/s]
Generating train split: 31039 examples [00:01, 31656.47 examples/s]
Generating train split: 34830 examples [00:01, 32117.89 examples/s]
Generating train split: 39052 examples [00:01, 21568.00 examples/s]
Generating train split: 43192 examples [00:01, 22815.56 examples/s]
Generating train split: 47447 examples [00:01, 25455.58 examples/s]
Generating train split: 51624 examples [00:01, 27436.69 examples/s]
Generating train split: 54698 examples [00:02, 27469.84 examples/s]
Generating train
+0:  split: 58845 examples [00:02, 29350.33 examples/s]
Generating train split: 63013 examples [00:02, 30744.59 examples/s]
Generating train split: 67127 examples [00:02, 31689.73 examples/s]
Generating train split: 70681 examples [00:02, 29826.04 examples/s]
Generating train split: 75608 examples [00:02, 26769.84 examples/s]
Generating train split: 79766 examples [00:02, 27364.90 examples/s]
Generating train split: 83874 examples [00:03, 28788.97 examples/s]
Generating train split: 88086 examples [00:03, 30205.66 examples/s]
Generating train split: 92253 examples [00:03, 31115.63 examples/s]
Generating train split: 96493 examples [00:03, 32034.48 examples/s]
Generating train split: 100792 examples [00:03, 32692.53 examples/s]
Generating train split: 105548 examples [00:03, 27559.57 examples/s]
Generating train split: 108718 examples [00:03, 23454.08 examples/s]
Generating train split: 112729 examples [00:04, 24044.03 examples/s]
Generating train split: 116714 examples [00:04, 25856.54 examples/s]
Generating trai
+0: n split: 120963 examples [00:04, 27993.38 examples/s]
Generating train split: 124982 examples [00:04, 29226.22 examples/s]
Generating train split: 129148 examples [00:04, 30567.91 examples/s]
Generating train split: 133327 examples [00:04, 31457.99 examples/s]
Generating train split: 137471 examples [00:04, 31909.79 examples/s]
Generating train split: 139320 examples [00:04, 28183.81 examples/s]
+0: [2025-11-23 14:42:06,820] [INFO] [axolotl.utils.data.wrappers.get_dataset_wrapper:88] [PID:1308402] [RANK:3] Loading dataset: /lustre/fswork/projects/rech/qwv/udv55np/dataset/ift/Nemotron-Super-49B-v1_5/thinking with base_type: chat_template and prompt_style: None
+0: 
Tokenizing Prompts (num_proc=32):   0%|          | 0/139320 [00:00<?, ? examples/s]
Tokenizing Prompts (num_proc=32):   1%|          | 1000/139320 [00:11<26:55, 85.61 examples/s]
Tokenizing Prompts (num_proc=32):   1%|▏         | 2000/139320 [00:12<11:52, 192.60 examples/s]
Tokenizing Prompts (num_proc=32):   2%|▏         | 3000/139320 [00:13<07:06, 319.79 examples/s]
Tokenizing Prompts (num_proc=32):   3%|▎         | 4000/139320 [00:13<04:23, 512.96 examples/s]
Tokenizing Prompts (num_proc=32):   4%|▍         | 6000/139320 [00:13<02:05, 1058.60 examples/s]
Tokenizing Prompts (num_proc=32):   5%|▌         | 7000/139320 [00:13<01:48, 1215.72 examples/s]
Tokenizing Prompts (num_proc=32):   6%|▌         | 8000/139320 [00:14<01:29, 1469.82 examples/s]
Tokenizing Prompts (num_proc=32):   6%|▋         | 9000/139320 [00:14<01:09, 1866.12 examples/s]
Tokenizing Prompts (num_proc=32):   8%|▊         | 11000/139320 [00:14<00:52, 2463.78 examples/s]
Tokenizing Prompts (num_proc=32):   9%|▊         | 1
+0: 2000/139320 [00:15<00:52, 2411.12 examples/s]
Tokenizing Prompts (num_proc=32):   9%|▉         | 13000/139320 [00:15<00:57, 2200.51 examples/s]
Tokenizing Prompts (num_proc=32):  10%|█         | 14000/139320 [00:15<00:48, 2590.40 examples/s]
Tokenizing Prompts (num_proc=32):  11%|█         | 15000/139320 [00:16<00:41, 3007.61 examples/s]
Tokenizing Prompts (num_proc=32):  11%|█▏        | 16000/139320 [00:16<00:34, 3602.91 examples/s]
Tokenizing Prompts (num_proc=32):  12%|█▏        | 17000/139320 [00:16<00:32, 3792.72 examples/s]
Tokenizing Prompts (num_proc=32):  14%|█▎        | 19000/139320 [00:18<00:59, 2005.37 examples/s]
Tokenizing Prompts (num_proc=32):  14%|█▍        | 20000/139320 [00:18<00:57, 2084.67 examples/s]
Tokenizing Prompts (num_proc=32):  15%|█▌        | 21000/139320 [00:18<00:45, 2591.82 examples/s]
Tokenizing Prompts (num_proc=32):  17%|█▋        | 23000/139320 [00:18<00:31, 3690.09 examples/s]
Tokenizing Prompts (num_proc=32):  17%|█▋        | 24000/13932
+0: 0 [00:19<00:31, 3664.78 examples/s]
Tokenizing Prompts (num_proc=32):  18%|█▊        | 25000/139320 [00:19<00:45, 2529.62 examples/s]
Tokenizing Prompts (num_proc=32):  19%|█▉        | 27000/139320 [00:20<00:29, 3754.66 examples/s]
Tokenizing Prompts (num_proc=32):  21%|██        | 29000/139320 [00:20<00:21, 5165.84 examples/s]
Tokenizing Prompts (num_proc=32):  22%|██▏       | 30000/139320 [00:20<00:26, 4168.55 examples/s]
Tokenizing Prompts (num_proc=32):  22%|██▏       | 31000/139320 [00:20<00:28, 3763.19 examples/s]
Tokenizing Prompts (num_proc=32):  23%|██▎       | 32000/139320 [00:21<00:27, 3890.94 examples/s]
Tokenizing Prompts (num_proc=32):  24%|██▎       | 33000/139320 [00:23<01:20, 1327.88 examples/s]
Tokenizing Prompts (num_proc=32):  24%|██▍       | 34000/139320 [00:23<01:08, 1532.59 examples/s]
Tokenizing Prompts (num_proc=32):  25%|██▌       | 35000/139320 [00:24<01:09, 1502.57 examples/s]
Tokenizing Prompts (num_proc=32):  26%|██▌       | 3
+0: 6000/139320 [00:24<01:05, 1584.34 examples/s]
Tokenizing Prompts (num_proc=32):  27%|██▋       | 38000/139320 [00:25<00:42, 2372.49 examples/s]
Tokenizing Prompts (num_proc=32):  28%|██▊       | 39000/139320 [00:25<00:38, 2587.82 examples/s]
Tokenizing Prompts (num_proc=32):  29%|██▊       | 40000/139320 [00:25<00:32, 3037.87 examples/s]
Tokenizing Prompts (num_proc=32):  30%|███       | 42000/139320 [00:25<00:20, 4685.74 examples/s]
Tokenizing Prompts (num_proc=32):  31%|███       | 43000/139320 [00:26<00:20, 4672.61 examples/s]
Tokenizing Prompts (num_proc=32):  32%|███▏      | 44000/139320 [00:26<00:21, 4509.33 examples/s]
Tokenizing Prompts (num_proc=32):  32%|███▏      | 45000/139320 [00:26<00:30, 3143.75 examples/s]
Tokenizing Prompts (num_proc=32):  33%|███▎      | 46000/139320 [00:27<00:33, 2761.65 examples/s]
Tokenizing Prompts (num_proc=32):  34%|███▍      | 48000/139320 [00:27<00:30, 3013.68 examples/s]
Tokenizing Prompts (num_proc=32): 
+0:  35%|███▌      | 49000/139320 [00:29<00:50, 1775.88 examples/s]
Tokenizing Prompts (num_proc=32):  36%|███▌      | 50000/139320 [00:29<00:44, 1998.96 examples/s]
Tokenizing Prompts (num_proc=32):  37%|███▋      | 51000/139320 [00:29<00:37, 2359.02 examples/s]
Tokenizing Prompts (num_proc=32):  37%|███▋      | 52000/139320 [00:30<00:31, 2732.23 examples/s]
Tokenizing Prompts (num_proc=32):  38%|███▊      | 53000/139320 [00:30<00:27, 3146.16 examples/s]
Tokenizing Prompts (num_proc=32):  39%|███▉      | 55000/139320 [00:31<00:31, 2654.71 examples/s]
Tokenizing Prompts (num_proc=32):  42%|████▏     | 58000/139320 [00:31<00:18, 4505.25 examples/s]
Tokenizing Prompts (num_proc=32):  42%|████▏     | 59000/139320 [00:32<00:25, 3196.66 examples/s]
Tokenizing Prompts (num_proc=32):  43%|████▎     | 60000/139320 [00:32<00:25, 3062.15 examples/s]
Tokenizing Prompts (num_proc=32):  45%|████▍     | 62000/139320 [00:32<00:18, 4189.02 ex
+0: amples/s]
Tokenizing Prompts (num_proc=32):  45%|████▌     | 63000/139320 [00:32<00:19, 3819.20 examples/s]
Tokenizing Prompts (num_proc=32):  46%|████▌     | 64000/139320 [00:33<00:26, 2855.73 examples/s]
Tokenizing Prompts (num_proc=32):  47%|████▋     | 65000/139320 [00:34<00:36, 2050.54 examples/s]
Tokenizing Prompts (num_proc=32):  47%|████▋     | 66000/139320 [00:35<00:45, 1596.86 examples/s]
Tokenizing Prompts (num_proc=32):  48%|████▊     | 67000/139320 [00:35<00:39, 1844.71 examples/s]
Tokenizing Prompts (num_proc=32):  49%|████▉     | 68000/139320 [00:36<00:37, 1923.84 examples/s]
Tokenizing Prompts (num_proc=32):  50%|████▉     | 69000/139320 [00:36<00:32, 2158.86 examples/s]
Tokenizing Prompts (num_proc=32):  51%|█████     | 71000/139320 [00:36<00:21, 3191.79 examples/s]
Tokenizing Prompts (num_proc=32):  52%|█████▏    | 72000/139320 [00:37<00:23, 2814.30 examples/s]
Tokenizing Prompts (num_proc=32):  52%|�
+0: ��████▏    | 73000/139320 [00:37<00:23, 2809.51 examples/s]
Tokenizing Prompts (num_proc=32):  53%|█████▎    | 74000/139320 [00:37<00:22, 2949.53 examples/s]
Tokenizing Prompts (num_proc=32):  55%|█████▍    | 76000/139320 [00:38<00:19, 3214.08 examples/s]
Tokenizing Prompts (num_proc=32):  55%|█████▌    | 77000/139320 [00:38<00:17, 3643.95 examples/s]
Tokenizing Prompts (num_proc=32):  56%|█████▌    | 78000/139320 [00:38<00:14, 4241.84 examples/s]
Tokenizing Prompts (num_proc=32):  57%|█████▋    | 79000/139320 [00:39<00:17, 3470.36 examples/s]
Tokenizing Prompts (num_proc=32):  58%|█████▊    | 81000/139320 [00:42<00:45, 1289.39 examples/s]
Tokenizing Prompts (num_proc=32):  59%|█████▉    | 82000/139320 [00:42<00:37, 1510.24 examples/s]
Tokenizing Prompts (num_proc=32):  60%|██████    | 84000/139320 [00:42<00:23, 2387.46 examples/s]
Tokenizing Prompts (num_proc=32):  63%|██████▎   | 88000/139
+0: 320 [00:43<00:13, 3774.85 examples/s]
Tokenizing Prompts (num_proc=32):  64%|██████▍   | 89000/139320 [00:43<00:12, 4087.97 examples/s]
Tokenizing Prompts (num_proc=32):  65%|██████▌   | 91000/139320 [00:43<00:10, 4494.54 examples/s]
Tokenizing Prompts (num_proc=32):  66%|██████▌   | 92000/139320 [00:44<00:16, 2865.92 examples/s]
Tokenizing Prompts (num_proc=32):  67%|██████▋   | 93000/139320 [00:44<00:16, 2760.39 examples/s]
Tokenizing Prompts (num_proc=32):  67%|██████▋   | 94000/139320 [00:45<00:18, 2491.74 examples/s]
Tokenizing Prompts (num_proc=32):  69%|██████▉   | 96000/139320 [00:45<00:12, 3366.24 examples/s]
Tokenizing Prompts (num_proc=32):  70%|██████▉   | 97000/139320 [00:46<00:18, 2342.56 examples/s]
Tokenizing Prompts (num_proc=32):  70%|███████   | 98000/139320 [00:47<00:18, 2242.72 examples/s]
Tokenizing Prompts (num_proc=32):  71%|███████   | 99000/139320 [00:47<00:19
+0: , 2094.95 examples/s]
Tokenizing Prompts (num_proc=32):  72%|███████▏  | 100000/139320 [00:47<00:15, 2602.51 examples/s]
Tokenizing Prompts (num_proc=32):  72%|███████▏  | 101000/139320 [00:47<00:12, 3107.20 examples/s]
Tokenizing Prompts (num_proc=32):  73%|███████▎  | 102000/139320 [00:48<00:16, 2225.79 examples/s]
Tokenizing Prompts (num_proc=32):  74%|███████▍  | 103000/139320 [00:48<00:14, 2469.42 examples/s]
Tokenizing Prompts (num_proc=32):  75%|███████▍  | 104000/139320 [00:49<00:12, 2862.41 examples/s]
Tokenizing Prompts (num_proc=32):  76%|███████▌  | 105354/139320 [00:49<00:10, 3205.19 examples/s]
Tokenizing Prompts (num_proc=32):  76%|███████▋  | 106354/139320 [00:49<00:09, 3609.57 examples/s]
Tokenizing Prompts (num_proc=32):  77%|███████▋  | 107354/139320 [00:49<00:09, 3514.96 examples/s]
Tokenizing Prompts (num_proc=32):  78%|███████▊  | 108354/139320 [
+0: 00:50<00:08, 3738.95 examples/s]
Tokenizing Prompts (num_proc=32):  78%|███████▊  | 109354/139320 [00:50<00:09, 3238.58 examples/s]
Tokenizing Prompts (num_proc=32):  79%|███████▉  | 110708/139320 [00:51<00:09, 2899.07 examples/s]
Tokenizing Prompts (num_proc=32):  80%|████████  | 112062/139320 [00:51<00:08, 3101.43 examples/s]
Tokenizing Prompts (num_proc=32):  81%|████████▏ | 113416/139320 [00:51<00:07, 3566.65 examples/s]
Tokenizing Prompts (num_proc=32):  82%|████████▏ | 114124/139320 [00:52<00:12, 2042.61 examples/s]
Tokenizing Prompts (num_proc=32):  82%|████████▏ | 114832/139320 [00:53<00:13, 1840.08 examples/s]
Tokenizing Prompts (num_proc=32):  83%|████████▎ | 116186/139320 [00:53<00:08, 2614.11 examples/s]
Tokenizing Prompts (num_proc=32):  84%|████████▍ | 117540/139320 [00:53<00:07, 2892.08 examples/s]
Tokenizing Prompts (num_proc=32):  86%|███████�
+0: ��▌ | 119540/139320 [00:54<00:06, 3246.61 examples/s]
Tokenizing Prompts (num_proc=32):  87%|████████▋ | 120894/139320 [00:54<00:04, 4048.64 examples/s]
Tokenizing Prompts (num_proc=32):  88%|████████▊ | 122894/139320 [00:54<00:02, 5497.44 examples/s]
Tokenizing Prompts (num_proc=32):  89%|████████▉ | 124602/139320 [00:54<00:02, 5104.62 examples/s]
Tokenizing Prompts (num_proc=32):  90%|█████████ | 125602/139320 [00:55<00:02, 4783.46 examples/s]
Tokenizing Prompts (num_proc=32):  91%|█████████ | 126956/139320 [00:55<00:02, 4506.93 examples/s]
Tokenizing Prompts (num_proc=32):  92%|█████████▏| 128310/139320 [00:55<00:02, 5328.87 examples/s]
Tokenizing Prompts (num_proc=32):  93%|█████████▎| 129664/139320 [00:56<00:02, 3893.71 examples/s]
Tokenizing Prompts (num_proc=32):  94%|█████████▍| 130664/139320 [00:56<00:02, 2911.81 examples/s]
Tokenizing Prompts (num_pr
+0: oc=32):  94%|█████████▍| 131372/139320 [00:57<00:03, 2332.87 examples/s]
Tokenizing Prompts (num_proc=32):  95%|█████████▍| 131726/139320 [00:57<00:03, 2323.78 examples/s]
Tokenizing Prompts (num_proc=32):  95%|█████████▌| 132726/139320 [00:57<00:02, 3098.35 examples/s]
Tokenizing Prompts (num_proc=32):  96%|█████████▌| 134080/139320 [00:58<00:01, 2692.31 examples/s]
Tokenizing Prompts (num_proc=32):  97%|█████████▋| 135141/139320 [00:58<00:01, 3284.93 examples/s]
Tokenizing Prompts (num_proc=32):  98%|█████████▊| 136141/139320 [00:58<00:00, 3574.24 examples/s]
Tokenizing Prompts (num_proc=32):  98%|█████████▊| 136848/139320 [00:59<00:00, 2948.64 examples/s]
Tokenizing Prompts (num_proc=32):  99%|█████████▊| 137554/139320 [01:00<00:01, 1747.53 examples/s]
Tokenizing Prompts (num_proc=32):  99%|█████████▉| 137908/139320 [01:00<00:00,
+0:  1605.17 examples/s]
Tokenizing Prompts (num_proc=32):  99%|█████████▉| 138261/139320 [01:00<00:00, 1505.02 examples/s]
Tokenizing Prompts (num_proc=32):  99%|█████████▉| 138614/139320 [01:01<00:00, 913.54 examples/s] 
Tokenizing Prompts (num_proc=32): 100%|█████████▉| 138967/139320 [01:02<00:00, 930.38 examples/s]
Tokenizing Prompts (num_proc=32): 100%|██████████| 139320/139320 [01:02<00:00, 1045.36 examples/s]
Tokenizing Prompts (num_proc=32): 100%|██████████| 139320/139320 [01:02<00:00, 2233.69 examples/s]
+0: 
Dropping Long Sequences (>16384) (num_proc=32):   0%|          | 0/557277 [00:00<?, ? examples/s]
Dropping Long Sequences (>16384) (num_proc=32):   0%|          | 1000/557277 [00:00<04:50, 1915.99 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):   2%|▏         | 13000/557277 [00:00<00:20, 26952.73 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):   4%|▍         | 25000/557277 [00:01<00:18, 29215.00 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):   6%|▌         | 34000/557277 [00:01<00:13, 39016.04 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):   9%|▉         | 50000/557277 [00:01<00:08, 60896.10 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  11%|█         | 60000/557277 [00:01<00:07, 65934.72 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  14%|█▍        | 80000/557277 [00:01<00:05, 95001.62 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  17%|█▋        | 94000/557277 [00:01<00:04, 103389.65 examples/s]
Drop
+0: ping Long Sequences (>16384) (num_proc=32):  20%|█▉        | 111000/557277 [00:01<00:03, 119697.17 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  22%|██▏       | 125000/557277 [00:01<00:03, 114636.34 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  25%|██▍       | 138000/557277 [00:01<00:03, 111534.76 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  27%|██▋       | 151000/557277 [00:02<00:03, 116098.17 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  29%|██▉       | 164000/557277 [00:02<00:03, 115198.80 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  32%|███▏      | 177000/557277 [00:02<00:03, 118577.80 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  34%|███▍      | 190000/557277 [00:02<00:03, 115713.55 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  37%|███▋      | 206000/557277 [00:02<00:02, 125157.69 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  39%|█
+0: ██▉      | 219000/557277 [00:02<00:02, 125513.77 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  43%|████▎     | 238000/557277 [00:02<00:02, 143336.69 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  45%|████▌     | 253000/557277 [00:02<00:02, 141215.45 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  48%|████▊     | 268000/557277 [00:02<00:02, 133208.71 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  51%|█████     | 282000/557277 [00:03<00:02, 126252.48 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  53%|█████▎    | 295000/557277 [00:03<00:02, 125238.82 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  55%|█████▌    | 308000/557277 [00:03<00:02, 122967.64 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  58%|█████▊    | 322000/557277 [00:03<00:01, 123992.24 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  61%|██████    |
+0:  339000/557277 [00:03<00:01, 130517.70 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  64%|██████▍   | 357000/557277 [00:03<00:01, 142108.44 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  67%|██████▋   | 372000/557277 [00:03<00:01, 139753.32 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  69%|██████▉   | 387000/557277 [00:03<00:01, 126869.42 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  72%|███████▏  | 400000/557277 [00:03<00:01, 127258.50 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  74%|███████▍  | 413000/557277 [00:04<00:01, 127123.77 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  77%|███████▋  | 429905/557277 [00:04<00:00, 138365.22 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  80%|███████▉  | 443960/557277 [00:04<00:01, 110398.76 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  82%|███
+0: █████▏ | 456960/557277 [00:05<00:02, 35259.66 examples/s] 
Dropping Long Sequences (>16384) (num_proc=32):  84%|████████▎ | 465960/557277 [00:06<00:03, 22962.44 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  85%|████████▍ | 472960/557277 [00:07<00:04, 17186.52 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  86%|████████▌ | 477960/557277 [00:07<00:05, 14188.64 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  86%|████████▋ | 481960/557277 [00:07<00:05, 14648.28 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  87%|████████▋ | 485960/557277 [00:08<00:06, 11774.17 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  88%|████████▊ | 488960/557277 [00:08<00:05, 12046.82 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  88%|████████▊ | 491960/557277 [00:09<00:05, 10980.53 examples/s]
Dropping Long Sequences (>163
+0: 84) (num_proc=32):  89%|████████▊ | 493960/557277 [00:09<00:05, 10659.56 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  89%|████████▉ | 495960/557277 [00:09<00:06, 9950.64 examples/s] 
Dropping Long Sequences (>16384) (num_proc=32):  90%|████████▉ | 498960/557277 [00:09<00:05, 10584.62 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  90%|████████▉ | 500960/557277 [00:10<00:05, 10738.66 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  90%|█████████ | 502960/557277 [00:10<00:05, 9213.29 examples/s] 
Dropping Long Sequences (>16384) (num_proc=32):  91%|█████████ | 504960/557277 [00:10<00:05, 9940.90 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  91%|█████████ | 506960/557277 [00:10<00:04, 10279.89 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  91%|█████████▏| 508960/557277 [00:10<00:04, 10593.40 examples
+0: /s]
Dropping Long Sequences (>16384) (num_proc=32):  92%|█████████▏| 510960/557277 [00:11<00:05, 8779.40 examples/s] 
Dropping Long Sequences (>16384) (num_proc=32):  92%|█████████▏| 511960/557277 [00:11<00:05, 8385.73 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  92%|█████████▏| 513960/557277 [00:11<00:04, 10192.01 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  93%|█████████▎| 516960/557277 [00:11<00:03, 11641.89 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  93%|█████████▎| 518960/557277 [00:12<00:04, 8320.90 examples/s] 
Dropping Long Sequences (>16384) (num_proc=32):  93%|█████████▎| 520960/557277 [00:12<00:04, 9027.68 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  94%|█████████▍| 523960/557277 [00:12<00:02, 11397.96 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  94%|█████████▍|
+0:  525960/557277 [00:12<00:04, 7561.36 examples/s] 
Dropping Long Sequences (>16384) (num_proc=32):  95%|█████████▍| 527960/557277 [00:13<00:03, 8513.27 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  95%|█████████▌| 531960/557277 [00:13<00:02, 11448.42 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  96%|█████████▌| 533960/557277 [00:13<00:02, 8166.66 examples/s] 
Dropping Long Sequences (>16384) (num_proc=32):  96%|█████████▌| 535960/557277 [00:13<00:02, 8800.14 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  97%|█████████▋| 538960/557277 [00:14<00:01, 11308.67 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  97%|█████████▋| 540960/557277 [00:14<00:01, 11954.27 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  97%|█████████▋| 542960/557277 [00:14<00:01, 7554.51 examples/s] 
Dropping Long Sequences (>16384) (num_
+0: proc=32):  98%|█████████▊| 545960/557277 [00:14<00:01, 9808.17 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  98%|█████████▊| 548374/557277 [00:15<00:00, 10730.95 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  99%|█████████▊| 550204/557277 [00:15<00:00, 11161.94 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  99%|█████████▉| 552204/557277 [00:15<00:00, 7548.87 examples/s] 
Dropping Long Sequences (>16384) (num_proc=32): 100%|█████████▉| 554619/557277 [00:15<00:00, 8395.22 examples/s]
Dropping Long Sequences (>16384) (num_proc=32): 100%|█████████▉| 556033/557277 [00:16<00:00, 9031.49 examples/s]
Dropping Long Sequences (>16384) (num_proc=32): 100%|██████████| 557277/557277 [00:16<00:00, 7709.79 examples/s]
Dropping Long Sequences (>16384) (num_proc=32): 100%|██████████| 557277/557277 [00:16<00:00, 33835.09 exampl
+0: es/s]
+0: 
Drop Samples with Zero Trainable Tokens (num_proc=32):   0%|          | 0/554837 [00:00<?, ? examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):   0%|          | 1000/554837 [00:00<04:42, 1959.85 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):   4%|▍         | 23000/554837 [00:00<00:10, 48985.26 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):   6%|▋         | 36000/554837 [00:00<00:08, 59695.31 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):   9%|▉         | 50000/554837 [00:00<00:06, 74317.37 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  13%|█▎        | 70000/554837 [00:01<00:04, 102827.76 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  15%|█▌        | 84000/554837 [00:01<00:04, 97083.09 examples/s] 
Drop Samples with Zero Trainable Tokens (num_proc=32):  19%|█▊        | 103000/554837 [00:01<00:03, 118090.34 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  21
+0: %|██▏       | 118000/554837 [00:01<00:03, 124507.63 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  24%|██▍       | 133000/554837 [00:01<00:03, 119916.51 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  28%|██▊       | 153000/554837 [00:01<00:02, 135125.34 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  30%|███       | 168000/554837 [00:01<00:03, 125003.24 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  33%|███▎      | 182000/554837 [00:01<00:03, 119258.30 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  36%|███▌      | 201000/554837 [00:02<00:02, 133228.64 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  39%|███▉      | 215000/554837 [00:02<00:02, 132588.15 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  42%|████▏     | 234000/554837 [00:02<00:02, 138113.10 examples/s]
Drop Samples with Zero Trainable Tokens (num_pro
+0: c=32):  45%|████▍     | 249000/554837 [00:02<00:02, 136254.40 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  47%|████▋     | 263000/554837 [00:02<00:02, 120643.77 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  51%|█████     | 282000/554837 [00:02<00:02, 133695.23 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  53%|█████▎    | 296000/554837 [00:02<00:02, 127333.83 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  56%|█████▌    | 310000/554837 [00:02<00:01, 128235.65 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  60%|█████▉    | 332000/554837 [00:02<00:01, 151292.47 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  63%|██████▎   | 348000/554837 [00:03<00:01, 133584.79 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  66%|██████▌   | 364000/554837 [00:03<00:01, 136120.47 examples/s]
Dr
+0: op Samples with Zero Trainable Tokens (num_proc=32):  68%|██████▊   | 379000/554837 [00:03<00:01, 138670.22 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  71%|███████   | 394000/554837 [00:03<00:01, 121217.23 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  74%|███████▍  | 412000/554837 [00:03<00:01, 130639.18 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  77%|███████▋  | 429695/554837 [00:03<00:00, 142075.13 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  81%|████████  | 448116/554837 [00:03<00:00, 150184.24 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  84%|████████▎ | 464133/554837 [00:05<00:02, 31880.94 examples/s] 
Drop Samples with Zero Trainable Tokens (num_proc=32):  86%|████████▌ | 476133/554837 [00:06<00:04, 18749.06 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  87%|�
+0: ��███████▋ | 485133/554837 [00:07<00:04, 16796.97 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  89%|████████▊ | 491133/554837 [00:08<00:04, 14336.19 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  89%|████████▉ | 496133/554837 [00:08<00:03, 15334.25 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  90%|█████████ | 500133/554837 [00:09<00:04, 12941.84 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  91%|█████████ | 504133/554837 [00:09<00:03, 13408.36 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  91%|█████████▏| 507133/554837 [00:09<00:04, 11557.25 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  92%|█████████▏| 509133/554837 [00:09<00:03, 12148.39 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  92%|█████████▏| 512133/554837 [0
+0: 0:10<00:03, 12016.85 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  93%|█████████▎| 514133/554837 [00:10<00:03, 10656.34 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  93%|█████████▎| 516133/554837 [00:10<00:03, 10662.66 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  94%|█████████▎| 519133/554837 [00:10<00:02, 12610.14 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  94%|█████████▍| 521133/554837 [00:11<00:02, 11534.67 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  94%|█████████▍| 523133/554837 [00:11<00:03, 9693.21 examples/s] 
Drop Samples with Zero Trainable Tokens (num_proc=32):  95%|█████████▍| 526133/554837 [00:11<00:02, 11321.73 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  95%|█████████▌| 528133/554837 [00:11<00:02, 10269.27 examples/s]
Drop
+0:  Samples with Zero Trainable Tokens (num_proc=32):  96%|█████████▌| 530133/554837 [00:11<00:02, 9961.53 examples/s] 
Drop Samples with Zero Trainable Tokens (num_proc=32):  96%|█████████▌| 532133/554837 [00:12<00:02, 10105.42 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  96%|█████████▋| 534133/554837 [00:12<00:01, 11698.27 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  97%|█████████▋| 536133/554837 [00:12<00:01, 9700.42 examples/s] 
Drop Samples with Zero Trainable Tokens (num_proc=32):  97%|█████████▋| 538133/554837 [00:12<00:01, 10208.58 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  97%|█████████▋| 540133/554837 [00:12<00:01, 11084.40 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  98%|█████████▊| 542133/554837 [00:12<00:01, 11963.08 examples/s]
Drop Samples with Zero Trainable Tokens (
+0: num_proc=32):  98%|█████████▊| 543471/554837 [00:13<00:01, 9466.74 examples/s] 
Drop Samples with Zero Trainable Tokens (num_proc=32):  98%|█████████▊| 545471/554837 [00:13<00:00, 9677.24 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  99%|█████████▉| 548471/554837 [00:13<00:00, 11498.05 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  99%|█████████▉| 550485/554837 [00:13<00:00, 10074.77 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  99%|█████████▉| 551823/554837 [00:14<00:00, 9971.88 examples/s] 
Drop Samples with Zero Trainable Tokens (num_proc=32): 100%|█████████▉| 553161/554837 [00:14<00:00, 8053.74 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32): 100%|█████████▉| 554499/554837 [00:14<00:00, 8062.92 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32): 100%|███████
+0: ███| 554837/554837 [00:14<00:00, 37739.44 examples/s]
+0: 
Add position_id column (Sample Packing) (num_proc=32):   0%|          | 0/554837 [00:00<?, ? examples/s]
Add position_id column (Sample Packing) (num_proc=32):   0%|          | 1000/554837 [00:00<05:46, 1597.67 examples/s]
Add position_id column (Sample Packing) (num_proc=32):   5%|▍         | 25000/554837 [00:00<00:12, 43992.63 examples/s]
Add position_id column (Sample Packing) (num_proc=32):   7%|▋         | 37000/554837 [00:01<00:11, 44430.99 examples/s]
Add position_id column (Sample Packing) (num_proc=32):   9%|▉         | 50000/554837 [00:01<00:11, 45311.91 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  13%|█▎        | 72000/554837 [00:01<00:06, 73000.01 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  15%|█▌        | 84000/554837 [00:01<00:07, 62004.52 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  18%|█▊        | 100000/554837 [00:01<00:05, 77168.76 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  20%|�
+0: ��█        | 112000/554837 [00:01<00:06, 71065.57 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  23%|██▎       | 130000/554837 [00:02<00:06, 66618.25 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  28%|██▊       | 153000/554837 [00:02<00:04, 90665.76 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  30%|██▉       | 165000/554837 [00:02<00:05, 75887.88 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  32%|███▏      | 178000/554837 [00:02<00:05, 65749.26 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  36%|███▌      | 199000/554837 [00:03<00:04, 87000.11 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  38%|███▊      | 211000/554837 [00:03<00:04, 74508.99 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  42%|████▏     | 233000/554837 [00:03<00:03, 94663.45 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  44%|�
+0: ��███▍     | 245000/554837 [00:03<00:03, 82286.36 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  46%|████▋     | 257000/554837 [00:03<00:03, 87555.29 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  48%|████▊     | 268000/554837 [00:03<00:03, 76750.69 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  51%|█████     | 281000/554837 [00:04<00:03, 84180.11 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  52%|█████▏    | 291000/554837 [00:04<00:03, 72556.47 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  55%|█████▍    | 303000/554837 [00:04<00:03, 81910.15 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  56%|█████▋    | 313000/554837 [00:04<00:03, 69190.31 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  59%|█████▉    | 330000/554837 [00:04<00:02, 88376.26 examples/s]
Add position_id column (Sampl
+0: e Packing) (num_proc=32):  61%|██████▏   | 341000/554837 [00:04<00:02, 76232.59 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  65%|██████▍   | 358000/554837 [00:04<00:02, 89996.10 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  67%|██████▋   | 369000/554837 [00:05<00:02, 76127.19 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  69%|██████▉   | 383000/554837 [00:05<00:02, 85597.39 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  71%|███████   | 393000/554837 [00:05<00:02, 72225.78 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  73%|███████▎  | 407000/554837 [00:05<00:01, 81332.34 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  75%|███████▌  | 417000/554837 [00:05<00:01, 71899.94 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  78%|███████▊  | 432370/554837 [0
+0: 0:05<00:01, 88489.09 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  80%|███████▉  | 442777/554837 [00:06<00:01, 91375.64 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  82%|████████▏ | 453133/554837 [00:07<00:04, 23012.96 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  83%|████████▎ | 461133/554837 [00:08<00:06, 13873.02 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  84%|████████▍ | 467133/554837 [00:10<00:08, 10019.78 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  85%|████████▍ | 471133/554837 [00:10<00:07, 11036.79 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  86%|████████▌ | 475133/554837 [00:11<00:10, 7823.82 examples/s] 
Add position_id column (Sample Packing) (num_proc=32):  86%|████████▌ | 478133/554837 [00:11<00:08, 8849.55 examples/s]
Add position_id colum
+0: n (Sample Packing) (num_proc=32):  87%|████████▋ | 481133/554837 [00:11<00:07, 10008.78 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  87%|████████▋ | 484133/554837 [00:12<00:11, 6262.87 examples/s] 
Add position_id column (Sample Packing) (num_proc=32):  88%|████████▊ | 488133/554837 [00:12<00:08, 8271.37 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  89%|████████▊ | 491133/554837 [00:14<00:11, 5317.54 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  89%|████████▉ | 495133/554837 [00:14<00:08, 7263.97 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  90%|████████▉ | 498133/554837 [00:15<00:11, 4998.43 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  90%|█████████ | 500133/554837 [00:15<00:09, 5784.20 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  91%|█████�
+0: �███ | 504133/554837 [00:15<00:06, 8220.50 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  91%|█████████▏| 507133/554837 [00:16<00:09, 5052.37 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  92%|█████████▏| 512133/554837 [00:16<00:05, 7562.37 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  93%|█████████▎| 515133/554837 [00:18<00:07, 5041.61 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  94%|█████████▎| 519133/554837 [00:18<00:05, 6873.41 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  94%|█████████▍| 521133/554837 [00:18<00:04, 7544.38 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  94%|█████████▍| 523133/554837 [00:19<00:06, 4728.05 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  95%|█████████▍| 526133/554837 [00:19<00:04, 620
+0: 1.34 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  95%|█████████▌| 528133/554837 [00:19<00:03, 7183.01 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  96%|█████████▌| 530133/554837 [00:20<00:05, 4245.62 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  96%|█████████▋| 534133/554837 [00:20<00:03, 6508.27 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  97%|█████████▋| 536133/554837 [00:21<00:02, 7438.76 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  97%|█████████▋| 538133/554837 [00:22<00:03, 4495.75 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  98%|█████████▊| 542133/554837 [00:22<00:01, 6713.91 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  98%|█████████▊| 544133/554837 [00:22<00:01, 7406.40 examples/s]
Add position_id column (Sam
+0: ple Packing) (num_proc=32):  98%|█████████▊| 546471/554837 [00:23<00:01, 4779.95 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  99%|█████████▉| 549471/554837 [00:23<00:00, 6594.13 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  99%|█████████▉| 551471/554837 [00:23<00:00, 6907.91 examples/s]
Add position_id column (Sample Packing) (num_proc=32): 100%|█████████▉| 552823/554837 [00:23<00:00, 7013.38 examples/s]
Add position_id column (Sample Packing) (num_proc=32): 100%|█████████▉| 554161/554837 [00:24<00:00, 6923.27 examples/s]
Add position_id column (Sample Packing) (num_proc=32): 100%|██████████| 554837/554837 [00:24<00:00, 22653.41 examples/s]
+0: 
Saving the dataset (0/32 shards):   0%|          | 0/554837 [00:00<?, ? examples/s]
Saving the dataset (0/32 shards):   1%|          | 5000/554837 [00:00<00:37, 14705.11 examples/s]
Saving the dataset (0/32 shards):  34%|███▍      | 189000/554837 [00:00<00:00, 551790.10 examples/s]
Saving the dataset (1/32 shards):  60%|██████    | 333339/554837 [00:00<00:00, 551790.10 examples/s]
Saving the dataset (2/32 shards):  62%|██████▏   | 343695/554837 [00:00<00:00, 551790.10 examples/s]
Saving the dataset (3/32 shards):  62%|██████▏   | 343695/554837 [00:00<00:00, 551790.10 examples/s]
Saving the dataset (4/32 shards):  62%|██████▏   | 343695/554837 [00:00<00:00, 551790.10 examples/s]
Saving the dataset (5/32 shards):  62%|██████▏   | 343695/554837 [00:00<00:00, 551790.10 examples/s]
Saving the dataset (6/32 shards):  63%|██████▎   | 349034/554837 [00:00<00:00, 551790.10 examples/s]
Saving the dataset (7/32 shards):  65%|█�
+0: �████▌   | 362712/554837 [00:00<00:00, 551790.10 examples/s]
Saving the dataset (7/32 shards):  66%|██████▌   | 364051/554837 [00:00<00:00, 903715.48 examples/s]
Saving the dataset (8/32 shards):  66%|██████▌   | 364051/554837 [00:00<00:00, 903715.48 examples/s]
Saving the dataset (9/32 shards):  67%|██████▋   | 370390/554837 [00:00<00:00, 903715.48 examples/s]
Saving the dataset (10/32 shards):  68%|██████▊   | 375390/554837 [00:00<00:00, 903715.48 examples/s]
Saving the dataset (11/32 shards):  71%|███████   | 394068/554837 [00:00<00:00, 903715.48 examples/s]
Saving the dataset (12/32 shards):  71%|███████   | 394068/554837 [00:00<00:00, 903715.48 examples/s]
Saving the dataset (13/32 shards):  74%|███████▎  | 408407/554837 [00:00<00:00, 903715.48 examples/s]
Saving the dataset (14/32 shards):  74%|███████▍  | 412424/554837 [00:00<00:00, 903715.48 examples/s]
Saving the dataset (15/3
+0: 2 shards):  77%|███████▋  | 425763/554837 [00:00<00:00, 903715.48 examples/s]
Saving the dataset (16/32 shards):  77%|███████▋  | 425763/554837 [00:00<00:00, 903715.48 examples/s]
Saving the dataset (17/32 shards):  77%|███████▋  | 425763/554837 [00:00<00:00, 903715.48 examples/s]
Saving the dataset (18/32 shards):  78%|███████▊  | 431102/554837 [00:00<00:00, 903715.48 examples/s]
Saving the dataset (19/32 shards):  79%|███████▉  | 440794/554837 [00:00<00:00, 903715.48 examples/s]
Saving the dataset (20/32 shards):  79%|███████▉  | 440794/554837 [00:00<00:00, 903715.48 examples/s]
Saving the dataset (21/32 shards):  79%|███████▉  | 440794/554837 [00:00<00:00, 903715.48 examples/s]
Saving the dataset (22/32 shards):  79%|███████▉  | 440794/554837 [00:00<00:00, 903715.48 examples/s]
Saving the dataset (23/32 shards):  80%|███████▉  | 441794/554837 [00:00<00:00, 903715.4
+0: 8 examples/s]
Saving the dataset (24/32 shards):  80%|████████  | 444133/554837 [00:00<00:00, 903715.48 examples/s]
Saving the dataset (24/32 shards):  91%|█████████ | 503133/554837 [00:00<00:00, 584369.69 examples/s]
Saving the dataset (25/32 shards):  97%|█████████▋| 540471/554837 [00:01<00:00, 584369.69 examples/s]
Saving the dataset (26/32 shards):  99%|█████████▊| 546809/554837 [00:01<00:00, 584369.69 examples/s]
Saving the dataset (27/32 shards):  99%|█████████▊| 547147/554837 [00:01<00:00, 584369.69 examples/s]
Saving the dataset (28/32 shards):  99%|█████████▊| 547485/554837 [00:01<00:00, 584369.69 examples/s]
Saving the dataset (29/32 shards):  99%|█████████▉| 548823/554837 [00:01<00:00, 584369.69 examples/s]
Saving the dataset (30/32 shards): 100%|█████████▉| 553161/554837 [00:01<00:00, 584369.69 examples/s]
Saving the dataset (31/32 shards): 100%|
+0: ██████████| 554837/554837 [00:01<00:00, 584369.69 examples/s]
Saving the dataset (32/32 shards): 100%|██████████| 554837/554837 [00:01<00:00, 584369.69 examples/s]
Saving the dataset (32/32 shards): 100%|██████████| 554837/554837 [00:01<00:00, 443188.36 examples/s]
+0: [2025-11-23 14:44:11,059] [INFO] [axolotl.utils.data.shared.load_preprocessed_dataset:472] [PID:1308399] [RANK:0] Loading prepared dataset from disk at /lustre/fswork/projects/rech/dgo/udv55np/dataset_gemma/Nemotron-Super-49B-v1_5/split_0.25/d383a692316f2e59f08e99509bd12afa...
+0: jzxh159:1308399:1308399 [0] NCCL INFO Bootstrap : Using ibp24s0:10.100.6.121<0>
+0: jzxh159:1308399:1308399 [0] NCCL INFO NET/Plugin: No plugin found (libnccl-net.so)
+0: jzxh159:1308399:1308399 [0] NCCL INFO NET/Plugin: Plugin load returned 2 : libnccl-net.so: cannot open shared object file: No such file or directory : when loading libnccl-net.so
+0: jzxh159:1308399:1308399 [0] NCCL INFO NET/Plugin: Using internal network plugin.
+0: jzxh159:1308399:1308399 [0] NCCL INFO cudaDriverVersion 12080
+0: NCCL version 2.21.5+cuda12.4
+1: jzxh160:364774:364774 [3] NCCL INFO cudaDriverVersion 12080
+0: jzxh159:1308399:1308399 [0] NCCL INFO Comm config Blocking set to 1
+1: jzxh160:364774:364774 [3] NCCL INFO Bootstrap : Using ibp24s0:10.100.6.125<0>
+0: jzxh159:1308400:1308400 [1] NCCL INFO cudaDriverVersion 12080
+1: jzxh160:364774:364774 [3] NCCL INFO NET/Plugin: No plugin found (libnccl-net.so)
+1: jzxh160:364774:364774 [3] NCCL INFO NET/Plugin: Plugin load returned 2 : libnccl-net.so: cannot open shared object file: No such file or directory : when loading libnccl-net.so
+1: jzxh160:364774:364774 [3] NCCL INFO NET/Plugin: Using internal network plugin.
+0: jzxh159:1308400:1308400 [1] NCCL INFO Bootstrap : Using ibp24s0:10.100.6.121<0>
+2: jzxh161:3283536:3283536 [0] NCCL INFO cudaDriverVersion 12080
+0: jzxh159:1308400:1308400 [1] NCCL INFO NET/Plugin: No plugin found (libnccl-net.so)
+0: jzxh159:1308400:1308400 [1] NCCL INFO NET/Plugin: Plugin load returned 2 : libnccl-net.so: cannot open shared object file: No such file or directory : when loading libnccl-net.so
+0: jzxh159:1308400:1308400 [1] NCCL INFO NET/Plugin: Using internal network plugin.
+1: jzxh160:364772:364772 [1] NCCL INFO cudaDriverVersion 12080
+2: jzxh161:3283538:3283538 [2] NCCL INFO cudaDriverVersion 12080
+1: jzxh160:364772:364772 [1] NCCL INFO Bootstrap : Using ibp24s0:10.100.6.125<0>
+0: jzxh159:1308400:1308400 [1] NCCL INFO Comm config Blocking set to 1
+1: jzxh160:364772:364772 [1] NCCL INFO NET/Plugin: No plugin found (libnccl-net.so)
+1: jzxh160:364772:364772 [1] NCCL INFO NET/Plugin: Plugin load returned 2 : libnccl-net.so: cannot open shared object file: No such file or directory : when loading libnccl-net.so
+1: jzxh160:364772:364772 [1] NCCL INFO NET/Plugin: Using internal network plugin.
+1: jzxh160:364773:364773 [2] NCCL INFO cudaDriverVersion 12080
+1: jzxh160:364774:364774 [3] NCCL INFO Comm config Blocking set to 1
+1: jzxh160:364773:364773 [2] NCCL INFO Bootstrap : Using ibp24s0:10.100.6.125<0>
+1: jzxh160:364773:364773 [2] NCCL INFO NET/Plugin: No plugin found (libnccl-net.so)
+1: jzxh160:364773:364773 [2] NCCL INFO NET/Plugin: Plugin load returned 2 : libnccl-net.so: cannot open shared object file: No such file or directory : when loading libnccl-net.so
+1: jzxh160:364773:364773 [2] NCCL INFO NET/Plugin: Using internal network plugin.
+1: jzxh160:364771:364771 [0] NCCL INFO cudaDriverVersion 12080
+1: jzxh160:364772:364772 [1] NCCL INFO Comm config Blocking set to 1
+1: jzxh160:364771:364771 [0] NCCL INFO Bootstrap : Using ibp24s0:10.100.6.125<0>
+2: jzxh161:3283539:3283539 [3] NCCL INFO cudaDriverVersion 12080
+1: jzxh160:364771:364771 [0] NCCL INFO NET/Plugin: No plugin found (libnccl-net.so)
+1: jzxh160:364771:364771 [0] NCCL INFO NET/Plugin: Plugin load returned 2 : libnccl-net.so: cannot open shared object file: No such file or directory : when loading libnccl-net.so
+1: jzxh160:364771:364771 [0] NCCL INFO NET/Plugin: Using internal network plugin.
+1: jzxh160:364773:364773 [2] NCCL INFO Comm config Blocking set to 1
+2: jzxh161:3283537:3283537 [1] NCCL INFO cudaDriverVersion 12080
+1: jzxh160:364771:364771 [0] NCCL INFO Comm config Blocking set to 1
+0: jzxh159:1308402:1308402 [3] NCCL INFO cudaDriverVersion 12080
+0: jzxh159:1308401:1308401 [2] NCCL INFO cudaDriverVersion 12080
+2: jzxh161:3283536:3283536 [0] NCCL INFO Bootstrap : Using ibp24s0:10.100.6.129<0>
+2: jzxh161:3283538:3283538 [2] NCCL INFO Bootstrap : Using ibp24s0:10.100.6.129<0>
+2: jzxh161:3283539:3283539 [3] NCCL INFO Bootstrap : Using ibp24s0:10.100.6.129<0>
+2: jzxh161:3283537:3283537 [1] NCCL INFO Bootstrap : Using ibp24s0:10.100.6.129<0>
+0: jzxh159:1308402:1308402 [3] NCCL INFO Bootstrap : Using ibp24s0:10.100.6.121<0>
+0: jzxh159:1308401:1308401 [2] NCCL INFO Bootstrap : Using ibp24s0:10.100.6.121<0>
+0: jzxh159:1308402:1308402 [3] NCCL INFO NET/Plugin: No plugin found (libnccl-net.so)
+0: jzxh159:1308402:1308402 [3] NCCL INFO NET/Plugin: Plugin load returned 2 : libnccl-net.so: cannot open shared object file: No such file or directory : when loading libnccl-net.so
+0: jzxh159:1308402:1308402 [3] NCCL INFO NET/Plugin: Using internal network plugin.
+0: jzxh159:1308401:1308401 [2] NCCL INFO NET/Plugin: No plugin found (libnccl-net.so)
+0: jzxh159:1308401:1308401 [2] NCCL INFO NET/Plugin: Plugin load returned 2 : libnccl-net.so: cannot open shared object file: No such file or directory : when loading libnccl-net.so
+0: jzxh159:1308401:1308401 [2] NCCL INFO NET/Plugin: Using internal network plugin.
+0: jzxh159:1308401:1308401 [2] NCCL INFO Comm config Blocking set to 1
+0: jzxh159:1308402:1308402 [3] NCCL INFO Comm config Blocking set to 1
+2: jzxh161:3283539:3283539 [3] NCCL INFO NET/Plugin: No plugin found (libnccl-net.so)
+2: jzxh161:3283538:3283538 [2] NCCL INFO NET/Plugin: No plugin found (libnccl-net.so)
+2: jzxh161:3283539:3283539 [3] NCCL INFO NET/Plugin: Plugin load returned 2 : libnccl-net.so: cannot open shared object file: No such file or directory : when loading libnccl-net.so
+2: jzxh161:3283539:3283539 [3] NCCL INFO NET/Plugin: Using internal network plugin.
+2: jzxh161:3283538:3283538 [2] NCCL INFO NET/Plugin: Plugin load returned 2 : libnccl-net.so: cannot open shared object file: No such file or directory : when loading libnccl-net.so
+2: jzxh161:3283537:3283537 [1] NCCL INFO NET/Plugin: No plugin found (libnccl-net.so)
+2: jzxh161:3283538:3283538 [2] NCCL INFO NET/Plugin: Using internal network plugin.
+2: jzxh161:3283536:3283536 [0] NCCL INFO NET/Plugin: No plugin found (libnccl-net.so)
+2: jzxh161:3283537:3283537 [1] NCCL INFO NET/Plugin: Plugin load returned 2 : libnccl-net.so: cannot open shared object file: No such file or directory : when loading libnccl-net.so
+2: jzxh161:3283537:3283537 [1] NCCL INFO NET/Plugin: Using internal network plugin.
+2: jzxh161:3283536:3283536 [0] NCCL INFO NET/Plugin: Plugin load returned 2 : libnccl-net.so: cannot open shared object file: No such file or directory : when loading libnccl-net.so
+2: jzxh161:3283536:3283536 [0] NCCL INFO NET/Plugin: Using internal network plugin.
+2: jzxh161:3283538:3283538 [2] NCCL INFO Comm config Blocking set to 1
+2: jzxh161:3283537:3283537 [1] NCCL INFO Comm config Blocking set to 1
+2: jzxh161:3283539:3283539 [3] NCCL INFO Comm config Blocking set to 1
+2: jzxh161:3283536:3283536 [0] NCCL INFO Comm config Blocking set to 1
+3: jzxh162:3160742:3160742 [0] NCCL INFO cudaDriverVersion 12080
+3: jzxh162:3160742:3160742 [0] NCCL INFO Bootstrap : Using ibp24s0:10.100.6.133<0>
+3: jzxh162:3160743:3160743 [1] NCCL INFO cudaDriverVersion 12080
+3: jzxh162:3160743:3160743 [1] NCCL INFO Bootstrap : Using ibp24s0:10.100.6.133<0>
+3: jzxh162:3160743:3160743 [1] NCCL INFO NET/Plugin: No plugin found (libnccl-net.so)
+3: jzxh162:3160742:3160742 [0] NCCL INFO NET/Plugin: No plugin found (libnccl-net.so)
+3: jzxh162:3160743:3160743 [1] NCCL INFO NET/Plugin: Plugin load returned 2 : libnccl-net.so: cannot open shared object file: No such file or directory : when loading libnccl-net.so
+3: jzxh162:3160743:3160743 [1] NCCL INFO NET/Plugin: Using internal network plugin.
+3: jzxh162:3160742:3160742 [0] NCCL INFO NET/Plugin: Plugin load returned 2 : libnccl-net.so: cannot open shared object file: No such file or directory : when loading libnccl-net.so
+3: jzxh162:3160742:3160742 [0] NCCL INFO NET/Plugin: Using internal network plugin.
+3: jzxh162:3160745:3160745 [3] NCCL INFO cudaDriverVersion 12080
+3: jzxh162:3160742:3160742 [0] NCCL INFO Comm config Blocking set to 1
+3: jzxh162:3160743:3160743 [1] NCCL INFO Comm config Blocking set to 1
+3: jzxh162:3160745:3160745 [3] NCCL INFO Bootstrap : Using ibp24s0:10.100.6.133<0>
+3: jzxh162:3160745:3160745 [3] NCCL INFO NET/Plugin: No plugin found (libnccl-net.so)
+3: jzxh162:3160745:3160745 [3] NCCL INFO NET/Plugin: Plugin load returned 2 : libnccl-net.so: cannot open shared object file: No such file or directory : when loading libnccl-net.so
+3: jzxh162:3160745:3160745 [3] NCCL INFO NET/Plugin: Using internal network plugin.
+3: jzxh162:3160744:3160744 [2] NCCL INFO cudaDriverVersion 12080
+3: jzxh162:3160745:3160745 [3] NCCL INFO Comm config Blocking set to 1
+3: jzxh162:3160744:3160744 [2] NCCL INFO Bootstrap : Using ibp24s0:10.100.6.133<0>
+3: jzxh162:3160744:3160744 [2] NCCL INFO NET/Plugin: No plugin found (libnccl-net.so)
+3: jzxh162:3160744:3160744 [2] NCCL INFO NET/Plugin: Plugin load returned 2 : libnccl-net.so: cannot open shared object file: No such file or directory : when loading libnccl-net.so
+3: jzxh162:3160744:3160744 [2] NCCL INFO NET/Plugin: Using internal network plugin.
+3: jzxh162:3160744:3160744 [2] NCCL INFO Comm config Blocking set to 1
+2: jzxh161:3283536:3284336 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [1]mlx5_1:1/IB [2]mlx5_2:1/IB [3]mlx5_3:1/IB [RO]; OOB ibp24s0:10.100.6.129<0>
+2: jzxh161:3283538:3284335 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [1]mlx5_1:1/IB [2]mlx5_2:1/IB [3]mlx5_3:1/IB [RO]; OOB ibp24s0:10.100.6.129<0>
+2: jzxh161:3283539:3284338 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [1]mlx5_1:1/IB [2]mlx5_2:1/IB [3]mlx5_3:1/IB [RO]; OOB ibp24s0:10.100.6.129<0>
+2: jzxh161:3283536:3284336 [0] NCCL INFO Using non-device net plugin version 0
+2: jzxh161:3283536:3284336 [0] NCCL INFO Using network IB
+2: jzxh161:3283537:3284337 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [1]mlx5_1:1/IB [2]mlx5_2:1/IB [3]mlx5_3:1/IB [RO]; OOB ibp24s0:10.100.6.129<0>
+2: jzxh161:3283538:3284335 [2] NCCL INFO Using non-device net plugin version 0
+2: jzxh161:3283538:3284335 [2] NCCL INFO Using network IB
+2: jzxh161:3283539:3284338 [3] NCCL INFO Using non-device net plugin version 0
+2: jzxh161:3283539:3284338 [3] NCCL INFO Using network IB
+2: jzxh161:3283537:3284337 [1] NCCL INFO Using non-device net plugin version 0
+2: jzxh161:3283537:3284337 [1] NCCL INFO Using network IB
+0: jzxh159:1308401:1310257 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [1]mlx5_1:1/IB [2]mlx5_2:1/IB [3]mlx5_3:1/IB [RO]; OOB ibp24s0:10.100.6.121<0>
+0: jzxh159:1308400:1310256 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [1]mlx5_1:1/IB [2]mlx5_2:1/IB [3]mlx5_3:1/IB [RO]; OOB ibp24s0:10.100.6.121<0>
+0: jzxh159:1308399:1310255 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [1]mlx5_1:1/IB [2]mlx5_2:1/IB [3]mlx5_3:1/IB [RO]; OOB ibp24s0:10.100.6.121<0>
+0: jzxh159:1308401:1310257 [2] NCCL INFO Using non-device net plugin version 0
+0: jzxh159:1308401:1310257 [2] NCCL INFO Using network IB
+0: jzxh159:1308402:1310258 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [1]mlx5_1:1/IB [2]mlx5_2:1/IB [3]mlx5_3:1/IB [RO]; OOB ibp24s0:10.100.6.121<0>
+0: jzxh159:1308400:1310256 [1] NCCL INFO Using non-device net plugin version 0
+0: jzxh159:1308400:1310256 [1] NCCL INFO Using network IB
+0: jzxh159:1308399:1310255 [0] NCCL INFO Using non-device net plugin version 0
+0: jzxh159:1308399:1310255 [0] NCCL INFO Using network IB
+0: jzxh159:1308402:1310258 [3] NCCL INFO Using non-device net plugin version 0
+0: jzxh159:1308402:1310258 [3] NCCL INFO Using network IB
+1: jzxh160:364774:365551 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [1]mlx5_1:1/IB [2]mlx5_2:1/IB [3]mlx5_3:1/IB [RO]; OOB ibp24s0:10.100.6.125<0>
+1: jzxh160:364772:365552 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [1]mlx5_1:1/IB [2]mlx5_2:1/IB [3]mlx5_3:1/IB [RO]; OOB ibp24s0:10.100.6.125<0>
+1: jzxh160:364771:365554 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [1]mlx5_1:1/IB [2]mlx5_2:1/IB [3]mlx5_3:1/IB [RO]; OOB ibp24s0:10.100.6.125<0>
+1: jzxh160:364774:365551 [3] NCCL INFO Using non-device net plugin version 0
+1: jzxh160:364774:365551 [3] NCCL INFO Using network IB
+1: jzxh160:364772:365552 [1] NCCL INFO Using non-device net plugin version 0
+1: jzxh160:364772:365552 [1] NCCL INFO Using network IB
+1: jzxh160:364773:365553 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [1]mlx5_1:1/IB [2]mlx5_2:1/IB [3]mlx5_3:1/IB [RO]; OOB ibp24s0:10.100.6.125<0>
+1: jzxh160:364771:365554 [0] NCCL INFO Using non-device net plugin version 0
+1: jzxh160:364771:365554 [0] NCCL INFO Using network IB
+1: jzxh160:364773:365553 [2] NCCL INFO Using non-device net plugin version 0
+1: jzxh160:364773:365553 [2] NCCL INFO Using network IB
+3: jzxh162:3160744:3161553 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [1]mlx5_1:1/IB [2]mlx5_2:1/IB [3]mlx5_3:1/IB [RO]; OOB ibp24s0:10.100.6.133<0>
+3: jzxh162:3160745:3161552 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [1]mlx5_1:1/IB [2]mlx5_2:1/IB [3]mlx5_3:1/IB [RO]; OOB ibp24s0:10.100.6.133<0>
+3: jzxh162:3160743:3161551 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [1]mlx5_1:1/IB [2]mlx5_2:1/IB [3]mlx5_3:1/IB [RO]; OOB ibp24s0:10.100.6.133<0>
+3: jzxh162:3160744:3161553 [2] NCCL INFO Using non-device net plugin version 0
+3: jzxh162:3160744:3161553 [2] NCCL INFO Using network IB
+3: jzxh162:3160742:3161550 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [1]mlx5_1:1/IB [2]mlx5_2:1/IB [3]mlx5_3:1/IB [RO]; OOB ibp24s0:10.100.6.133<0>
+3: jzxh162:3160745:3161552 [3] NCCL INFO Using non-device net plugin version 0
+3: jzxh162:3160745:3161552 [3] NCCL INFO Using network IB
+3: jzxh162:3160743:3161551 [1] NCCL INFO Using non-device net plugin version 0
+3: jzxh162:3160743:3161551 [1] NCCL INFO Using network IB
+3: jzxh162:3160742:3161550 [0] NCCL INFO Using non-device net plugin version 0
+3: jzxh162:3160742:3161550 [0] NCCL INFO Using network IB
+0: jzxh159:1308399:1310255 [0] NCCL INFO ncclCommInitRank comm 0x5650a3b6e180 rank 0 nranks 16 cudaDev 0 nvmlDev 0 busId 1b000 commId 0x524c94fe7e5cd79a - Init START
+0: jzxh159:1308400:1310256 [1] NCCL INFO ncclCommInitRank comm 0x5576720d8870 rank 1 nranks 16 cudaDev 1 nvmlDev 1 busId 2c000 commId 0x524c94fe7e5cd79a - Init START
+0: jzxh159:1308402:1310258 [3] NCCL INFO ncclCommInitRank comm 0x55cdfc201020 rank 3 nranks 16 cudaDev 3 nvmlDev 3 busId ad000 commId 0x524c94fe7e5cd79a - Init START
+0: jzxh159:1308401:1310257 [2] NCCL INFO ncclCommInitRank comm 0x55cb335512a0 rank 2 nranks 16 cudaDev 2 nvmlDev 2 busId 9d000 commId 0x524c94fe7e5cd79a - Init START
+2: jzxh161:3283539:3284338 [3] NCCL INFO ncclCommInitRank comm 0x557efcb6a4d0 rank 11 nranks 16 cudaDev 3 nvmlDev 3 busId ad000 commId 0x524c94fe7e5cd79a - Init START
+2: jzxh161:3283537:3284337 [1] NCCL INFO ncclCommInitRank comm 0x55f84f114b10 rank 9 nranks 16 cudaDev 1 nvmlDev 1 busId 2c000 commId 0x524c94fe7e5cd79a - Init START
+3: jzxh162:3160743:3161551 [1] NCCL INFO ncclCommInitRank comm 0x55dd2bde7570 rank 13 nranks 16 cudaDev 1 nvmlDev 1 busId 2c000 commId 0x524c94fe7e5cd79a - Init START
+3: jzxh162:3160745:3161552 [3] NCCL INFO ncclCommInitRank comm 0x55c152c5c3b0 rank 15 nranks 16 cudaDev 3 nvmlDev 3 busId ad000 commId 0x524c94fe7e5cd79a - Init START
+3: jzxh162:3160744:3161553 [2] NCCL INFO ncclCommInitRank comm 0x56427a552c70 rank 14 nranks 16 cudaDev 2 nvmlDev 2 busId 9d000 commId 0x524c94fe7e5cd79a - Init START
+3: jzxh162:3160742:3161550 [0] NCCL INFO ncclCommInitRank comm 0x5649a9566180 rank 12 nranks 16 cudaDev 0 nvmlDev 0 busId 1b000 commId 0x524c94fe7e5cd79a - Init START
+1: jzxh160:364773:365553 [2] NCCL INFO ncclCommInitRank comm 0x56520ef52d20 rank 6 nranks 16 cudaDev 2 nvmlDev 2 busId 9d000 commId 0x524c94fe7e5cd79a - Init START
+1: jzxh160:364771:365554 [0] NCCL INFO ncclCommInitRank comm 0x55a894a256f0 rank 4 nranks 16 cudaDev 0 nvmlDev 0 busId 1b000 commId 0x524c94fe7e5cd79a - Init START
+1: jzxh160:364774:365551 [3] NCCL INFO ncclCommInitRank comm 0x557fffcd2ae0 rank 7 nranks 16 cudaDev 3 nvmlDev 3 busId ad000 commId 0x524c94fe7e5cd79a - Init START
+1: jzxh160:364772:365552 [1] NCCL INFO ncclCommInitRank comm 0x556d3b785610 rank 5 nranks 16 cudaDev 1 nvmlDev 1 busId 2c000 commId 0x524c94fe7e5cd79a - Init START
+2: jzxh161:3283538:3284335 [2] NCCL INFO ncclCommInitRank comm 0x5643a3152d00 rank 10 nranks 16 cudaDev 2 nvmlDev 2 busId 9d000 commId 0x524c94fe7e5cd79a - Init START
+2: jzxh161:3283536:3284336 [0] NCCL INFO ncclCommInitRank comm 0x55c8d4c4af80 rank 8 nranks 16 cudaDev 0 nvmlDev 0 busId 1b000 commId 0x524c94fe7e5cd79a - Init START
+2: jzxh161:3283536:3284336 [0] NCCL INFO Setting affinity for GPU 0 to ffffff,00000000,00000000,00ffffff
+2: jzxh161:3283536:3284336 [0] NCCL INFO NVLS multicast support is not available on dev 0
+0: jzxh159:1308401:1310257 [2] NCCL INFO Setting affinity for GPU 2 to ff,ffff0000,00000000,000000ff,ffff0000,00000000
+0: jzxh159:1308401:1310257 [2] NCCL INFO NVLS multicast support is not available on dev 2
+0: jzxh159:1308402:1310258 [3] NCCL INFO Setting affinity for GPU 3 to ffffff00,00000000,00000000,ffffff00,00000000,00000000
+0: jzxh159:1308402:1310258 [3] NCCL INFO NVLS multicast support is not available on dev 3
+0: jzxh159:1308400:1310256 [1] NCCL INFO Setting affinity for GPU 1 to ffff,ff000000,00000000,0000ffff,ff000000
+0: jzxh159:1308400:1310256 [1] NCCL INFO NVLS multicast support is not available on dev 1
+0: jzxh159:1308399:1310255 [0] NCCL INFO Setting affinity for GPU 0 to ffffff,00000000,00000000,00ffffff
+0: jzxh159:1308399:1310255 [0] NCCL INFO NVLS multicast support is not available on dev 0
+2: jzxh161:3283538:3284335 [2] NCCL INFO Setting affinity for GPU 2 to ff,ffff0000,00000000,000000ff,ffff0000,00000000
+2: jzxh161:3283538:3284335 [2] NCCL INFO NVLS multicast support is not available on dev 2
+2: jzxh161:3283539:3284338 [3] NCCL INFO Setting affinity for GPU 3 to ffffff00,00000000,00000000,ffffff00,00000000,00000000
+2: jzxh161:3283537:3284337 [1] NCCL INFO Setting affinity for GPU 1 to ffff,ff000000,00000000,0000ffff,ff000000
+2: jzxh161:3283537:3284337 [1] NCCL INFO NVLS multicast support is not available on dev 1
+2: jzxh161:3283539:3284338 [3] NCCL INFO NVLS multicast support is not available on dev 3
+3: jzxh162:3160742:3161550 [0] NCCL INFO Setting affinity for GPU 0 to ffffff,00000000,00000000,00ffffff
+3: jzxh162:3160742:3161550 [0] NCCL INFO NVLS multicast support is not available on dev 0
+3: jzxh162:3160743:3161551 [1] NCCL INFO Setting affinity for GPU 1 to ffff,ff000000,00000000,0000ffff,ff000000
+3: jzxh162:3160743:3161551 [1] NCCL INFO NVLS multicast support is not available on dev 1
+3: jzxh162:3160744:3161553 [2] NCCL INFO Setting affinity for GPU 2 to ff,ffff0000,00000000,000000ff,ffff0000,00000000
+1: jzxh160:364771:365554 [0] NCCL INFO Setting affinity for GPU 0 to ffffff,00000000,00000000,00ffffff
+1: jzxh160:364771:365554 [0] NCCL INFO NVLS multicast support is not available on dev 0
+3: jzxh162:3160744:3161553 [2] NCCL INFO NVLS multicast support is not available on dev 2
+3: jzxh162:3160745:3161552 [3] NCCL INFO Setting affinity for GPU 3 to ffffff00,00000000,00000000,ffffff00,00000000,00000000
+3: jzxh162:3160745:3161552 [3] NCCL INFO NVLS multicast support is not available on dev 3
+1: jzxh160:364772:365552 [1] NCCL INFO Setting affinity for GPU 1 to ffff,ff000000,00000000,0000ffff,ff000000
+1: jzxh160:364772:365552 [1] NCCL INFO NVLS multicast support is not available on dev 1
+1: jzxh160:364773:365553 [2] NCCL INFO Setting affinity for GPU 2 to ff,ffff0000,00000000,000000ff,ffff0000,00000000
+1: jzxh160:364773:365553 [2] NCCL INFO NVLS multicast support is not available on dev 2
+1: jzxh160:364774:365551 [3] NCCL INFO Setting affinity for GPU 3 to ffffff00,00000000,00000000,ffffff00,00000000,00000000
+1: jzxh160:364774:365551 [3] NCCL INFO NVLS multicast support is not available on dev 3
+0: jzxh159:1308399:1310255 [0] NCCL INFO comm 0x5650a3b6e180 rank 0 nRanks 16 nNodes 4 localRanks 4 localRank 0 MNNVL 0
+0: jzxh159:1308400:1310256 [1] NCCL INFO comm 0x5576720d8870 rank 1 nRanks 16 nNodes 4 localRanks 4 localRank 1 MNNVL 0
+0: jzxh159:1308402:1310258 [3] NCCL INFO comm 0x55cdfc201020 rank 3 nRanks 16 nNodes 4 localRanks 4 localRank 3 MNNVL 0
+0: jzxh159:1308399:1310255 [0] NCCL INFO Channel 00/16 :    0   1   2   3   4   5   6   7   8   9  10  11  12  13  14  15
+0: jzxh159:1308401:1310257 [2] NCCL INFO comm 0x55cb335512a0 rank 2 nRanks 16 nNodes 4 localRanks 4 localRank 2 MNNVL 0
+0: jzxh159:1308399:1310255 [0] NCCL INFO Channel 01/16 :    0   3   2   5   4   7   6   9   8  11  10  13  12  15  14   1
+0: jzxh159:1308399:1310255 [0] NCCL INFO Channel 02/16 :    0   3   6   5   4   7  10   9   8  11  14  13  12  15   2   1
+0: jzxh159:1308399:1310255 [0] NCCL INFO Channel 03/16 :    0   1   2   7   4   5   6  11   8   9  10  15  12  13  14   3
+0: jzxh159:1308399:1310255 [0] NCCL INFO Channel 04/16 :    0   1   2   3   4   5   6   7   8   9  10  11  12  13  14  15
+0: jzxh159:1308399:1310255 [0] NCCL INFO Channel 05/16 :    0   3   2   5   4   7   6   9   8  11  10  13  12  15  14   1
+0: jzxh159:1308399:1310255 [0] NCCL INFO Channel 06/16 :    0   3   6   5   4   7  10   9   8  11  14  13  12  15   2   1
+0: jzxh159:1308400:1310256 [1] NCCL INFO Trees [0] 2/-1/-1->1->0 [1] 2/9/-1->1->-1 [2] 3/-1/-1->1->0 [3] 0/-1/-1->1->3 [4] -1/-1/-1->1->2 [5] 3/9/-1->1->-1 [6] -1/-1/-1->1->3 [7] 0/-1/-1->1->2 [8] 2/-1/-1->1->0 [9] 2/-1/-1->1->5 [10] 3/-1/-1->1->0 [11] 0/-1/-1->1->3 [12] -1/-1/-1->1->2 [13] 3/-1/-1->1->5 [14] -1/-1/-1->1->3 [15] 0/-1/-1->1->2
+0: jzxh159:1308402:1310258 [3] NCCL INFO Trees [0] -1/-1/-1->3->2 [1] 0/-1/-1->3->2 [2] -1/-1/-1->3->1 [3] 1/11/-1->3->-1 [4] 2/-1/-1->3->0 [5] 0/-1/-1->3->1 [6] 1/-1/-1->3->0 [7] 2/11/-1->3->-1 [8] -1/-1/-1->3->2 [9] 0/-1/-1->3->2 [10] -1/-1/-1->3->1 [11] 1/-1/-1->3->7 [12] 2/-1/-1->3->0 [13] 0/-1/-1->3->1 [14] 1/-1/-1->3->0 [15] 2/-1/-1->3->7
+0: jzxh159:1308399:1310255 [0] NCCL INFO Channel 07/16 :    0   1   2   7   4   5   6  11   8   9  10  15  12  13  14   3
+0: jzxh159:1308400:1310256 [1] NCCL INFO P2P Chunksize set to 131072
+0: jzxh159:1308402:1310258 [3] NCCL INFO P2P Chunksize set to 131072
+0: jzxh159:1308399:1310255 [0] NCCL INFO Channel 08/16 :    0   1   2   3   4   5   6   7   8   9  10  11  12  13  14  15
+0: jzxh159:1308401:1310257 [2] NCCL INFO Trees [0] 3/-1/-1->2->1 [1] 3/-1/-1->2->1 [2] 0/10/-1->2->-1 [3] -1/-1/-1->2->0 [4] 1/-1/-1->2->3 [5] -1/-1/-1->2->0 [6] 0/10/-1->2->-1 [7] 1/-1/-1->2->3 [8] 3/-1/-1->2->1 [9] 3/-1/-1->2->1 [10] 0/-1/-1->2->6 [11] -1/-1/-1->2->0 [12] 1/-1/-1->2->3 [13] -1/-1/-1->2->0 [14] 0/-1/-1->2->6 [15] 1/-1/-1->2->3
+0: jzxh159:1308399:1310255 [0] NCCL INFO Channel 09/16 :    0   3   2   5   4   7   6   9   8  11  10  13  12  15  14   1
+0: jzxh159:1308401:1310257 [2] NCCL INFO P2P Chunksize set to 131072
+0: jzxh159:1308399:1310255 [0] NCCL INFO Channel 10/16 :    0   3   6   5   4   7  10   9   8  11  14  13  12  15   2   1
+0: jzxh159:1308399:1310255 [0] NCCL INFO Channel 11/16 :    0   1   2   7   4   5   6  11   8   9  10  15  12  13  14   3
+0: jzxh159:1308399:1310255 [0] NCCL INFO Channel 12/16 :    0   1   2   3   4   5   6   7   8   9  10  11  12  13  14  15
+0: jzxh159:1308399:1310255 [0] NCCL INFO Channel 13/16 :    0   3   2   5   4   7   6   9   8  11  10  13  12  15  14   1
+0: jzxh159:1308399:1310255 [0] NCCL INFO Channel 14/16 :    0   3   6   5   4   7  10   9   8  11  14  13  12  15   2   1
+1: jzxh160:364774:365551 [3] NCCL INFO comm 0x557fffcd2ae0 rank 7 nRanks 16 nNodes 4 localRanks 4 localRank 3 MNNVL 0
+1: jzxh160:364773:365553 [2] NCCL INFO comm 0x56520ef52d20 rank 6 nRanks 16 nNodes 4 localRanks 4 localRank 2 MNNVL 0
+3: jzxh162:3160744:3161553 [2] NCCL INFO comm 0x56427a552c70 rank 14 nRanks 16 nNodes 4 localRanks 4 localRank 2 MNNVL 0
+3: jzxh162:3160745:3161552 [3] NCCL INFO comm 0x55c152c5c3b0 rank 15 nRanks 16 nNodes 4 localRanks 4 localRank 3 MNNVL 0
+3: jzxh162:3160743:3161551 [1] NCCL INFO comm 0x55dd2bde7570 rank 13 nRanks 16 nNodes 4 localRanks 4 localRank 1 MNNVL 0
+3: jzxh162:3160742:3161550 [0] NCCL INFO comm 0x5649a9566180 rank 12 nRanks 16 nNodes 4 localRanks 4 localRank 0 MNNVL 0
+0: jzxh159:1308399:1310255 [0] NCCL INFO Channel 15/16 :    0   1   2   7   4   5   6  11   8   9  10  15  12  13  14   3
+0: jzxh159:1308399:1310255 [0] NCCL INFO Trees [0] 1/8/-1->0->-1 [1] -1/-1/-1->0->3 [2] 1/-1/-1->0->2 [3] 2/-1/-1->0->1 [4] 3/8/-1->0->-1 [5] 2/-1/-1->0->3 [6] 3/-1/-1->0->2 [7] -1/-1/-1->0->1 [8] 1/-1/-1->0->4 [9] -1/-1/-1->0->3 [10] 1/-1/-1->0->2 [11] 2/-1/-1->0->1 [12] 3/-1/-1->0->4 [13] 2/-1/-1->0->3 [14] 3/-1/-1->0->2 [15] -1/-1/-1->0->1
+0: jzxh159:1308399:1310255 [0] NCCL INFO P2P Chunksize set to 131072
+1: jzxh160:364774:365551 [3] NCCL INFO Trees [0] -1/-1/-1->7->6 [1] 4/-1/-1->7->6 [2] -1/-1/-1->7->5 [3] 5/-1/-1->7->11 [4] 6/-1/-1->7->4 [5] 4/-1/-1->7->5 [6] 5/-1/-1->7->4 [7] 6/-1/-1->7->11 [8] -1/-1/-1->7->6 [9] 4/-1/-1->7->6 [10] -1/-1/-1->7->5 [11] 5/11/3->7->15 [12] 6/-1/-1->7->4 [13] 4/-1/-1->7->5 [14] 5/-1/-1->7->4 [15] 6/11/3->7->15
+1: jzxh160:364774:365551 [3] NCCL INFO P2P Chunksize set to 131072
+3: jzxh162:3160744:3161553 [2] NCCL INFO Trees [0] 15/-1/-1->14->13 [1] 15/-1/-1->14->13 [2] 12/-1/-1->14->10 [3] -1/-1/-1->14->12 [4] 13/-1/-1->14->15 [5] -1/-1/-1->14->12 [6] 12/-1/-1->14->10 [7] 13/-1/-1->14->15 [8] 15/-1/-1->14->13 [9] 15/-1/-1->14->13 [10] 12/6/-1->14->-1 [11] -1/-1/-1->14->12 [12] 13/-1/-1->14->15 [13] -1/-1/-1->14->12 [14] 12/6/-1->14->-1 [15] 13/-1/-1->14->15
+3: jzxh162:3160744:3161553 [2] NCCL INFO P2P Chunksize set to 131072
+3: jzxh162:3160743:3161551 [1] NCCL INFO Trees [0] 14/-1/-1->13->12 [1] 14/-1/-1->13->9 [2] 15/-1/-1->13->12 [3] 12/-1/-1->13->15 [4] -1/-1/-1->13->14 [5] 15/-1/-1->13->9 [6] -1/-1/-1->13->15 [7] 12/-1/-1->13->14 [8] 14/-1/-1->13->12 [9] 14/5/-1->13->-1 [10] 15/-1/-1->13->12 [11] 12/-1/-1->13->15 [12] -1/-1/-1->13->14 [13] 15/5/-1->13->-1 [14] -1/-1/-1->13->15 [15] 12/-1/-1->13->14
+3: jzxh162:3160743:3161551 [1] NCCL INFO P2P Chunksize set to 131072
+1: jzxh160:364773:365553 [2] NCCL INFO Trees [0] 7/-1/-1->6->5 [1] 7/-1/-1->6->5 [2] 4/-1/-1->6->10 [3] -1/-1/-1->6->4 [4] 5/-1/-1->6->7 [5] -1/-1/-1->6->4 [6] 4/-1/-1->6->10 [7] 5/-1/-1->6->7 [8] 7/-1/-1->6->5 [9] 7/-1/-1->6->5 [10] 4/10/2->6->14 [11] -1/-1/-1->6->4 [12] 5/-1/-1->6->7 [13] -1/-1/-1->6->4 [14] 4/10/2->6->14 [15] 5/-1/-1->6->7
+1: jzxh160:364773:365553 [2] NCCL INFO P2P Chunksize set to 131072
+1: jzxh160:364772:365552 [1] NCCL INFO comm 0x556d3b785610 rank 5 nRanks 16 nNodes 4 localRanks 4 localRank 1 MNNVL 0
+1: jzxh160:364771:365554 [0] NCCL INFO comm 0x55a894a256f0 rank 4 nRanks 16 nNodes 4 localRanks 4 localRank 0 MNNVL 0
+1: jzxh160:364772:365552 [1] NCCL INFO Trees [0] 6/-1/-1->5->4 [1] 6/-1/-1->5->9 [2] 7/-1/-1->5->4 [3] 4/-1/-1->5->7 [4] -1/-1/-1->5->6 [5] 7/-1/-1->5->9 [6] -1/-1/-1->5->7 [7] 4/-1/-1->5->6 [8] 6/-1/-1->5->4 [9] 6/9/1->5->13 [10] 7/-1/-1->5->4 [11] 4/-1/-1->5->7 [12] -1/-1/-1->5->6 [13] 7/9/1->5->13 [14] -1/-1/-1->5->7 [15] 4/-1/-1->5->6
+1: jzxh160:364772:365552 [1] NCCL INFO P2P Chunksize set to 131072
+1: jzxh160:364771:365554 [0] NCCL INFO Trees [0] 5/-1/-1->4->8 [1] -1/-1/-1->4->7 [2] 5/-1/-1->4->6 [3] 6/-1/-1->4->5 [4] 7/-1/-1->4->8 [5] 6/-1/-1->4->7 [6] 7/-1/-1->4->6 [7] -1/-1/-1->4->5 [8] 5/8/0->4->12 [9] -1/-1/-1->4->7 [10] 5/-1/-1->4->6 [11] 6/-1/-1->4->5 [12] 7/8/0->4->12 [13] 6/-1/-1->4->7 [14] 7/-1/-1->4->6 [15] -1/-1/-1->4->5
+2: jzxh161:3283539:3284338 [3] NCCL INFO comm 0x557efcb6a4d0 rank 11 nRanks 16 nNodes 4 localRanks 4 localRank 3 MNNVL 0
+2: jzxh161:3283538:3284335 [2] NCCL INFO comm 0x5643a3152d00 rank 10 nRanks 16 nNodes 4 localRanks 4 localRank 2 MNNVL 0
+2: jzxh161:3283539:3284338 [3] NCCL INFO Trees [0] -1/-1/-1->11->10 [1] 8/-1/-1->11->10 [2] -1/-1/-1->11->9 [3] 9/7/15->11->3 [4] 10/-1/-1->11->8 [5] 8/-1/-1->11->9 [6] 9/-1/-1->11->8 [7] 10/7/15->11->3 [8] -1/-1/-1->11->10 [9] 8/-1/-1->11->10 [10] -1/-1/-1->11->9 [11] 9/-1/-1->11->7 [12] 10/-1/-1->11->8 [13] 8/-1/-1->11->9 [14] 9/-1/-1->11->8 [15] 10/-1/-1->11->7
+2: jzxh161:3283539:3284338 [3] NCCL INFO P2P Chunksize set to 131072
+3: jzxh162:3160742:3161550 [0] NCCL INFO Trees [0] 13/-1/-1->12->8 [1] -1/-1/-1->12->15 [2] 13/-1/-1->12->14 [3] 14/-1/-1->12->13 [4] 15/-1/-1->12->8 [5] 14/-1/-1->12->15 [6] 15/-1/-1->12->14 [7] -1/-1/-1->12->13 [8] 13/4/-1->12->-1 [9] -1/-1/-1->12->15 [10] 13/-1/-1->12->14 [11] 14/-1/-1->12->13 [12] 15/4/-1->12->-1 [13] 14/-1/-1->12->15 [14] 15/-1/-1->12->14 [15] -1/-1/-1->12->13
+3: jzxh162:3160745:3161552 [3] NCCL INFO Trees [0] -1/-1/-1->15->14 [1] 12/-1/-1->15->14 [2] -1/-1/-1->15->13 [3] 13/-1/-1->15->11 [4] 14/-1/-1->15->12 [5] 12/-1/-1->15->13 [6] 13/-1/-1->15->12 [7] 14/-1/-1->15->11 [8] -1/-1/-1->15->14 [9] 12/-1/-1->15->14 [10] -1/-1/-1->15->13 [11] 13/7/-1->15->-1 [12] 14/-1/-1->15->12 [13] 12/-1/-1->15->13 [14] 13/-1/-1->15->12 [15] 14/7/-1->15->-1
+3: jzxh162:3160742:3161550 [0] NCCL INFO P2P Chunksize set to 131072
+3: jzxh162:3160745:3161552 [3] NCCL INFO P2P Chunksize set to 131072
+1: jzxh160:364771:365554 [0] NCCL INFO P2P Chunksize set to 131072
+2: jzxh161:3283538:3284335 [2] NCCL INFO Trees [0] 11/-1/-1->10->9 [1] 11/-1/-1->10->9 [2] 8/6/14->10->2 [3] -1/-1/-1->10->8 [4] 9/-1/-1->10->11 [5] -1/-1/-1->10->8 [6] 8/6/14->10->2 [7] 9/-1/-1->10->11 [8] 11/-1/-1->10->9 [9] 11/-1/-1->10->9 [10] 8/-1/-1->10->6 [11] -1/-1/-1->10->8 [12] 9/-1/-1->10->11 [13] -1/-1/-1->10->8 [14] 8/-1/-1->10->6 [15] 9/-1/-1->10->11
+2: jzxh161:3283538:3284335 [2] NCCL INFO P2P Chunksize set to 131072
+2: jzxh161:3283537:3284337 [1] NCCL INFO comm 0x55f84f114b10 rank 9 nRanks 16 nNodes 4 localRanks 4 localRank 1 MNNVL 0
+2: jzxh161:3283536:3284336 [0] NCCL INFO comm 0x55c8d4c4af80 rank 8 nRanks 16 nNodes 4 localRanks 4 localRank 0 MNNVL 0
+2: jzxh161:3283537:3284337 [1] NCCL INFO Trees [0] 10/-1/-1->9->8 [1] 10/5/13->9->1 [2] 11/-1/-1->9->8 [3] 8/-1/-1->9->11 [4] -1/-1/-1->9->10 [5] 11/5/13->9->1 [6] -1/-1/-1->9->11 [7] 8/-1/-1->9->10 [8] 10/-1/-1->9->8 [9] 10/-1/-1->9->5 [10] 11/-1/-1->9->8 [11] 8/-1/-1->9->11 [12] -1/-1/-1->9->10 [13] 11/-1/-1->9->5 [14] -1/-1/-1->9->11 [15] 8/-1/-1->9->10
+2: jzxh161:3283537:3284337 [1] NCCL INFO P2P Chunksize set to 131072
+2: jzxh161:3283536:3284336 [0] NCCL INFO Trees [0] 9/4/12->8->0 [1] -1/-1/-1->8->11 [2] 9/-1/-1->8->10 [3] 10/-1/-1->8->9 [4] 11/4/12->8->0 [5] 10/-1/-1->8->11 [6] 11/-1/-1->8->10 [7] -1/-1/-1->8->9 [8] 9/-1/-1->8->4 [9] -1/-1/-1->8->11 [10] 9/-1/-1->8->10 [11] 10/-1/-1->8->9 [12] 11/-1/-1->8->4 [13] 10/-1/-1->8->11 [14] 11/-1/-1->8->10 [15] -1/-1/-1->8->9
+2: jzxh161:3283536:3284336 [0] NCCL INFO P2P Chunksize set to 131072
+3: jzxh162:3160743:3161551 [1] NCCL INFO Channel 00/0 : 13[1] -> 14[2] via P2P/CUMEM
+2: jzxh161:3283538:3284335 [2] NCCL INFO Channel 00/0 : 10[2] -> 11[3] via P2P/CUMEM
+0: jzxh159:1308401:1310257 [2] NCCL INFO Channel 00/0 : 2[2] -> 3[3] via P2P/CUMEM
+1: jzxh160:364772:365552 [1] NCCL INFO Channel 00/0 : 5[1] -> 6[2] via P2P/CUMEM
+2: jzxh161:3283538:3284335 [2] NCCL INFO Channel 04/0 : 10[2] -> 11[3] via P2P/CUMEM
+3: jzxh162:3160744:3161553 [2] NCCL INFO Channel 00/0 : 14[2] -> 15[3] via P2P/CUMEM
+0: jzxh159:1308400:1310256 [1] NCCL INFO Channel 00/0 : 1[1] -> 2[2] via P2P/CUMEM
+3: jzxh162:3160743:3161551 [1] NCCL INFO Channel 03/0 : 13[1] -> 14[2] via P2P/CUMEM
+0: jzxh159:1308400:1310256 [1] NCCL INFO Channel 03/0 : 1[1] -> 2[2] via P2P/CUMEM
+2: jzxh161:3283538:3284335 [2] NCCL INFO Channel 08/0 : 10[2] -> 11[3] via P2P/CUMEM
+3: jzxh162:3160744:3161553 [2] NCCL INFO Channel 04/0 : 14[2] -> 15[3] via P2P/CUMEM
+1: jzxh160:364773:365553 [2] NCCL INFO Channel 00/0 : 6[2] -> 7[3] via P2P/CUMEM
+3: jzxh162:3160743:3161551 [1] NCCL INFO Channel 04/0 : 13[1] -> 14[2] via P2P/CUMEM
+1: jzxh160:364772:365552 [1] NCCL INFO Channel 03/0 : 5[1] -> 6[2] via P2P/CUMEM
+0: jzxh159:1308401:1310257 [2] NCCL INFO Channel 04/0 : 2[2] -> 3[3] via P2P/CUMEM
+3: jzxh162:3160744:3161553 [2] NCCL INFO Channel 08/0 : 14[2] -> 15[3] via P2P/CUMEM
+1: jzxh160:364772:365552 [1] NCCL INFO Channel 04/0 : 5[1] -> 6[2] via P2P/CUMEM
+2: jzxh161:3283537:3284337 [1] NCCL INFO Channel 00/0 : 9[1] -> 10[2] via P2P/CUMEM
+2: jzxh161:3283538:3284335 [2] NCCL INFO Channel 12/0 : 10[2] -> 11[3] via P2P/CUMEM
+3: jzxh162:3160743:3161551 [1] NCCL INFO Channel 07/0 : 13[1] -> 14[2] via P2P/CUMEM
+3: jzxh162:3160744:3161553 [2] NCCL INFO Channel 12/0 : 14[2] -> 15[3] via P2P/CUMEM
+0: jzxh159:1308401:1310257 [2] NCCL INFO Channel 08/0 : 2[2] -> 3[3] via P2P/CUMEM
+1: jzxh160:364773:365553 [2] NCCL INFO Channel 04/0 : 6[2] -> 7[3] via P2P/CUMEM
+2: jzxh161:3283537:3284337 [1] NCCL INFO Channel 03/0 : 9[1] -> 10[2] via P2P/CUMEM
+3: jzxh162:3160743:3161551 [1] NCCL INFO Channel 08/0 : 13[1] -> 14[2] via P2P/CUMEM
+1: jzxh160:364772:365552 [1] NCCL INFO Channel 07/0 : 5[1] -> 6[2] via P2P/CUMEM
+3: jzxh162:3160743:3161551 [1] NCCL INFO Channel 11/0 : 13[1] -> 14[2] via P2P/CUMEM
+1: jzxh160:364773:365553 [2] NCCL INFO Channel 08/0 : 6[2] -> 7[3] via P2P/CUMEM
+2: jzxh161:3283537:3284337 [1] NCCL INFO Channel 04/0 : 9[1] -> 10[2] via P2P/CUMEM
+0: jzxh159:1308400:1310256 [1] NCCL INFO Channel 04/0 : 1[1] -> 2[2] via P2P/CUMEM
+0: jzxh159:1308401:1310257 [2] NCCL INFO Channel 12/0 : 2[2] -> 3[3] via P2P/CUMEM
+1: jzxh160:364772:365552 [1] NCCL INFO Channel 08/0 : 5[1] -> 6[2] via P2P/CUMEM
+2: jzxh161:3283537:3284337 [1] NCCL INFO Channel 07/0 : 9[1] -> 10[2] via P2P/CUMEM
+3: jzxh162:3160743:3161551 [1] NCCL INFO Channel 12/0 : 13[1] -> 14[2] via P2P/CUMEM
+0: jzxh159:1308400:1310256 [1] NCCL INFO Channel 07/0 : 1[1] -> 2[2] via P2P/CUMEM
+1: jzxh160:364773:365553 [2] NCCL INFO Channel 12/0 : 6[2] -> 7[3] via P2P/CUMEM
+2: jzxh161:3283537:3284337 [1] NCCL INFO Channel 08/0 : 9[1] -> 10[2] via P2P/CUMEM
+3: jzxh162:3160743:3161551 [1] NCCL INFO Channel 15/0 : 13[1] -> 14[2] via P2P/CUMEM
+0: jzxh159:1308400:1310256 [1] NCCL INFO Channel 08/0 : 1[1] -> 2[2] via P2P/CUMEM
+1: jzxh160:364772:365552 [1] NCCL INFO Channel 11/0 : 5[1] -> 6[2] via P2P/CUMEM
+2: jzxh161:3283537:3284337 [1] NCCL INFO Channel 11/0 : 9[1] -> 10[2] via P2P/CUMEM
+0: jzxh159:1308400:1310256 [1] NCCL INFO Channel 11/0 : 1[1] -> 2[2] via P2P/CUMEM
+1: jzxh160:364772:365552 [1] NCCL INFO Channel 12/0 : 5[1] -> 6[2] via P2P/CUMEM
+2: jzxh161:3283537:3284337 [1] NCCL INFO Channel 12/0 : 9[1] -> 10[2] via P2P/CUMEM
+0: jzxh159:1308400:1310256 [1] NCCL INFO Channel 12/0 : 1[1] -> 2[2] via P2P/CUMEM
+1: jzxh160:364772:365552 [1] NCCL INFO Channel 15/0 : 5[1] -> 6[2] via P2P/CUMEM
+2: jzxh161:3283537:3284337 [1] NCCL INFO Channel 15/0 : 9[1] -> 10[2] via P2P/CUMEM
+0: jzxh159:1308400:1310256 [1] NCCL INFO Channel 15/0 : 1[1] -> 2[2] via P2P/CUMEM
+1: jzxh160:364771:365554 [0] NCCL INFO Channel 00/0 : 3[3] -> 4[0] [receive] via NET/IB/0/GDRDMA
+1: jzxh160:364774:365551 [3] NCCL INFO Channel 00/0 : 7[3] -> 8[0] [send] via NET/IB/0(4)/GDRDMA
+1: jzxh160:364771:365554 [0] NCCL INFO Channel 04/0 : 3[3] -> 4[0] [receive] via NET/IB/0/GDRDMA
+1: jzxh160:364774:365551 [3] NCCL INFO Channel 04/0 : 7[3] -> 8[0] [send] via NET/IB/0(4)/GDRDMA
+1: jzxh160:364771:365554 [0] NCCL INFO Channel 08/0 : 3[3] -> 4[0] [receive] via NET/IB/0/GDRDMA
+0: jzxh159:1308399:1310255 [0] NCCL INFO Channel 00/0 : 15[3] -> 0[0] [receive] via NET/IB/0/GDRDMA
+0: jzxh159:1308402:1310258 [3] NCCL INFO Channel 00/0 : 3[3] -> 4[0] [send] via NET/IB/0(0)/GDRDMA
+0: jzxh159:1308399:1310255 [0] NCCL INFO Channel 04/0 : 15[3] -> 0[0] [receive] via NET/IB/0/GDRDMA
+1: jzxh160:364774:365551 [3] NCCL INFO Channel 08/0 : 7[3] -> 8[0] [send] via NET/IB/0(4)/GDRDMA
+1: jzxh160:364771:365554 [0] NCCL INFO Channel 12/0 : 3[3] -> 4[0] [receive] via NET/IB/0/GDRDMA
+1: jzxh160:364774:365551 [3] NCCL INFO Channel 12/0 : 7[3] -> 8[0] [send] via NET/IB/0(4)/GDRDMA
+1: jzxh160:364771:365554 [0] NCCL INFO Channel 00/0 : 4[0] -> 5[1] via P2P/CUMEM
+2: jzxh161:3283536:3284336 [0] NCCL INFO Channel 00/0 : 7[3] -> 8[0] [receive] via NET/IB/0/GDRDMA
+2: jzxh161:3283539:3284338 [3] NCCL INFO Channel 00/0 : 11[3] -> 12[0] [send] via NET/IB/0(8)/GDRDMA
+2: jzxh161:3283536:3284336 [0] NCCL INFO Channel 04/0 : 7[3] -> 8[0] [receive] via NET/IB/0/GDRDMA
+0: jzxh159:1308402:1310258 [3] NCCL INFO Channel 04/0 : 3[3] -> 4[0] [send] via NET/IB/0(0)/GDRDMA
+0: jzxh159:1308399:1310255 [0] NCCL INFO Channel 08/0 : 15[3] -> 0[0] [receive] via NET/IB/0/GDRDMA
+0: jzxh159:1308402:1310258 [3] NCCL INFO Channel 08/0 : 3[3] -> 4[0] [send] via NET/IB/0(0)/GDRDMA
+2: jzxh161:3283539:3284338 [3] NCCL INFO Channel 04/0 : 11[3] -> 12[0] [send] via NET/IB/0(8)/GDRDMA
+0: jzxh159:1308399:1310255 [0] NCCL INFO Channel 12/0 : 15[3] -> 0[0] [receive] via NET/IB/0/GDRDMA
+1: jzxh160:364771:365554 [0] NCCL INFO Channel 03/0 : 4[0] -> 5[1] via P2P/CUMEM
+0: jzxh159:1308399:1310255 [0] NCCL INFO Channel 00/0 : 0[0] -> 1[1] via P2P/CUMEM
+0: jzxh159:1308402:1310258 [3] NCCL INFO Channel 12/0 : 3[3] -> 4[0] [send] via NET/IB/0(0)/GDRDMA
+1: jzxh160:364771:365554 [0] NCCL INFO Channel 04/0 : 4[0] -> 5[1] via P2P/CUMEM
+2: jzxh161:3283536:3284336 [0] NCCL INFO Channel 08/0 : 7[3] -> 8[0] [receive] via NET/IB/0/GDRDMA
+2: jzxh161:3283539:3284338 [3] NCCL INFO Channel 08/0 : 11[3] -> 12[0] [send] via NET/IB/0(8)/GDRDMA
+2: jzxh161:3283536:3284336 [0] NCCL INFO Channel 12/0 : 7[3] -> 8[0] [receive] via NET/IB/0/GDRDMA
+2: jzxh161:3283539:3284338 [3] NCCL INFO Channel 12/0 : 11[3] -> 12[0] [send] via NET/IB/0(8)/GDRDMA
+2: jzxh161:3283536:3284336 [0] NCCL INFO Channel 00/0 : 8[0] -> 9[1] via P2P/CUMEM
+1: jzxh160:364771:365554 [0] NCCL INFO Channel 07/0 : 4[0] -> 5[1] via P2P/CUMEM
+0: jzxh159:1308399:1310255 [0] NCCL INFO Channel 03/0 : 0[0] -> 1[1] via P2P/CUMEM
+1: jzxh160:364771:365554 [0] NCCL INFO Channel 08/0 : 4[0] -> 5[1] via P2P/CUMEM
+0: jzxh159:1308399:1310255 [0] NCCL INFO Channel 04/0 : 0[0] -> 1[1] via P2P/CUMEM
+3: jzxh162:3160742:3161550 [0] NCCL INFO Channel 00/0 : 11[3] -> 12[0] [receive] via NET/IB/0/GDRDMA
+2: jzxh161:3283536:3284336 [0] NCCL INFO Channel 03/0 : 8[0] -> 9[1] via P2P/CUMEM
+1: jzxh160:364771:365554 [0] NCCL INFO Channel 11/0 : 4[0] -> 5[1] via P2P/CUMEM
+3: jzxh162:3160745:3161552 [3] NCCL INFO Channel 00/0 : 15[3] -> 0[0] [send] via NET/IB/0(12)/GDRDMA
+0: jzxh159:1308399:1310255 [0] NCCL INFO Channel 07/0 : 0[0] -> 1[1] via P2P/CUMEM
+3: jzxh162:3160742:3161550 [0] NCCL INFO Channel 04/0 : 11[3] -> 12[0] [receive] via NET/IB/0/GDRDMA
+3: jzxh162:3160745:3161552 [3] NCCL INFO Channel 04/0 : 15[3] -> 0[0] [send] via NET/IB/0(12)/GDRDMA
+3: jzxh162:3160742:3161550 [0] NCCL INFO Channel 08/0 : 11[3] -> 12[0] [receive] via NET/IB/0/GDRDMA
+1: jzxh160:364771:365554 [0] NCCL INFO Channel 12/0 : 4[0] -> 5[1] via P2P/CUMEM
+2: jzxh161:3283536:3284336 [0] NCCL INFO Channel 04/0 : 8[0] -> 9[1] via P2P/CUMEM
+3: jzxh162:3160745:3161552 [3] NCCL INFO Channel 08/0 : 15[3] -> 0[0] [send] via NET/IB/0(12)/GDRDMA
+0: jzxh159:1308399:1310255 [0] NCCL INFO Channel 08/0 : 0[0] -> 1[1] via P2P/CUMEM
+3: jzxh162:3160742:3161550 [0] NCCL INFO Channel 12/0 : 11[3] -> 12[0] [receive] via NET/IB/0/GDRDMA
+3: jzxh162:3160745:3161552 [3] NCCL INFO Channel 12/0 : 15[3] -> 0[0] [send] via NET/IB/0(12)/GDRDMA
+3: jzxh162:3160742:3161550 [0] NCCL INFO Channel 00/0 : 12[0] -> 13[1] via P2P/CUMEM
+1: jzxh160:364771:365554 [0] NCCL INFO Channel 15/0 : 4[0] -> 5[1] via P2P/CUMEM
+0: jzxh159:1308399:1310255 [0] NCCL INFO Channel 11/0 : 0[0] -> 1[1] via P2P/CUMEM
+2: jzxh161:3283536:3284336 [0] NCCL INFO Channel 07/0 : 8[0] -> 9[1] via P2P/CUMEM
+0: jzxh159:1308399:1310255 [0] NCCL INFO Channel 12/0 : 0[0] -> 1[1] via P2P/CUMEM
+2: jzxh161:3283536:3284336 [0] NCCL INFO Channel 08/0 : 8[0] -> 9[1] via P2P/CUMEM
+2: jzxh161:3283536:3284336 [0] NCCL INFO Channel 11/0 : 8[0] -> 9[1] via P2P/CUMEM
+1: jzxh160:364771:365554 [0] NCCL INFO Channel 01/0 : 4[0] -> 7[3] via P2P/CUMEM
+3: jzxh162:3160742:3161550 [0] NCCL INFO Channel 03/0 : 12[0] -> 13[1] via P2P/CUMEM
+0: jzxh159:1308399:1310255 [0] NCCL INFO Channel 15/0 : 0[0] -> 1[1] via P2P/CUMEM
+2: jzxh161:3283536:3284336 [0] NCCL INFO Channel 12/0 : 8[0] -> 9[1] via P2P/CUMEM
+1: jzxh160:364771:365554 [0] NCCL INFO Channel 02/0 : 4[0] -> 7[3] via P2P/CUMEM
+3: jzxh162:3160742:3161550 [0] NCCL INFO Channel 04/0 : 12[0] -> 13[1] via P2P/CUMEM
+1: jzxh160:364771:365554 [0] NCCL INFO Channel 05/0 : 4[0] -> 7[3] via P2P/CUMEM
+2: jzxh161:3283536:3284336 [0] NCCL INFO Channel 15/0 : 8[0] -> 9[1] via P2P/CUMEM
+3: jzxh162:3160742:3161550 [0] NCCL INFO Channel 07/0 : 12[0] -> 13[1] via P2P/CUMEM
+1: jzxh160:364771:365554 [0] NCCL INFO Channel 06/0 : 4[0] -> 7[3] via P2P/CUMEM
+0: jzxh159:1308399:1310255 [0] NCCL INFO Channel 01/0 : 0[0] -> 3[3] via P2P/CUMEM
+3: jzxh162:3160742:3161550 [0] NCCL INFO Channel 08/0 : 12[0] -> 13[1] via P2P/CUMEM
+1: jzxh160:364771:365554 [0] NCCL INFO Channel 09/0 : 4[0] -> 7[3] via P2P/CUMEM
+0: jzxh159:1308399:1310255 [0] NCCL INFO Channel 02/0 : 0[0] -> 3[3] via P2P/CUMEM
+3: jzxh162:3160742:3161550 [0] NCCL INFO Channel 11/0 : 12[0] -> 13[1] via P2P/CUMEM
+1: jzxh160:364771:365554 [0] NCCL INFO Channel 10/0 : 4[0] -> 7[3] via P2P/CUMEM
+2: jzxh161:3283536:3284336 [0] NCCL INFO Channel 01/0 : 8[0] -> 11[3] via P2P/CUMEM
+0: jzxh159:1308399:1310255 [0] NCCL INFO Channel 05/0 : 0[0] -> 3[3] via P2P/CUMEM
+1: jzxh160:364771:365554 [0] NCCL INFO Channel 13/0 : 4[0] -> 7[3] via P2P/CUMEM
+3: jzxh162:3160742:3161550 [0] NCCL INFO Channel 12/0 : 12[0] -> 13[1] via P2P/CUMEM
+0: jzxh159:1308399:1310255 [0] NCCL INFO Channel 06/0 : 0[0] -> 3[3] via P2P/CUMEM
+2: jzxh161:3283536:3284336 [0] NCCL INFO Channel 02/0 : 8[0] -> 11[3] via P2P/CUMEM
+1: jzxh160:364771:365554 [0] NCCL INFO Channel 14/0 : 4[0] -> 7[3] via P2P/CUMEM
+0: jzxh159:1308399:1310255 [0] NCCL INFO Channel 09/0 : 0[0] -> 3[3] via P2P/CUMEM
+3: jzxh162:3160742:3161550 [0] NCCL INFO Channel 15/0 : 12[0] -> 13[1] via P2P/CUMEM
+2: jzxh161:3283536:3284336 [0] NCCL INFO Channel 05/0 : 8[0] -> 11[3] via P2P/CUMEM
+2: jzxh161:3283536:3284336 [0] NCCL INFO Channel 06/0 : 8[0] -> 11[3] via P2P/CUMEM
+0: jzxh159:1308399:1310255 [0] NCCL INFO Channel 10/0 : 0[0] -> 3[3] via P2P/CUMEM
+0: jzxh159:1308399:1310255 [0] NCCL INFO Channel 13/0 : 0[0] -> 3[3] via P2P/CUMEM
+2: jzxh161:3283536:3284336 [0] NCCL INFO Channel 09/0 : 8[0] -> 11[3] via P2P/CUMEM
+0: jzxh159:1308399:1310255 [0] NCCL INFO Channel 14/0 : 0[0] -> 3[3] via P2P/CUMEM
+2: jzxh161:3283536:3284336 [0] NCCL INFO Channel 10/0 : 8[0] -> 11[3] via P2P/CUMEM
+2: jzxh161:3283536:3284336 [0] NCCL INFO Channel 13/0 : 8[0] -> 11[3] via P2P/CUMEM
+3: jzxh162:3160742:3161550 [0] NCCL INFO Channel 01/0 : 12[0] -> 15[3] via P2P/CUMEM
+2: jzxh161:3283536:3284336 [0] NCCL INFO Channel 14/0 : 8[0] -> 11[3] via P2P/CUMEM
+3: jzxh162:3160742:3161550 [0] NCCL INFO Channel 02/0 : 12[0] -> 15[3] via P2P/CUMEM
+3: jzxh162:3160742:3161550 [0] NCCL INFO Channel 05/0 : 12[0] -> 15[3] via P2P/CUMEM
+3: jzxh162:3160742:3161550 [0] NCCL INFO Channel 06/0 : 12[0] -> 15[3] via P2P/CUMEM
+3: jzxh162:3160742:3161550 [0] NCCL INFO Channel 09/0 : 12[0] -> 15[3] via P2P/CUMEM
+3: jzxh162:3160742:3161550 [0] NCCL INFO Channel 10/0 : 12[0] -> 15[3] via P2P/CUMEM
+3: jzxh162:3160742:3161550 [0] NCCL INFO Channel 13/0 : 12[0] -> 15[3] via P2P/CUMEM
+3: jzxh162:3160742:3161550 [0] NCCL INFO Channel 14/0 : 12[0] -> 15[3] via P2P/CUMEM
+1: jzxh160:364773:365553 [2] NCCL INFO Channel 02/0 : 3[3] -> 6[2] [receive] via NET/IB/2/GDRDMA
+1: jzxh160:364774:365551 [3] NCCL INFO Channel 02/0 : 7[3] -> 10[2] [send] via NET/IB/2(6)/GDRDMA
+1: jzxh160:364773:365553 [2] NCCL INFO Channel 06/0 : 3[3] -> 6[2] [receive] via NET/IB/2/GDRDMA
+1: jzxh160:364774:365551 [3] NCCL INFO Channel 06/0 : 7[3] -> 10[2] [send] via NET/IB/2(6)/GDRDMA
+1: jzxh160:364773:365553 [2] NCCL INFO Channel 10/0 : 3[3] -> 6[2] [receive] via NET/IB/2/GDRDMA
+1: jzxh160:364774:365551 [3] NCCL INFO Channel 10/0 : 7[3] -> 10[2] [send] via NET/IB/2(6)/GDRDMA
+1: jzxh160:364773:365553 [2] NCCL INFO Channel 14/0 : 3[3] -> 6[2] [receive] via NET/IB/2/GDRDMA
+0: jzxh159:1308401:1310257 [2] NCCL INFO Channel 02/0 : 15[3] -> 2[2] [receive] via NET/IB/2/GDRDMA
+0: jzxh159:1308402:1310258 [3] NCCL INFO Channel 02/0 : 3[3] -> 6[2] [send] via NET/IB/2(2)/GDRDMA
+0: jzxh159:1308401:1310257 [2] NCCL INFO Channel 06/0 : 15[3] -> 2[2] [receive] via NET/IB/2/GDRDMA
+0: jzxh159:1308402:1310258 [3] NCCL INFO Channel 06/0 : 3[3] -> 6[2] [send] via NET/IB/2(2)/GDRDMA
+0: jzxh159:1308401:1310257 [2] NCCL INFO Channel 10/0 : 15[3] -> 2[2] [receive] via NET/IB/2/GDRDMA
+0: jzxh159:1308402:1310258 [3] NCCL INFO Channel 10/0 : 3[3] -> 6[2] [send] via NET/IB/2(2)/GDRDMA
+0: jzxh159:1308401:1310257 [2] NCCL INFO Channel 14/0 : 15[3] -> 2[2] [receive] via NET/IB/2/GDRDMA
+1: jzxh160:364774:365551 [3] NCCL INFO Channel 14/0 : 7[3] -> 10[2] [send] via NET/IB/2(6)/GDRDMA
+0: jzxh159:1308402:1310258 [3] NCCL INFO Channel 14/0 : 3[3] -> 6[2] [send] via NET/IB/2(2)/GDRDMA
+2: jzxh161:3283538:3284335 [2] NCCL INFO Channel 02/0 : 7[3] -> 10[2] [receive] via NET/IB/2/GDRDMA
+2: jzxh161:3283539:3284338 [3] NCCL INFO Channel 02/0 : 11[3] -> 14[2] [send] via NET/IB/2(10)/GDRDMA
+2: jzxh161:3283538:3284335 [2] NCCL INFO Channel 06/0 : 7[3] -> 10[2] [receive] via NET/IB/2/GDRDMA
+2: jzxh161:3283539:3284338 [3] NCCL INFO Channel 06/0 : 11[3] -> 14[2] [send] via NET/IB/2(10)/GDRDMA
+2: jzxh161:3283538:3284335 [2] NCCL INFO Channel 10/0 : 7[3] -> 10[2] [receive] via NET/IB/2/GDRDMA
+2: jzxh161:3283539:3284338 [3] NCCL INFO Channel 10/0 : 11[3] -> 14[2] [send] via NET/IB/2(10)/GDRDMA
+2: jzxh161:3283538:3284335 [2] NCCL INFO Channel 14/0 : 7[3] -> 10[2] [receive] via NET/IB/2/GDRDMA
+2: jzxh161:3283539:3284338 [3] NCCL INFO Channel 14/0 : 11[3] -> 14[2] [send] via NET/IB/2(10)/GDRDMA
+3: jzxh162:3160744:3161553 [2] NCCL INFO Channel 02/0 : 11[3] -> 14[2] [receive] via NET/IB/2/GDRDMA
+3: jzxh162:3160745:3161552 [3] NCCL INFO Channel 02/0 : 15[3] -> 2[2] [send] via NET/IB/2(14)/GDRDMA
+3: jzxh162:3160744:3161553 [2] NCCL INFO Channel 06/0 : 11[3] -> 14[2] [receive] via NET/IB/2/GDRDMA
+3: jzxh162:3160745:3161552 [3] NCCL INFO Channel 06/0 : 15[3] -> 2[2] [send] via NET/IB/2(14)/GDRDMA
+3: jzxh162:3160744:3161553 [2] NCCL INFO Channel 10/0 : 11[3] -> 14[2] [receive] via NET/IB/2/GDRDMA
+3: jzxh162:3160745:3161552 [3] NCCL INFO Channel 10/0 : 15[3] -> 2[2] [send] via NET/IB/2(14)/GDRDMA
+3: jzxh162:3160744:3161553 [2] NCCL INFO Channel 14/0 : 11[3] -> 14[2] [receive] via NET/IB/2/GDRDMA
+3: jzxh162:3160745:3161552 [3] NCCL INFO Channel 14/0 : 15[3] -> 2[2] [send] via NET/IB/2(14)/GDRDMA
+1: jzxh160:364772:365552 [1] NCCL INFO Channel 01/0 : 2[2] -> 5[1] [receive] via NET/IB/1/GDRDMA
+1: jzxh160:364773:365553 [2] NCCL INFO Channel 01/0 : 6[2] -> 9[1] [send] via NET/IB/1(5)/GDRDMA
+1: jzxh160:364772:365552 [1] NCCL INFO Channel 05/0 : 2[2] -> 5[1] [receive] via NET/IB/1/GDRDMA
+1: jzxh160:364773:365553 [2] NCCL INFO Channel 05/0 : 6[2] -> 9[1] [send] via NET/IB/1(5)/GDRDMA
+1: jzxh160:364772:365552 [1] NCCL INFO Channel 09/0 : 2[2] -> 5[1] [receive] via NET/IB/1/GDRDMA
+1: jzxh160:364773:365553 [2] NCCL INFO Channel 09/0 : 6[2] -> 9[1] [send] via NET/IB/1(5)/GDRDMA
+1: jzxh160:364772:365552 [1] NCCL INFO Channel 13/0 : 2[2] -> 5[1] [receive] via NET/IB/1/GDRDMA
+1: jzxh160:364773:365553 [2] NCCL INFO Channel 13/0 : 6[2] -> 9[1] [send] via NET/IB/1(5)/GDRDMA
+2: jzxh161:3283537:3284337 [1] NCCL INFO Channel 01/0 : 6[2] -> 9[1] [receive] via NET/IB/1/GDRDMA
+0: jzxh159:1308400:1310256 [1] NCCL INFO Channel 01/0 : 14[2] -> 1[1] [receive] via NET/IB/1/GDRDMA
+2: jzxh161:3283538:3284335 [2] NCCL INFO Channel 01/0 : 10[2] -> 13[1] [send] via NET/IB/1(9)/GDRDMA
+0: jzxh159:1308401:1310257 [2] NCCL INFO Channel 01/0 : 2[2] -> 5[1] [send] via NET/IB/1(1)/GDRDMA
+2: jzxh161:3283537:3284337 [1] NCCL INFO Channel 05/0 : 6[2] -> 9[1] [receive] via NET/IB/1/GDRDMA
+2: jzxh161:3283538:3284335 [2] NCCL INFO Channel 05/0 : 10[2] -> 13[1] [send] via NET/IB/1(9)/GDRDMA
+2: jzxh161:3283537:3284337 [1] NCCL INFO Channel 09/0 : 6[2] -> 9[1] [receive] via NET/IB/1/GDRDMA
+0: jzxh159:1308400:1310256 [1] NCCL INFO Channel 05/0 : 14[2] -> 1[1] [receive] via NET/IB/1/GDRDMA
+2: jzxh161:3283538:3284335 [2] NCCL INFO Channel 09/0 : 10[2] -> 13[1] [send] via NET/IB/1(9)/GDRDMA
+0: jzxh159:1308401:1310257 [2] NCCL INFO Channel 05/0 : 2[2] -> 5[1] [send] via NET/IB/1(1)/GDRDMA
+0: jzxh159:1308400:1310256 [1] NCCL INFO Channel 09/0 : 14[2] -> 1[1] [receive] via NET/IB/1/GDRDMA
+0: jzxh159:1308401:1310257 [2] NCCL INFO Channel 09/0 : 2[2] -> 5[1] [send] via NET/IB/1(1)/GDRDMA
+0: jzxh159:1308400:1310256 [1] NCCL INFO Channel 13/0 : 14[2] -> 1[1] [receive] via NET/IB/1/GDRDMA
+0: jzxh159:1308401:1310257 [2] NCCL INFO Channel 13/0 : 2[2] -> 5[1] [send] via NET/IB/1(1)/GDRDMA
+2: jzxh161:3283537:3284337 [1] NCCL INFO Channel 13/0 : 6[2] -> 9[1] [receive] via NET/IB/1/GDRDMA
+2: jzxh161:3283538:3284335 [2] NCCL INFO Channel 13/0 : 10[2] -> 13[1] [send] via NET/IB/1(9)/GDRDMA
+3: jzxh162:3160743:3161551 [1] NCCL INFO Channel 01/0 : 10[2] -> 13[1] [receive] via NET/IB/1/GDRDMA
+2: jzxh161:3283537:3284337 [1] NCCL INFO Channel 01/0 : 9[1] -> 8[0] via P2P/CUMEM
+3: jzxh162:3160744:3161553 [2] NCCL INFO Channel 01/0 : 14[2] -> 1[1] [send] via NET/IB/1(13)/GDRDMA
+3: jzxh162:3160743:3161551 [1] NCCL INFO Channel 05/0 : 10[2] -> 13[1] [receive] via NET/IB/1/GDRDMA
+3: jzxh162:3160744:3161553 [2] NCCL INFO Channel 05/0 : 14[2] -> 1[1] [send] via NET/IB/1(13)/GDRDMA
+3: jzxh162:3160743:3161551 [1] NCCL INFO Channel 09/0 : 10[2] -> 13[1] [receive] via NET/IB/1/GDRDMA
+3: jzxh162:3160744:3161553 [2] NCCL INFO Channel 09/0 : 14[2] -> 1[1] [send] via NET/IB/1(13)/GDRDMA
+2: jzxh161:3283537:3284337 [1] NCCL INFO Channel 02/0 : 9[1] -> 8[0] via P2P/CUMEM
+3: jzxh162:3160743:3161551 [1] NCCL INFO Channel 13/0 : 10[2] -> 13[1] [receive] via NET/IB/1/GDRDMA
+3: jzxh162:3160744:3161553 [2] NCCL INFO Channel 13/0 : 14[2] -> 1[1] [send] via NET/IB/1(13)/GDRDMA
+2: jzxh161:3283537:3284337 [1] NCCL INFO Channel 05/0 : 9[1] -> 8[0] via P2P/CUMEM
+2: jzxh161:3283537:3284337 [1] NCCL INFO Channel 06/0 : 9[1] -> 8[0] via P2P/CUMEM
+1: jzxh160:364772:365552 [1] NCCL INFO Channel 01/0 : 5[1] -> 4[0] via P2P/CUMEM
+1: jzxh160:364772:365552 [1] NCCL INFO Channel 02/0 : 5[1] -> 4[0] via P2P/CUMEM
+2: jzxh161:3283537:3284337 [1] NCCL INFO Channel 09/0 : 9[1] -> 8[0] via P2P/CUMEM
+2: jzxh161:3283537:3284337 [1] NCCL INFO Channel 10/0 : 9[1] -> 8[0] via P2P/CUMEM
+1: jzxh160:364772:365552 [1] NCCL INFO Channel 05/0 : 5[1] -> 4[0] via P2P/CUMEM
+2: jzxh161:3283537:3284337 [1] NCCL INFO Channel 13/0 : 9[1] -> 8[0] via P2P/CUMEM
+3: jzxh162:3160743:3161551 [1] NCCL INFO Channel 01/0 : 13[1] -> 12[0] via P2P/CUMEM
+1: jzxh160:364772:365552 [1] NCCL INFO Channel 06/0 : 5[1] -> 4[0] via P2P/CUMEM
+0: jzxh159:1308400:1310256 [1] NCCL INFO Channel 01/0 : 1[1] -> 0[0] via P2P/CUMEM
+3: jzxh162:3160743:3161551 [1] NCCL INFO Channel 02/0 : 13[1] -> 12[0] via P2P/CUMEM
+2: jzxh161:3283537:3284337 [1] NCCL INFO Channel 14/0 : 9[1] -> 8[0] via P2P/CUMEM
+0: jzxh159:1308400:1310256 [1] NCCL INFO Channel 02/0 : 1[1] -> 0[0] via P2P/CUMEM
+1: jzxh160:364772:365552 [1] NCCL INFO Channel 09/0 : 5[1] -> 4[0] via P2P/CUMEM
+3: jzxh162:3160743:3161551 [1] NCCL INFO Channel 05/0 : 13[1] -> 12[0] via P2P/CUMEM
+0: jzxh159:1308400:1310256 [1] NCCL INFO Channel 05/0 : 1[1] -> 0[0] via P2P/CUMEM
+1: jzxh160:364772:365552 [1] NCCL INFO Channel 10/0 : 5[1] -> 4[0] via P2P/CUMEM
+3: jzxh162:3160743:3161551 [1] NCCL INFO Channel 06/0 : 13[1] -> 12[0] via P2P/CUMEM
+1: jzxh160:364772:365552 [1] NCCL INFO Channel 13/0 : 5[1] -> 4[0] via P2P/CUMEM
+0: jzxh159:1308400:1310256 [1] NCCL INFO Channel 06/0 : 1[1] -> 0[0] via P2P/CUMEM
+1: jzxh160:364772:365552 [1] NCCL INFO Channel 14/0 : 5[1] -> 4[0] via P2P/CUMEM
+3: jzxh162:3160743:3161551 [1] NCCL INFO Channel 09/0 : 13[1] -> 12[0] via P2P/CUMEM
+0: jzxh159:1308400:1310256 [1] NCCL INFO Channel 09/0 : 1[1] -> 0[0] via P2P/CUMEM
+3: jzxh162:3160743:3161551 [1] NCCL INFO Channel 10/0 : 13[1] -> 12[0] via P2P/CUMEM
+0: jzxh159:1308400:1310256 [1] NCCL INFO Channel 10/0 : 1[1] -> 0[0] via P2P/CUMEM
+3: jzxh162:3160743:3161551 [1] NCCL INFO Channel 13/0 : 13[1] -> 12[0] via P2P/CUMEM
+0: jzxh159:1308400:1310256 [1] NCCL INFO Channel 13/0 : 1[1] -> 0[0] via P2P/CUMEM
+3: jzxh162:3160743:3161551 [1] NCCL INFO Channel 14/0 : 13[1] -> 12[0] via P2P/CUMEM
+0: jzxh159:1308400:1310256 [1] NCCL INFO Channel 14/0 : 1[1] -> 0[0] via P2P/CUMEM
+0: jzxh159:1308402:1310258 [3] NCCL INFO Channel 03/0 : 14[2] -> 3[3] [receive] via NET/IB/3/GDRDMA
+0: jzxh159:1308401:1310257 [2] NCCL INFO Channel 03/0 : 2[2] -> 7[3] [send] via NET/IB/3(3)/GDRDMA
+0: jzxh159:1308402:1310258 [3] NCCL INFO Channel 07/0 : 14[2] -> 3[3] [receive] via NET/IB/3/GDRDMA
+0: jzxh159:1308401:1310257 [2] NCCL INFO Channel 07/0 : 2[2] -> 7[3] [send] via NET/IB/3(3)/GDRDMA
+0: jzxh159:1308402:1310258 [3] NCCL INFO Channel 11/0 : 14[2] -> 3[3] [receive] via NET/IB/3/GDRDMA
+0: jzxh159:1308401:1310257 [2] NCCL INFO Channel 11/0 : 2[2] -> 7[3] [send] via NET/IB/3(3)/GDRDMA
+0: jzxh159:1308402:1310258 [3] NCCL INFO Channel 15/0 : 14[2] -> 3[3] [receive] via NET/IB/3/GDRDMA
+0: jzxh159:1308401:1310257 [2] NCCL INFO Channel 15/0 : 2[2] -> 7[3] [send] via NET/IB/3(3)/GDRDMA
+1: jzxh160:364774:365551 [3] NCCL INFO Channel 03/0 : 2[2] -> 7[3] [receive] via NET/IB/3/GDRDMA
+1: jzxh160:364773:365553 [2] NCCL INFO Channel 03/0 : 6[2] -> 11[3] [send] via NET/IB/3(7)/GDRDMA
+3: jzxh162:3160745:3161552 [3] NCCL INFO Channel 03/0 : 10[2] -> 15[3] [receive] via NET/IB/3/GDRDMA
+3: jzxh162:3160744:3161553 [2] NCCL INFO Channel 03/0 : 14[2] -> 3[3] [send] via NET/IB/3(15)/GDRDMA
+0: jzxh159:1308402:1310258 [3] NCCL INFO Channel 03/0 : 3[3] -> 0[0] via P2P/CUMEM
+1: jzxh160:364774:365551 [3] NCCL INFO Channel 07/0 : 2[2] -> 7[3] [receive] via NET/IB/3/GDRDMA
+3: jzxh162:3160745:3161552 [3] NCCL INFO Channel 07/0 : 10[2] -> 15[3] [receive] via NET/IB/3/GDRDMA
+1: jzxh160:364773:365553 [2] NCCL INFO Channel 07/0 : 6[2] -> 11[3] [send] via NET/IB/3(7)/GDRDMA
+3: jzxh162:3160744:3161553 [2] NCCL INFO Channel 07/0 : 14[2] -> 3[3] [send] via NET/IB/3(15)/GDRDMA
+1: jzxh160:364774:365551 [3] NCCL INFO Channel 11/0 : 2[2] -> 7[3] [receive] via NET/IB/3/GDRDMA
+3: jzxh162:3160745:3161552 [3] NCCL INFO Channel 11/0 : 10[2] -> 15[3] [receive] via NET/IB/3/GDRDMA
+1: jzxh160:364773:365553 [2] NCCL INFO Channel 11/0 : 6[2] -> 11[3] [send] via NET/IB/3(7)/GDRDMA
+3: jzxh162:3160744:3161553 [2] NCCL INFO Channel 11/0 : 14[2] -> 3[3] [send] via NET/IB/3(15)/GDRDMA
+1: jzxh160:364774:365551 [3] NCCL INFO Channel 15/0 : 2[2] -> 7[3] [receive] via NET/IB/3/GDRDMA
+3: jzxh162:3160745:3161552 [3] NCCL INFO Channel 15/0 : 10[2] -> 15[3] [receive] via NET/IB/3/GDRDMA
+1: jzxh160:364773:365553 [2] NCCL INFO Channel 15/0 : 6[2] -> 11[3] [send] via NET/IB/3(7)/GDRDMA
+3: jzxh162:3160744:3161553 [2] NCCL INFO Channel 15/0 : 14[2] -> 3[3] [send] via NET/IB/3(15)/GDRDMA
+1: jzxh160:364774:365551 [3] NCCL INFO Channel 03/0 : 7[3] -> 4[0] via P2P/CUMEM
+0: jzxh159:1308402:1310258 [3] NCCL INFO Channel 07/0 : 3[3] -> 0[0] via P2P/CUMEM
+0: jzxh159:1308402:1310258 [3] NCCL INFO Channel 11/0 : 3[3] -> 0[0] via P2P/CUMEM
+1: jzxh160:364774:365551 [3] NCCL INFO Channel 07/0 : 7[3] -> 4[0] via P2P/CUMEM
+2: jzxh161:3283538:3284335 [2] NCCL INFO Channel 03/0 : 10[2] -> 15[3] [send] via NET/IB/3(11)/GDRDMA
+1: jzxh160:364774:365551 [3] NCCL INFO Channel 11/0 : 7[3] -> 4[0] via P2P/CUMEM
+0: jzxh159:1308402:1310258 [3] NCCL INFO Channel 15/0 : 3[3] -> 0[0] via P2P/CUMEM
+1: jzxh160:364774:365551 [3] NCCL INFO Channel 15/0 : 7[3] -> 4[0] via P2P/CUMEM
+2: jzxh161:3283539:3284338 [3] NCCL INFO Channel 03/0 : 6[2] -> 11[3] [receive] via NET/IB/3/GDRDMA
+2: jzxh161:3283538:3284335 [2] NCCL INFO Channel 07/0 : 10[2] -> 15[3] [send] via NET/IB/3(11)/GDRDMA
+2: jzxh161:3283539:3284338 [3] NCCL INFO Channel 07/0 : 6[2] -> 11[3] [receive] via NET/IB/3/GDRDMA
+2: jzxh161:3283538:3284335 [2] NCCL INFO Channel 11/0 : 10[2] -> 15[3] [send] via NET/IB/3(11)/GDRDMA
+2: jzxh161:3283539:3284338 [3] NCCL INFO Channel 11/0 : 6[2] -> 11[3] [receive] via NET/IB/3/GDRDMA
+2: jzxh161:3283538:3284335 [2] NCCL INFO Channel 15/0 : 10[2] -> 15[3] [send] via NET/IB/3(11)/GDRDMA
+2: jzxh161:3283539:3284338 [3] NCCL INFO Channel 15/0 : 6[2] -> 11[3] [receive] via NET/IB/3/GDRDMA
+3: jzxh162:3160744:3161553 [2] NCCL INFO Channel 02/0 : 14[2] -> 13[1] via P2P/CUMEM
+0: jzxh159:1308401:1310257 [2] NCCL INFO Channel 02/0 : 2[2] -> 1[1] via P2P/CUMEM
+1: jzxh160:364774:365551 [3] NCCL INFO Channel 01/0 : 7[3] -> 6[2] via P2P/CUMEM
+0: jzxh159:1308402:1310258 [3] NCCL INFO Channel 01/0 : 3[3] -> 2[2] via P2P/CUMEM
+0: jzxh159:1308401:1310257 [2] NCCL INFO Channel 06/0 : 2[2] -> 1[1] via P2P/CUMEM
+2: jzxh161:3283539:3284338 [3] NCCL INFO Channel 03/0 : 11[3] -> 8[0] via P2P/CUMEM
+3: jzxh162:3160745:3161552 [3] NCCL INFO Channel 03/0 : 15[3] -> 12[0] via P2P/CUMEM
+3: jzxh162:3160744:3161553 [2] NCCL INFO Channel 06/0 : 14[2] -> 13[1] via P2P/CUMEM
+1: jzxh160:364774:365551 [3] NCCL INFO Channel 05/0 : 7[3] -> 6[2] via P2P/CUMEM
+0: jzxh159:1308402:1310258 [3] NCCL INFO Channel 05/0 : 3[3] -> 2[2] via P2P/CUMEM
+2: jzxh161:3283539:3284338 [3] NCCL INFO Channel 07/0 : 11[3] -> 8[0] via P2P/CUMEM
+3: jzxh162:3160745:3161552 [3] NCCL INFO Channel 07/0 : 15[3] -> 12[0] via P2P/CUMEM
+3: jzxh162:3160744:3161553 [2] NCCL INFO Channel 10/0 : 14[2] -> 13[1] via P2P/CUMEM
+0: jzxh159:1308401:1310257 [2] NCCL INFO Channel 10/0 : 2[2] -> 1[1] via P2P/CUMEM
+1: jzxh160:364774:365551 [3] NCCL INFO Channel 09/0 : 7[3] -> 6[2] via P2P/CUMEM
+2: jzxh161:3283539:3284338 [3] NCCL INFO Channel 11/0 : 11[3] -> 8[0] via P2P/CUMEM
+3: jzxh162:3160744:3161553 [2] NCCL INFO Channel 14/0 : 14[2] -> 13[1] via P2P/CUMEM
+2: jzxh161:3283539:3284338 [3] NCCL INFO Channel 15/0 : 11[3] -> 8[0] via P2P/CUMEM
+3: jzxh162:3160745:3161552 [3] NCCL INFO Channel 11/0 : 15[3] -> 12[0] via P2P/CUMEM
+0: jzxh159:1308402:1310258 [3] NCCL INFO Channel 09/0 : 3[3] -> 2[2] via P2P/CUMEM
+0: jzxh159:1308401:1310257 [2] NCCL INFO Channel 14/0 : 2[2] -> 1[1] via P2P/CUMEM
+1: jzxh160:364774:365551 [3] NCCL INFO Channel 13/0 : 7[3] -> 6[2] via P2P/CUMEM
+3: jzxh162:3160745:3161552 [3] NCCL INFO Channel 15/0 : 15[3] -> 12[0] via P2P/CUMEM
+2: jzxh161:3283538:3284335 [2] NCCL INFO Channel 02/0 : 10[2] -> 9[1] via P2P/CUMEM
+2: jzxh161:3283538:3284335 [2] NCCL INFO Channel 06/0 : 10[2] -> 9[1] via P2P/CUMEM
+3: jzxh162:3160745:3161552 [3] NCCL INFO Channel 01/0 : 15[3] -> 14[2] via P2P/CUMEM
+0: jzxh159:1308402:1310258 [3] NCCL INFO Channel 13/0 : 3[3] -> 2[2] via P2P/CUMEM
+2: jzxh161:3283539:3284338 [3] NCCL INFO Channel 01/0 : 11[3] -> 10[2] via P2P/CUMEM
+2: jzxh161:3283538:3284335 [2] NCCL INFO Channel 10/0 : 10[2] -> 9[1] via P2P/CUMEM
+3: jzxh162:3160745:3161552 [3] NCCL INFO Channel 05/0 : 15[3] -> 14[2] via P2P/CUMEM
+1: jzxh160:364773:365553 [2] NCCL INFO Channel 02/0 : 6[2] -> 5[1] via P2P/CUMEM
+1: jzxh160:364773:365553 [2] NCCL INFO Channel 06/0 : 6[2] -> 5[1] via P2P/CUMEM
+2: jzxh161:3283539:3284338 [3] NCCL INFO Channel 05/0 : 11[3] -> 10[2] via P2P/CUMEM
+3: jzxh162:3160745:3161552 [3] NCCL INFO Channel 09/0 : 15[3] -> 14[2] via P2P/CUMEM
+2: jzxh161:3283538:3284335 [2] NCCL INFO Channel 14/0 : 10[2] -> 9[1] via P2P/CUMEM
+2: jzxh161:3283539:3284338 [3] NCCL INFO Channel 09/0 : 11[3] -> 10[2] via P2P/CUMEM
+3: jzxh162:3160745:3161552 [3] NCCL INFO Channel 13/0 : 15[3] -> 14[2] via P2P/CUMEM
+1: jzxh160:364773:365553 [2] NCCL INFO Channel 10/0 : 6[2] -> 5[1] via P2P/CUMEM
+2: jzxh161:3283539:3284338 [3] NCCL INFO Channel 13/0 : 11[3] -> 10[2] via P2P/CUMEM
+1: jzxh160:364773:365553 [2] NCCL INFO Channel 14/0 : 6[2] -> 5[1] via P2P/CUMEM
+0: jzxh159:1308401:1310257 [2] NCCL INFO Connected all rings
+0: jzxh159:1308399:1310255 [0] NCCL INFO Connected all rings
+0: jzxh159:1308402:1310258 [3] NCCL INFO Connected all rings
+0: jzxh159:1308400:1310256 [1] NCCL INFO Connected all rings
+2: jzxh161:3283536:3284336 [0] NCCL INFO Connected all rings
+0: jzxh159:1308399:1310255 [0] NCCL INFO Channel 02/0 : 0[0] -> 1[1] via P2P/CUMEM
+1: jzxh160:364773:365553 [2] NCCL INFO Connected all rings
+1: jzxh160:364774:365551 [3] NCCL INFO Connected all rings
+1: jzxh160:364771:365554 [0] NCCL INFO Connected all rings
+1: jzxh160:364772:365552 [1] NCCL INFO Connected all rings
+3: jzxh162:3160744:3161553 [2] NCCL INFO Connected all rings
+3: jzxh162:3160745:3161552 [3] NCCL INFO Connected all rings
+2: jzxh161:3283536:3284336 [0] NCCL INFO Channel 02/0 : 8[0] -> 9[1] via P2P/CUMEM
+2: jzxh161:3283536:3284336 [0] NCCL INFO Channel 10/0 : 8[0] -> 9[1] via P2P/CUMEM
+1: jzxh160:364771:365554 [0] NCCL INFO Channel 02/0 : 4[0] -> 5[1] via P2P/CUMEM
+3: jzxh162:3160742:3161550 [0] NCCL INFO Connected all rings
+3: jzxh162:3160743:3161551 [1] NCCL INFO Connected all rings
+1: jzxh160:364771:365554 [0] NCCL INFO Channel 10/0 : 4[0] -> 5[1] via P2P/CUMEM
+3: jzxh162:3160742:3161550 [0] NCCL INFO Channel 02/0 : 12[0] -> 13[1] via P2P/CUMEM
+0: jzxh159:1308399:1310255 [0] NCCL INFO Channel 10/0 : 0[0] -> 1[1] via P2P/CUMEM
+2: jzxh161:3283537:3284337 [1] NCCL INFO Connected all rings
+0: jzxh159:1308401:1310257 [2] NCCL INFO Channel 01/0 : 2[2] -> 3[3] via P2P/CUMEM
+2: jzxh161:3283539:3284338 [3] NCCL INFO Connected all rings
+2: jzxh161:3283538:3284335 [2] NCCL INFO Connected all rings
+0: jzxh159:1308401:1310257 [2] NCCL INFO Channel 07/0 : 2[2] -> 3[3] via P2P/CUMEM
+0: jzxh159:1308400:1310256 [1] NCCL INFO Channel 01/0 : 1[1] -> 2[2] via P2P/CUMEM
+1: jzxh160:364772:365552 [1] NCCL INFO Channel 01/0 : 5[1] -> 6[2] via P2P/CUMEM
+3: jzxh162:3160742:3161550 [0] NCCL INFO Channel 10/0 : 12[0] -> 13[1] via P2P/CUMEM
+0: jzxh159:1308401:1310257 [2] NCCL INFO Channel 09/0 : 2[2] -> 3[3] via P2P/CUMEM
+1: jzxh160:364773:365553 [2] NCCL INFO Channel 01/0 : 6[2] -> 7[3] via P2P/CUMEM
+3: jzxh162:3160744:3161553 [2] NCCL INFO Channel 01/0 : 14[2] -> 15[3] via P2P/CUMEM
+0: jzxh159:1308400:1310256 [1] NCCL INFO Channel 09/0 : 1[1] -> 2[2] via P2P/CUMEM
+1: jzxh160:364772:365552 [1] NCCL INFO Channel 09/0 : 5[1] -> 6[2] via P2P/CUMEM
+2: jzxh161:3283537:3284337 [1] NCCL INFO Channel 01/0 : 9[1] -> 10[2] via P2P/CUMEM
+3: jzxh162:3160744:3161553 [2] NCCL INFO Channel 07/0 : 14[2] -> 15[3] via P2P/CUMEM
+0: jzxh159:1308401:1310257 [2] NCCL INFO Channel 15/0 : 2[2] -> 3[3] via P2P/CUMEM
+1: jzxh160:364773:365553 [2] NCCL INFO Channel 07/0 : 6[2] -> 7[3] via P2P/CUMEM
+0: jzxh159:1308399:1310255 [0] NCCL INFO Channel 02/0 : 0[0] -> 2[2] via P2P/CUMEM
+2: jzxh161:3283538:3284335 [2] NCCL INFO Channel 01/0 : 10[2] -> 11[3] via P2P/CUMEM
+0: jzxh159:1308400:1310256 [1] NCCL INFO Channel 02/0 : 1[1] -> 3[3] via P2P/CUMEM
+1: jzxh160:364773:365553 [2] NCCL INFO Channel 09/0 : 6[2] -> 7[3] via P2P/CUMEM
+1: jzxh160:364771:365554 [0] NCCL INFO Channel 02/0 : 4[0] -> 6[2] via P2P/CUMEM
+3: jzxh162:3160743:3161551 [1] NCCL INFO Channel 01/0 : 13[1] -> 14[2] via P2P/CUMEM
+0: jzxh159:1308399:1310255 [0] NCCL INFO Channel 03/0 : 0[0] -> 2[2] via P2P/CUMEM
+1: jzxh160:364773:365553 [2] NCCL INFO Channel 15/0 : 6[2] -> 7[3] via P2P/CUMEM
+2: jzxh161:3283537:3284337 [1] NCCL INFO Channel 09/0 : 9[1] -> 10[2] via P2P/CUMEM
+0: jzxh159:1308400:1310256 [1] NCCL INFO Channel 03/0 : 1[1] -> 3[3] via P2P/CUMEM
+2: jzxh161:3283538:3284335 [2] NCCL INFO Channel 07/0 : 10[2] -> 11[3] via P2P/CUMEM
+3: jzxh162:3160744:3161553 [2] NCCL INFO Channel 09/0 : 14[2] -> 15[3] via P2P/CUMEM
+3: jzxh162:3160743:3161551 [1] NCCL INFO Channel 09/0 : 13[1] -> 14[2] via P2P/CUMEM
+0: jzxh159:1308399:1310255 [0] NCCL INFO Channel 05/0 : 0[0] -> 2[2] via P2P/CUMEM
+1: jzxh160:364771:365554 [0] NCCL INFO Channel 03/0 : 4[0] -> 6[2] via P2P/CUMEM
+2: jzxh161:3283536:3284336 [0] NCCL INFO Channel 02/0 : 8[0] -> 10[2] via P2P/CUMEM
+2: jzxh161:3283538:3284335 [2] NCCL INFO Channel 09/0 : 10[2] -> 11[3] via P2P/CUMEM
+3: jzxh162:3160744:3161553 [2] NCCL INFO Channel 15/0 : 14[2] -> 15[3] via P2P/CUMEM
+0: jzxh159:1308400:1310256 [1] NCCL INFO Channel 05/0 : 1[1] -> 3[3] via P2P/CUMEM
+1: jzxh160:364771:365554 [0] NCCL INFO Channel 05/0 : 4[0] -> 6[2] via P2P/CUMEM
+1: jzxh160:364772:365552 [1] NCCL INFO Channel 02/0 : 5[1] -> 7[3] via P2P/CUMEM
+2: jzxh161:3283538:3284335 [2] NCCL INFO Channel 15/0 : 10[2] -> 11[3] via P2P/CUMEM
+0: jzxh159:1308400:1310256 [1] NCCL INFO Channel 06/0 : 1[1] -> 3[3] via P2P/CUMEM
+0: jzxh159:1308399:1310255 [0] NCCL INFO Channel 06/0 : 0[0] -> 2[2] via P2P/CUMEM
+2: jzxh161:3283536:3284336 [0] NCCL INFO Channel 03/0 : 8[0] -> 10[2] via P2P/CUMEM
+1: jzxh160:364772:365552 [1] NCCL INFO Channel 03/0 : 5[1] -> 7[3] via P2P/CUMEM
+0: jzxh159:1308399:1310255 [0] NCCL INFO Channel 10/0 : 0[0] -> 2[2] via P2P/CUMEM
+0: jzxh159:1308400:1310256 [1] NCCL INFO Channel 10/0 : 1[1] -> 3[3] via P2P/CUMEM
+1: jzxh160:364771:365554 [0] NCCL INFO Channel 06/0 : 4[0] -> 6[2] via P2P/CUMEM
+2: jzxh161:3283536:3284336 [0] NCCL INFO Channel 05/0 : 8[0] -> 10[2] via P2P/CUMEM
+0: jzxh159:1308399:1310255 [0] NCCL INFO Channel 11/0 : 0[0] -> 2[2] via P2P/CUMEM
+2: jzxh161:3283537:3284337 [1] NCCL INFO Channel 02/0 : 9[1] -> 11[3] via P2P/CUMEM
+0: jzxh159:1308400:1310256 [1] NCCL INFO Channel 11/0 : 1[1] -> 3[3] via P2P/CUMEM
+2: jzxh161:3283536:3284336 [0] NCCL INFO Channel 06/0 : 8[0] -> 10[2] via P2P/CUMEM
+1: jzxh160:364772:365552 [1] NCCL INFO Channel 05/0 : 5[1] -> 7[3] via P2P/CUMEM
+0: jzxh159:1308399:1310255 [0] NCCL INFO Channel 13/0 : 0[0] -> 2[2] via P2P/CUMEM
+0: jzxh159:1308400:1310256 [1] NCCL INFO Channel 13/0 : 1[1] -> 3[3] via P2P/CUMEM
+1: jzxh160:364771:365554 [0] NCCL INFO Channel 10/0 : 4[0] -> 6[2] via P2P/CUMEM
+0: jzxh159:1308400:1310256 [1] NCCL INFO Channel 14/0 : 1[1] -> 3[3] via P2P/CUMEM
+1: jzxh160:364771:365554 [0] NCCL INFO Channel 11/0 : 4[0] -> 6[2] via P2P/CUMEM
+3: jzxh162:3160742:3161550 [0] NCCL INFO Channel 02/0 : 12[0] -> 14[2] via P2P/CUMEM
+0: jzxh159:1308399:1310255 [0] NCCL INFO Channel 14/0 : 0[0] -> 2[2] via P2P/CUMEM
+3: jzxh162:3160743:3161551 [1] NCCL INFO Channel 02/0 : 13[1] -> 15[3] via P2P/CUMEM
+1: jzxh160:364772:365552 [1] NCCL INFO Channel 06/0 : 5[1] -> 7[3] via P2P/CUMEM
+3: jzxh162:3160742:3161550 [0] NCCL INFO Channel 03/0 : 12[0] -> 14[2] via P2P/CUMEM
+1: jzxh160:364772:365552 [1] NCCL INFO Channel 10/0 : 5[1] -> 7[3] via P2P/CUMEM
+1: jzxh160:364771:365554 [0] NCCL INFO Channel 13/0 : 4[0] -> 6[2] via P2P/CUMEM
+1: jzxh160:364771:365554 [0] NCCL INFO Channel 14/0 : 4[0] -> 6[2] via P2P/CUMEM
+2: jzxh161:3283536:3284336 [0] NCCL INFO Channel 10/0 : 8[0] -> 10[2] via P2P/CUMEM
+3: jzxh162:3160743:3161551 [1] NCCL INFO Channel 03/0 : 13[1] -> 15[3] via P2P/CUMEM
+1: jzxh160:364772:365552 [1] NCCL INFO Channel 11/0 : 5[1] -> 7[3] via P2P/CUMEM
+3: jzxh162:3160742:3161550 [0] NCCL INFO Channel 05/0 : 12[0] -> 14[2] via P2P/CUMEM
+2: jzxh161:3283536:3284336 [0] NCCL INFO Channel 11/0 : 8[0] -> 10[2] via P2P/CUMEM
+1: jzxh160:364772:365552 [1] NCCL INFO Channel 13/0 : 5[1] -> 7[3] via P2P/CUMEM
+2: jzxh161:3283537:3284337 [1] NCCL INFO Channel 03/0 : 9[1] -> 11[3] via P2P/CUMEM
+3: jzxh162:3160742:3161550 [0] NCCL INFO Channel 06/0 : 12[0] -> 14[2] via P2P/CUMEM
+2: jzxh161:3283536:3284336 [0] NCCL INFO Channel 13/0 : 8[0] -> 10[2] via P2P/CUMEM
+3: jzxh162:3160743:3161551 [1] NCCL INFO Channel 05/0 : 13[1] -> 15[3] via P2P/CUMEM
+2: jzxh161:3283537:3284337 [1] NCCL INFO Channel 05/0 : 9[1] -> 11[3] via P2P/CUMEM
+1: jzxh160:364772:365552 [1] NCCL INFO Channel 14/0 : 5[1] -> 7[3] via P2P/CUMEM
+2: jzxh161:3283536:3284336 [0] NCCL INFO Channel 14/0 : 8[0] -> 10[2] via P2P/CUMEM
+0: jzxh159:1308399:1310255 [0] NCCL INFO Channel 04/0 : 0[0] -> 3[3] via P2P/CUMEM
+2: jzxh161:3283537:3284337 [1] NCCL INFO Channel 06/0 : 9[1] -> 11[3] via P2P/CUMEM
+3: jzxh162:3160743:3161551 [1] NCCL INFO Channel 06/0 : 13[1] -> 15[3] via P2P/CUMEM
+0: jzxh159:1308401:1310257 [2] NCCL INFO Channel 10/0 : 2[2] -> 6[2] [send] via NET/IB/2/GDRDMA
+0: jzxh159:1308401:1310257 [2] NCCL INFO Channel 14/0 : 2[2] -> 6[2] [send] via NET/IB/2/GDRDMA
+0: jzxh159:1308400:1310256 [1] NCCL INFO Channel 09/0 : 1[1] -> 5[1] [send] via NET/IB/1/GDRDMA
+0: jzxh159:1308399:1310255 [0] NCCL INFO Channel 12/0 : 0[0] -> 3[3] via P2P/CUMEM
+0: jzxh159:1308400:1310256 [1] NCCL INFO Channel 13/0 : 1[1] -> 5[1] [send] via NET/IB/1/GDRDMA
+2: jzxh161:3283537:3284337 [1] NCCL INFO Channel 10/0 : 9[1] -> 11[3] via P2P/CUMEM
+3: jzxh162:3160742:3161550 [0] NCCL INFO Channel 10/0 : 12[0] -> 14[2] via P2P/CUMEM
+1: jzxh160:364771:365554 [0] NCCL INFO Channel 04/0 : 4[0] -> 7[3] via P2P/CUMEM
+1: jzxh160:364773:365553 [2] NCCL INFO Channel 10/0 : 2[2] -> 6[2] [receive] via NET/IB/2/GDRDMA
+2: jzxh161:3283537:3284337 [1] NCCL INFO Channel 11/0 : 9[1] -> 11[3] via P2P/CUMEM
+1: jzxh160:364772:365552 [1] NCCL INFO Channel 09/0 : 1[1] -> 5[1] [receive] via NET/IB/1/GDRDMA
+1: jzxh160:364773:365553 [2] NCCL INFO Channel 14/0 : 2[2] -> 6[2] [receive] via NET/IB/2/GDRDMA
+0: jzxh159:1308399:1310255 [0] NCCL INFO Channel 08/0 : 0[0] -> 4[0] [send] via NET/IB/0/GDRDMA
+0: jzxh159:1308402:1310258 [3] NCCL INFO Channel 11/0 : 3[3] -> 7[3] [send] via NET/IB/3/GDRDMA
+0: jzxh159:1308399:1310255 [0] NCCL INFO Channel 12/0 : 0[0] -> 4[0] [send] via NET/IB/0/GDRDMA
+0: jzxh159:1308402:1310258 [3] NCCL INFO Channel 15/0 : 3[3] -> 7[3] [send] via NET/IB/3/GDRDMA
+1: jzxh160:364771:365554 [0] NCCL INFO Channel 12/0 : 4[0] -> 7[3] via P2P/CUMEM
+1: jzxh160:364772:365552 [1] NCCL INFO Channel 13/0 : 1[1] -> 5[1] [receive] via NET/IB/1/GDRDMA
+2: jzxh161:3283537:3284337 [1] NCCL INFO Channel 13/0 : 9[1] -> 11[3] via P2P/CUMEM
+1: jzxh160:364773:365553 [2] NCCL INFO Channel 02/0 : 6[2] -> 10[2] [send] via NET/IB/2/GDRDMA
+1: jzxh160:364772:365552 [1] NCCL INFO Channel 01/0 : 5[1] -> 9[1] [send] via NET/IB/1/GDRDMA
+0: jzxh159:1308401:1310257 [2] NCCL INFO Channel 02/0 : 10[2] -> 2[2] [receive] via NET/IB/2/GDRDMA
+1: jzxh160:364773:365553 [2] NCCL INFO Channel 06/0 : 6[2] -> 10[2] [send] via NET/IB/2/GDRDMA
+1: jzxh160:364772:365552 [1] NCCL INFO Channel 05/0 : 5[1] -> 9[1] [send] via NET/IB/1/GDRDMA
+1: jzxh160:364773:365553 [2] NCCL INFO Channel 10/0 : 6[2] -> 10[2] [send] via NET/IB/2/GDRDMA
+1: jzxh160:364772:365552 [1] NCCL INFO Channel 09/0 : 5[1] -> 9[1] [send] via NET/IB/1/GDRDMA
+1: jzxh160:364773:365553 [2] NCCL INFO Channel 14/0 : 6[2] -> 10[2] [send] via NET/IB/2/GDRDMA
+0: jzxh159:1308401:1310257 [2] NCCL INFO Channel 06/0 : 10[2] -> 2[2] [receive] via NET/IB/2/GDRDMA
+0: jzxh159:1308400:1310256 [1] NCCL INFO Channel 01/0 : 9[1] -> 1[1] [receive] via NET/IB/1/GDRDMA
+0: jzxh159:1308401:1310257 [2] NCCL INFO Channel 02/0 : 2[2] -> 10[2] [send] via NET/IB/2/GDRDMA
+2: jzxh161:3283537:3284337 [1] NCCL INFO Channel 14/0 : 9[1] -> 11[3] via P2P/CUMEM
+0: jzxh159:1308401:1310257 [2] NCCL INFO Channel 06/0 : 2[2] -> 10[2] [send] via NET/IB/2/GDRDMA
+0: jzxh159:1308400:1310256 [1] NCCL INFO Channel 05/0 : 9[1] -> 1[1] [receive] via NET/IB/1/GDRDMA
+0: jzxh159:1308400:1310256 [1] NCCL INFO Channel 01/0 : 1[1] -> 9[1] [send] via NET/IB/1/GDRDMA
+0: jzxh159:1308400:1310256 [1] NCCL INFO Channel 05/0 : 1[1] -> 9[1] [send] via NET/IB/1/GDRDMA
+2: jzxh161:3283536:3284336 [0] NCCL INFO Channel 04/0 : 8[0] -> 11[3] via P2P/CUMEM
+2: jzxh161:3283537:3284337 [1] NCCL INFO Channel 01/0 : 5[1] -> 9[1] [receive] via NET/IB/1/GDRDMA
+1: jzxh160:364772:365552 [1] NCCL INFO Channel 13/0 : 5[1] -> 9[1] [send] via NET/IB/1/GDRDMA
+1: jzxh160:364771:365554 [0] NCCL INFO Channel 08/0 : 0[0] -> 4[0] [receive] via NET/IB/0/GDRDMA
+1: jzxh160:364774:365551 [3] NCCL INFO Channel 11/0 : 3[3] -> 7[3] [receive] via NET/IB/3/GDRDMA
+1: jzxh160:364771:365554 [0] NCCL INFO Channel 12/0 : 0[0] -> 4[0] [receive] via NET/IB/0/GDRDMA
+1: jzxh160:364774:365551 [3] NCCL INFO Channel 15/0 : 3[3] -> 7[3] [receive] via NET/IB/3/GDRDMA
+1: jzxh160:364774:365551 [3] NCCL INFO Channel 03/0 : 7[3] -> 11[3] [send] via NET/IB/3/GDRDMA
+1: jzxh160:364771:365554 [0] NCCL INFO Channel 00/0 : 4[0] -> 8[0] [send] via NET/IB/0/GDRDMA
+1: jzxh160:364774:365551 [3] NCCL INFO Channel 07/0 : 7[3] -> 11[3] [send] via NET/IB/3/GDRDMA
+1: jzxh160:364771:365554 [0] NCCL INFO Channel 04/0 : 4[0] -> 8[0] [send] via NET/IB/0/GDRDMA
+0: jzxh159:1308402:1310258 [3] NCCL INFO Channel 03/0 : 11[3] -> 3[3] [receive] via NET/IB/3/GDRDMA
+0: jzxh159:1308399:1310255 [0] NCCL INFO Channel 00/0 : 8[0] -> 0[0] [receive] via NET/IB/0/GDRDMA
+0: jzxh159:1308402:1310258 [3] NCCL INFO Channel 07/0 : 11[3] -> 3[3] [receive] via NET/IB/3/GDRDMA
+0: jzxh159:1308399:1310255 [0] NCCL INFO Channel 04/0 : 8[0] -> 0[0] [receive] via NET/IB/0/GDRDMA
+0: jzxh159:1308399:1310255 [0] NCCL INFO Channel 00/0 : 0[0] -> 8[0] [send] via NET/IB/0/GDRDMA
+0: jzxh159:1308402:1310258 [3] NCCL INFO Channel 03/0 : 3[3] -> 11[3] [send] via NET/IB/3/GDRDMA
+2: jzxh161:3283538:3284335 [2] NCCL INFO Channel 02/0 : 6[2] -> 10[2] [receive] via NET/IB/2/GDRDMA
+2: jzxh161:3283537:3284337 [1] NCCL INFO Channel 05/0 : 5[1] -> 9[1] [receive] via NET/IB/1/GDRDMA
+2: jzxh161:3283538:3284335 [2] NCCL INFO Channel 06/0 : 6[2] -> 10[2] [receive] via NET/IB/2/GDRDMA
+2: jzxh161:3283536:3284336 [0] NCCL INFO Channel 12/0 : 8[0] -> 11[3] via P2P/CUMEM
+2: jzxh161:3283537:3284337 [1] NCCL INFO Channel 09/0 : 5[1] -> 9[1] [receive] via NET/IB/1/GDRDMA
+2: jzxh161:3283538:3284335 [2] NCCL INFO Channel 10/0 : 6[2] -> 10[2] [receive] via NET/IB/2/GDRDMA
+1: jzxh160:364774:365551 [3] NCCL INFO Channel 11/0 : 7[3] -> 11[3] [send] via NET/IB/3/GDRDMA
+1: jzxh160:364771:365554 [0] NCCL INFO Channel 08/0 : 4[0] -> 8[0] [send] via NET/IB/0/GDRDMA
+1: jzxh160:364774:365551 [3] NCCL INFO Channel 15/0 : 7[3] -> 11[3] [send] via NET/IB/3/GDRDMA
+1: jzxh160:364771:365554 [0] NCCL INFO Channel 12/0 : 4[0] -> 8[0] [send] via NET/IB/0/GDRDMA
+2: jzxh161:3283538:3284335 [2] NCCL INFO Channel 14/0 : 6[2] -> 10[2] [receive] via NET/IB/2/GDRDMA
+2: jzxh161:3283537:3284337 [1] NCCL INFO Channel 13/0 : 5[1] -> 9[1] [receive] via NET/IB/1/GDRDMA
+2: jzxh161:3283538:3284335 [2] NCCL INFO Channel 02/0 : 10[2] -> 14[2] [send] via NET/IB/2/GDRDMA
+2: jzxh161:3283537:3284337 [1] NCCL INFO Channel 01/0 : 9[1] -> 13[1] [send] via NET/IB/1/GDRDMA
+2: jzxh161:3283538:3284335 [2] NCCL INFO Channel 06/0 : 10[2] -> 14[2] [send] via NET/IB/2/GDRDMA
+2: jzxh161:3283537:3284337 [1] NCCL INFO Channel 05/0 : 9[1] -> 13[1] [send] via NET/IB/1/GDRDMA
+1: jzxh160:364773:365553 [2] NCCL INFO Channel 10/0 : 14[2] -> 6[2] [receive] via NET/IB/2/GDRDMA
+1: jzxh160:364773:365553 [2] NCCL INFO Channel 14/0 : 14[2] -> 6[2] [receive] via NET/IB/2/GDRDMA
+1: jzxh160:364772:365552 [1] NCCL INFO Channel 09/0 : 13[1] -> 5[1] [receive] via NET/IB/1/GDRDMA
+1: jzxh160:364773:365553 [2] NCCL INFO Channel 10/0 : 6[2] -> 14[2] [send] via NET/IB/2/GDRDMA
+2: jzxh161:3283539:3284338 [3] NCCL INFO Channel 03/0 : 7[3] -> 11[3] [receive] via NET/IB/3/GDRDMA
+2: jzxh161:3283536:3284336 [0] NCCL INFO Channel 00/0 : 4[0] -> 8[0] [receive] via NET/IB/0/GDRDMA
+2: jzxh161:3283539:3284338 [3] NCCL INFO Channel 07/0 : 7[3] -> 11[3] [receive] via NET/IB/3/GDRDMA
+2: jzxh161:3283536:3284336 [0] NCCL INFO Channel 04/0 : 4[0] -> 8[0] [receive] via NET/IB/0/GDRDMA
+2: jzxh161:3283539:3284338 [3] NCCL INFO Channel 11/0 : 7[3] -> 11[3] [receive] via NET/IB/3/GDRDMA
+2: jzxh161:3283536:3284336 [0] NCCL INFO Channel 08/0 : 4[0] -> 8[0] [receive] via NET/IB/0/GDRDMA
+2: jzxh161:3283539:3284338 [3] NCCL INFO Channel 15/0 : 7[3] -> 11[3] [receive] via NET/IB/3/GDRDMA
+2: jzxh161:3283536:3284336 [0] NCCL INFO Channel 12/0 : 4[0] -> 8[0] [receive] via NET/IB/0/GDRDMA
+2: jzxh161:3283539:3284338 [3] NCCL INFO Channel 03/0 : 11[3] -> 15[3] [send] via NET/IB/3/GDRDMA
+2: jzxh161:3283536:3284336 [0] NCCL INFO Channel 00/0 : 8[0] -> 12[0] [send] via NET/IB/0/GDRDMA
+2: jzxh161:3283539:3284338 [3] NCCL INFO Channel 07/0 : 11[3] -> 15[3] [send] via NET/IB/3/GDRDMA
+1: jzxh160:364772:365552 [1] NCCL INFO Channel 13/0 : 13[1] -> 5[1] [receive] via NET/IB/1/GDRDMA
+1: jzxh160:364773:365553 [2] NCCL INFO Channel 14/0 : 6[2] -> 14[2] [send] via NET/IB/2/GDRDMA
+1: jzxh160:364772:365552 [1] NCCL INFO Channel 09/0 : 5[1] -> 13[1] [send] via NET/IB/1/GDRDMA
+1: jzxh160:364772:365552 [1] NCCL INFO Channel 13/0 : 5[1] -> 13[1] [send] via NET/IB/1/GDRDMA
+1: jzxh160:364774:365551 [3] NCCL INFO Channel 11/0 : 15[3] -> 7[3] [receive] via NET/IB/3/GDRDMA
+2: jzxh161:3283536:3284336 [0] NCCL INFO Channel 04/0 : 8[0] -> 12[0] [send] via NET/IB/0/GDRDMA
+1: jzxh160:364774:365551 [3] NCCL INFO Channel 15/0 : 15[3] -> 7[3] [receive] via NET/IB/3/GDRDMA
+1: jzxh160:364771:365554 [0] NCCL INFO Channel 08/0 : 12[0] -> 4[0] [receive] via NET/IB/0/GDRDMA
+1: jzxh160:364774:365551 [3] NCCL INFO Channel 11/0 : 7[3] -> 15[3] [send] via NET/IB/3/GDRDMA
+1: jzxh160:364771:365554 [0] NCCL INFO Channel 12/0 : 12[0] -> 4[0] [receive] via NET/IB/0/GDRDMA
+1: jzxh160:364774:365551 [3] NCCL INFO Channel 15/0 : 7[3] -> 15[3] [send] via NET/IB/3/GDRDMA
+1: jzxh160:364771:365554 [0] NCCL INFO Channel 08/0 : 4[0] -> 12[0] [send] via NET/IB/0/GDRDMA
+1: jzxh160:364771:365554 [0] NCCL INFO Channel 12/0 : 4[0] -> 12[0] [send] via NET/IB/0/GDRDMA
+0: jzxh159:1308399:1310255 [0] NCCL INFO Channel 04/0 : 0[0] -> 8[0] [send] via NET/IB/0/GDRDMA
+3: jzxh162:3160742:3161550 [0] NCCL INFO Channel 11/0 : 12[0] -> 14[2] via P2P/CUMEM
+0: jzxh159:1308402:1310258 [3] NCCL INFO Channel 07/0 : 3[3] -> 11[3] [send] via NET/IB/3/GDRDMA
+3: jzxh162:3160743:3161551 [1] NCCL INFO Channel 10/0 : 13[1] -> 15[3] via P2P/CUMEM
+3: jzxh162:3160742:3161550 [0] NCCL INFO Channel 13/0 : 12[0] -> 14[2] via P2P/CUMEM
+3: jzxh162:3160743:3161551 [1] NCCL INFO Channel 11/0 : 13[1] -> 15[3] via P2P/CUMEM
+3: jzxh162:3160742:3161550 [0] NCCL INFO Channel 14/0 : 12[0] -> 14[2] via P2P/CUMEM
+3: jzxh162:3160743:3161551 [1] NCCL INFO Channel 13/0 : 13[1] -> 15[3] via P2P/CUMEM
+3: jzxh162:3160743:3161551 [1] NCCL INFO Channel 14/0 : 13[1] -> 15[3] via P2P/CUMEM
+3: jzxh162:3160742:3161550 [0] NCCL INFO Channel 04/0 : 12[0] -> 15[3] via P2P/CUMEM
+3: jzxh162:3160744:3161553 [2] NCCL INFO Channel 02/0 : 10[2] -> 14[2] [receive] via NET/IB/2/GDRDMA
+3: jzxh162:3160744:3161553 [2] NCCL INFO Channel 06/0 : 10[2] -> 14[2] [receive] via NET/IB/2/GDRDMA
+3: jzxh162:3160742:3161550 [0] NCCL INFO Channel 12/0 : 12[0] -> 15[3] via P2P/CUMEM
+3: jzxh162:3160744:3161553 [2] NCCL INFO Channel 10/0 : 6[2] -> 14[2] [receive] via NET/IB/2/GDRDMA
+3: jzxh162:3160744:3161553 [2] NCCL INFO Channel 14/0 : 6[2] -> 14[2] [receive] via NET/IB/2/GDRDMA
+3: jzxh162:3160743:3161551 [1] NCCL INFO Channel 01/0 : 9[1] -> 13[1] [receive] via NET/IB/1/GDRDMA
+3: jzxh162:3160744:3161553 [2] NCCL INFO Channel 10/0 : 14[2] -> 6[2] [send] via NET/IB/2/GDRDMA
+3: jzxh162:3160743:3161551 [1] NCCL INFO Channel 05/0 : 9[1] -> 13[1] [receive] via NET/IB/1/GDRDMA
+3: jzxh162:3160744:3161553 [2] NCCL INFO Channel 14/0 : 14[2] -> 6[2] [send] via NET/IB/2/GDRDMA
+3: jzxh162:3160743:3161551 [1] NCCL INFO Channel 09/0 : 5[1] -> 13[1] [receive] via NET/IB/1/GDRDMA
+3: jzxh162:3160744:3161553 [2] NCCL INFO Channel 02/0 : 14[2] -> 10[2] [send] via NET/IB/2/GDRDMA
+3: jzxh162:3160743:3161551 [1] NCCL INFO Channel 13/0 : 5[1] -> 13[1] [receive] via NET/IB/1/GDRDMA
+3: jzxh162:3160744:3161553 [2] NCCL INFO Channel 06/0 : 14[2] -> 10[2] [send] via NET/IB/2/GDRDMA
+1: jzxh160:364773:365553 [2] NCCL INFO Channel 02/0 : 10[2] -> 6[2] [receive] via NET/IB/2/GDRDMA
+3: jzxh162:3160742:3161550 [0] NCCL INFO Channel 00/0 : 8[0] -> 12[0] [receive] via NET/IB/0/GDRDMA
+3: jzxh162:3160745:3161552 [3] NCCL INFO Channel 03/0 : 11[3] -> 15[3] [receive] via NET/IB/3/GDRDMA
+3: jzxh162:3160743:3161551 [1] NCCL INFO Channel 09/0 : 13[1] -> 5[1] [send] via NET/IB/1/GDRDMA
+3: jzxh162:3160745:3161552 [3] NCCL INFO Channel 07/0 : 11[3] -> 15[3] [receive] via NET/IB/3/GDRDMA
+3: jzxh162:3160742:3161550 [0] NCCL INFO Channel 04/0 : 8[0] -> 12[0] [receive] via NET/IB/0/GDRDMA
+3: jzxh162:3160743:3161551 [1] NCCL INFO Channel 13/0 : 13[1] -> 5[1] [send] via NET/IB/1/GDRDMA
+1: jzxh160:364773:365553 [2] NCCL INFO Channel 06/0 : 10[2] -> 6[2] [receive] via NET/IB/2/GDRDMA
+1: jzxh160:364772:365552 [1] NCCL INFO Channel 01/0 : 9[1] -> 5[1] [receive] via NET/IB/1/GDRDMA
+1: jzxh160:364773:365553 [2] NCCL INFO Channel 10/0 : 10[2] -> 6[2] [receive] via NET/IB/2/GDRDMA
+1: jzxh160:364772:365552 [1] NCCL INFO Channel 05/0 : 9[1] -> 5[1] [receive] via NET/IB/1/GDRDMA
+1: jzxh160:364773:365553 [2] NCCL INFO Channel 14/0 : 10[2] -> 6[2] [receive] via NET/IB/2/GDRDMA
+2: jzxh161:3283538:3284335 [2] NCCL INFO Channel 02/0 : 2[2] -> 10[2] [receive] via NET/IB/2/GDRDMA
+3: jzxh162:3160745:3161552 [3] NCCL INFO Channel 11/0 : 7[3] -> 15[3] [receive] via NET/IB/3/GDRDMA
+3: jzxh162:3160743:3161551 [1] NCCL INFO Channel 01/0 : 13[1] -> 9[1] [send] via NET/IB/1/GDRDMA
+3: jzxh162:3160742:3161550 [0] NCCL INFO Channel 08/0 : 4[0] -> 12[0] [receive] via NET/IB/0/GDRDMA
+3: jzxh162:3160745:3161552 [3] NCCL INFO Channel 15/0 : 7[3] -> 15[3] [receive] via NET/IB/3/GDRDMA
+3: jzxh162:3160743:3161551 [1] NCCL INFO Channel 05/0 : 13[1] -> 9[1] [send] via NET/IB/1/GDRDMA
+3: jzxh162:3160742:3161550 [0] NCCL INFO Channel 12/0 : 4[0] -> 12[0] [receive] via NET/IB/0/GDRDMA
+3: jzxh162:3160745:3161552 [3] NCCL INFO Channel 11/0 : 15[3] -> 7[3] [send] via NET/IB/3/GDRDMA
+3: jzxh162:3160745:3161552 [3] NCCL INFO Channel 15/0 : 15[3] -> 7[3] [send] via NET/IB/3/GDRDMA
+1: jzxh160:364773:365553 [2] NCCL INFO Channel 10/0 : 6[2] -> 2[2] [send] via NET/IB/2/GDRDMA
+1: jzxh160:364772:365552 [1] NCCL INFO Channel 09/0 : 9[1] -> 5[1] [receive] via NET/IB/1/GDRDMA
+1: jzxh160:364773:365553 [2] NCCL INFO Channel 14/0 : 6[2] -> 2[2] [send] via NET/IB/2/GDRDMA
+1: jzxh160:364772:365552 [1] NCCL INFO Channel 13/0 : 9[1] -> 5[1] [receive] via NET/IB/1/GDRDMA
+1: jzxh160:364772:365552 [1] NCCL INFO Channel 09/0 : 5[1] -> 1[1] [send] via NET/IB/1/GDRDMA
+1: jzxh160:364772:365552 [1] NCCL INFO Channel 13/0 : 5[1] -> 1[1] [send] via NET/IB/1/GDRDMA
+2: jzxh161:3283539:3284338 [3] NCCL INFO Channel 03/0 : 3[3] -> 11[3] [receive] via NET/IB/3/GDRDMA
+3: jzxh162:3160745:3161552 [3] NCCL INFO Channel 03/0 : 15[3] -> 11[3] [send] via NET/IB/3/GDRDMA
+0: jzxh159:1308401:1310257 [2] NCCL INFO Channel 10/0 : 6[2] -> 2[2] [receive] via NET/IB/2/GDRDMA
+2: jzxh161:3283538:3284335 [2] NCCL INFO Channel 06/0 : 2[2] -> 10[2] [receive] via NET/IB/2/GDRDMA
+2: jzxh161:3283537:3284337 [1] NCCL INFO Channel 01/0 : 1[1] -> 9[1] [receive] via NET/IB/1/GDRDMA
+3: jzxh162:3160745:3161552 [3] NCCL INFO Channel 07/0 : 15[3] -> 11[3] [send] via NET/IB/3/GDRDMA
+3: jzxh162:3160742:3161550 [0] NCCL INFO Channel 08/0 : 12[0] -> 4[0] [send] via NET/IB/0/GDRDMA
+2: jzxh161:3283539:3284338 [3] NCCL INFO Channel 07/0 : 3[3] -> 11[3] [receive] via NET/IB/3/GDRDMA
+2: jzxh161:3283538:3284335 [2] NCCL INFO Channel 02/0 : 10[2] -> 2[2] [send] via NET/IB/2/GDRDMA
+3: jzxh162:3160742:3161550 [0] NCCL INFO Channel 12/0 : 12[0] -> 4[0] [send] via NET/IB/0/GDRDMA
+2: jzxh161:3283537:3284337 [1] NCCL INFO Channel 05/0 : 1[1] -> 9[1] [receive] via NET/IB/1/GDRDMA
+2: jzxh161:3283538:3284335 [2] NCCL INFO Channel 06/0 : 10[2] -> 2[2] [send] via NET/IB/2/GDRDMA
+2: jzxh161:3283539:3284338 [3] NCCL INFO Channel 03/0 : 11[3] -> 3[3] [send] via NET/IB/3/GDRDMA
+2: jzxh161:3283537:3284337 [1] NCCL INFO Channel 01/0 : 9[1] -> 1[1] [send] via NET/IB/1/GDRDMA
+0: jzxh159:1308402:1310258 [3] NCCL INFO Channel 11/0 : 7[3] -> 3[3] [receive] via NET/IB/3/GDRDMA
+0: jzxh159:1308401:1310257 [2] NCCL INFO Channel 14/0 : 6[2] -> 2[2] [receive] via NET/IB/2/GDRDMA
+0: jzxh159:1308402:1310258 [3] NCCL INFO Channel 15/0 : 7[3] -> 3[3] [receive] via NET/IB/3/GDRDMA
+1: jzxh160:364774:365551 [3] NCCL INFO Channel 03/0 : 11[3] -> 7[3] [receive] via NET/IB/3/GDRDMA
+1: jzxh160:364774:365551 [3] NCCL INFO Channel 07/0 : 11[3] -> 7[3] [receive] via NET/IB/3/GDRDMA
+1: jzxh160:364774:365551 [3] NCCL INFO Channel 11/0 : 11[3] -> 7[3] [receive] via NET/IB/3/GDRDMA
+1: jzxh160:364774:365551 [3] NCCL INFO Channel 15/0 : 11[3] -> 7[3] [receive] via NET/IB/3/GDRDMA
+3: jzxh162:3160742:3161550 [0] NCCL INFO Channel 00/0 : 12[0] -> 8[0] [send] via NET/IB/0/GDRDMA
+3: jzxh162:3160742:3161550 [0] NCCL INFO Channel 04/0 : 12[0] -> 8[0] [send] via NET/IB/0/GDRDMA
+1: jzxh160:364771:365554 [0] NCCL INFO Channel 00/0 : 8[0] -> 4[0] [receive] via NET/IB/0/GDRDMA
+1: jzxh160:364774:365551 [3] NCCL INFO Channel 11/0 : 7[3] -> 3[3] [send] via NET/IB/3/GDRDMA
+2: jzxh161:3283539:3284338 [3] NCCL INFO Channel 07/0 : 11[3] -> 3[3] [send] via NET/IB/3/GDRDMA
+1: jzxh160:364774:365551 [3] NCCL INFO Channel 15/0 : 7[3] -> 3[3] [send] via NET/IB/3/GDRDMA
+1: jzxh160:364771:365554 [0] NCCL INFO Channel 04/0 : 8[0] -> 4[0] [receive] via NET/IB/0/GDRDMA
+2: jzxh161:3283537:3284337 [1] NCCL INFO Channel 05/0 : 9[1] -> 1[1] [send] via NET/IB/1/GDRDMA
+1: jzxh160:364771:365554 [0] NCCL INFO Channel 08/0 : 8[0] -> 4[0] [receive] via NET/IB/0/GDRDMA
+2: jzxh161:3283538:3284335 [2] NCCL INFO Channel 02/0 : 14[2] -> 10[2] [receive] via NET/IB/2/GDRDMA
+2: jzxh161:3283539:3284338 [3] NCCL INFO Channel 03/0 : 15[3] -> 11[3] [receive] via NET/IB/3/GDRDMA
+2: jzxh161:3283538:3284335 [2] NCCL INFO Channel 06/0 : 14[2] -> 10[2] [receive] via NET/IB/2/GDRDMA
+2: jzxh161:3283537:3284337 [1] NCCL INFO Channel 01/0 : 13[1] -> 9[1] [receive] via NET/IB/1/GDRDMA
+2: jzxh161:3283538:3284335 [2] NCCL INFO Channel 02/0 : 10[2] -> 6[2] [send] via NET/IB/2/GDRDMA
+2: jzxh161:3283539:3284338 [3] NCCL INFO Channel 07/0 : 15[3] -> 11[3] [receive] via NET/IB/3/GDRDMA
+2: jzxh161:3283537:3284337 [1] NCCL INFO Channel 05/0 : 13[1] -> 9[1] [receive] via NET/IB/1/GDRDMA
+0: jzxh159:1308401:1310257 [2] NCCL INFO Channel 02/0 : 2[2] -> 0[0] via P2P/CUMEM
+1: jzxh160:364771:365554 [0] NCCL INFO Channel 12/0 : 8[0] -> 4[0] [receive] via NET/IB/0/GDRDMA
+3: jzxh162:3160744:3161553 [2] NCCL INFO Channel 02/0 : 14[2] -> 12[0] via P2P/CUMEM
+0: jzxh159:1308401:1310257 [2] NCCL INFO Channel 03/0 : 2[2] -> 0[0] via P2P/CUMEM
+1: jzxh160:364771:365554 [0] NCCL INFO Channel 08/0 : 4[0] -> 0[0] [send] via NET/IB/0/GDRDMA
+1: jzxh160:364771:365554 [0] NCCL INFO Channel 12/0 : 4[0] -> 0[0] [send] via NET/IB/0/GDRDMA
+1: jzxh160:364773:365553 [2] NCCL INFO Channel 02/0 : 6[2] -> 4[0] via P2P/CUMEM
+2: jzxh161:3283538:3284335 [2] NCCL INFO Channel 06/0 : 10[2] -> 6[2] [send] via NET/IB/2/GDRDMA
+2: jzxh161:3283539:3284338 [3] NCCL INFO Channel 03/0 : 11[3] -> 7[3] [send] via NET/IB/3/GDRDMA
+0: jzxh159:1308400:1310256 [1] NCCL INFO Channel 09/0 : 5[1] -> 1[1] [receive] via NET/IB/1/GDRDMA
+2: jzxh161:3283537:3284337 [1] NCCL INFO Channel 01/0 : 9[1] -> 5[1] [send] via NET/IB/1/GDRDMA
+2: jzxh161:3283538:3284335 [2] NCCL INFO Channel 10/0 : 10[2] -> 6[2] [send] via NET/IB/2/GDRDMA
+2: jzxh161:3283539:3284338 [3] NCCL INFO Channel 07/0 : 11[3] -> 7[3] [send] via NET/IB/3/GDRDMA
+2: jzxh161:3283537:3284337 [1] NCCL INFO Channel 05/0 : 9[1] -> 5[1] [send] via NET/IB/1/GDRDMA
+2: jzxh161:3283538:3284335 [2] NCCL INFO Channel 14/0 : 10[2] -> 6[2] [send] via NET/IB/2/GDRDMA
+2: jzxh161:3283539:3284338 [3] NCCL INFO Channel 11/0 : 11[3] -> 7[3] [send] via NET/IB/3/GDRDMA
+2: jzxh161:3283537:3284337 [1] NCCL INFO Channel 09/0 : 9[1] -> 5[1] [send] via NET/IB/1/GDRDMA
+2: jzxh161:3283539:3284338 [3] NCCL INFO Channel 15/0 : 11[3] -> 7[3] [send] via NET/IB/3/GDRDMA
+2: jzxh161:3283537:3284337 [1] NCCL INFO Channel 13/0 : 9[1] -> 5[1] [send] via NET/IB/1/GDRDMA
+0: jzxh159:1308401:1310257 [2] NCCL INFO Channel 05/0 : 2[2] -> 0[0] via P2P/CUMEM
+0: jzxh159:1308400:1310256 [1] NCCL INFO Channel 13/0 : 5[1] -> 1[1] [receive] via NET/IB/1/GDRDMA
+3: jzxh162:3160745:3161552 [3] NCCL INFO Channel 01/0 : 15[3] -> 12[0] via P2P/CUMEM
+3: jzxh162:3160744:3161553 [2] NCCL INFO Channel 03/0 : 14[2] -> 12[0] via P2P/CUMEM
+1: jzxh160:364773:365553 [2] NCCL INFO Channel 03/0 : 6[2] -> 4[0] via P2P/CUMEM
+0: jzxh159:1308401:1310257 [2] NCCL INFO Channel 06/0 : 2[2] -> 0[0] via P2P/CUMEM
+3: jzxh162:3160745:3161552 [3] NCCL INFO Channel 04/0 : 15[3] -> 12[0] via P2P/CUMEM
+1: jzxh160:364773:365553 [2] NCCL INFO Channel 05/0 : 6[2] -> 4[0] via P2P/CUMEM
+1: jzxh160:364773:365553 [2] NCCL INFO Channel 06/0 : 6[2] -> 4[0] via P2P/CUMEM
+0: jzxh159:1308401:1310257 [2] NCCL INFO Channel 10/0 : 2[2] -> 0[0] via P2P/CUMEM
+1: jzxh160:364773:365553 [2] NCCL INFO Channel 10/0 : 6[2] -> 4[0] via P2P/CUMEM
+3: jzxh162:3160744:3161553 [2] NCCL INFO Channel 05/0 : 14[2] -> 12[0] via P2P/CUMEM
+3: jzxh162:3160745:3161552 [3] NCCL INFO Channel 05/0 : 15[3] -> 12[0] via P2P/CUMEM
+0: jzxh159:1308401:1310257 [2] NCCL INFO Channel 11/0 : 2[2] -> 0[0] via P2P/CUMEM
+1: jzxh160:364773:365553 [2] NCCL INFO Channel 11/0 : 6[2] -> 4[0] via P2P/CUMEM
+3: jzxh162:3160744:3161553 [2] NCCL INFO Channel 06/0 : 14[2] -> 12[0] via P2P/CUMEM
+0: jzxh159:1308401:1310257 [2] NCCL INFO Channel 13/0 : 2[2] -> 0[0] via P2P/CUMEM
+2: jzxh161:3283538:3284335 [2] NCCL INFO Channel 02/0 : 10[2] -> 8[0] via P2P/CUMEM
+0: jzxh159:1308402:1310258 [3] NCCL INFO Channel 01/0 : 3[3] -> 0[0] via P2P/CUMEM
+1: jzxh160:364773:365553 [2] NCCL INFO Channel 13/0 : 6[2] -> 4[0] via P2P/CUMEM
+2: jzxh161:3283538:3284335 [2] NCCL INFO Channel 03/0 : 10[2] -> 8[0] via P2P/CUMEM
+3: jzxh162:3160745:3161552 [3] NCCL INFO Channel 06/0 : 15[3] -> 12[0] via P2P/CUMEM
+1: jzxh160:364774:365551 [3] NCCL INFO Channel 01/0 : 7[3] -> 4[0] via P2P/CUMEM
+3: jzxh162:3160744:3161553 [2] NCCL INFO Channel 10/0 : 14[2] -> 12[0] via P2P/CUMEM
+0: jzxh159:1308401:1310257 [2] NCCL INFO Channel 14/0 : 2[2] -> 0[0] via P2P/CUMEM
+0: jzxh159:1308402:1310258 [3] NCCL INFO Channel 04/0 : 3[3] -> 0[0] via P2P/CUMEM
+3: jzxh162:3160745:3161552 [3] NCCL INFO Channel 09/0 : 15[3] -> 12[0] via P2P/CUMEM
+2: jzxh161:3283539:3284338 [3] NCCL INFO Channel 01/0 : 11[3] -> 8[0] via P2P/CUMEM
+3: jzxh162:3160744:3161553 [2] NCCL INFO Channel 11/0 : 14[2] -> 12[0] via P2P/CUMEM
+2: jzxh161:3283536:3284336 [0] NCCL INFO Channel 00/0 : 0[0] -> 8[0] [receive] via NET/IB/0/GDRDMA
+2: jzxh161:3283536:3284336 [0] NCCL INFO Channel 04/0 : 0[0] -> 8[0] [receive] via NET/IB/0/GDRDMA
+2: jzxh161:3283538:3284335 [2] NCCL INFO Channel 05/0 : 10[2] -> 8[0] via P2P/CUMEM
+1: jzxh160:364773:365553 [2] NCCL INFO Channel 14/0 : 6[2] -> 4[0] via P2P/CUMEM
+0: jzxh159:1308402:1310258 [3] NCCL INFO Channel 05/0 : 3[3] -> 0[0] via P2P/CUMEM
+3: jzxh162:3160744:3161553 [2] NCCL INFO Channel 13/0 : 14[2] -> 12[0] via P2P/CUMEM
+3: jzxh162:3160745:3161552 [3] NCCL INFO Channel 12/0 : 15[3] -> 12[0] via P2P/CUMEM
+2: jzxh161:3283539:3284338 [3] NCCL INFO Channel 04/0 : 11[3] -> 8[0] via P2P/CUMEM
+2: jzxh161:3283536:3284336 [0] NCCL INFO Channel 00/0 : 8[0] -> 0[0] [send] via NET/IB/0/GDRDMA
+2: jzxh161:3283536:3284336 [0] NCCL INFO Channel 04/0 : 8[0] -> 0[0] [send] via NET/IB/0/GDRDMA
+0: jzxh159:1308399:1310255 [0] NCCL INFO Channel 08/0 : 4[0] -> 0[0] [receive] via NET/IB/0/GDRDMA
+0: jzxh159:1308402:1310258 [3] NCCL INFO Channel 06/0 : 3[3] -> 0[0] via P2P/CUMEM
+0: jzxh159:1308399:1310255 [0] NCCL INFO Channel 12/0 : 4[0] -> 0[0] [receive] via NET/IB/0/GDRDMA
+3: jzxh162:3160744:3161553 [2] NCCL INFO Channel 14/0 : 14[2] -> 12[0] via P2P/CUMEM
+0: jzxh159:1308402:1310258 [3] NCCL INFO Channel 09/0 : 3[3] -> 0[0] via P2P/CUMEM
+2: jzxh161:3283538:3284335 [2] NCCL INFO Channel 06/0 : 10[2] -> 8[0] via P2P/CUMEM
+2: jzxh161:3283539:3284338 [3] NCCL INFO Channel 05/0 : 11[3] -> 8[0] via P2P/CUMEM
+2: jzxh161:3283536:3284336 [0] NCCL INFO Channel 00/0 : 12[0] -> 8[0] [receive] via NET/IB/0/GDRDMA
+2: jzxh161:3283536:3284336 [0] NCCL INFO Channel 04/0 : 12[0] -> 8[0] [receive] via NET/IB/0/GDRDMA
+2: jzxh161:3283538:3284335 [2] NCCL INFO Channel 10/0 : 10[2] -> 8[0] via P2P/CUMEM
+2: jzxh161:3283539:3284338 [3] NCCL INFO Channel 06/0 : 11[3] -> 8[0] via P2P/CUMEM
+2: jzxh161:3283536:3284336 [0] NCCL INFO Channel 00/0 : 8[0] -> 4[0] [send] via NET/IB/0/GDRDMA
+3: jzxh162:3160745:3161552 [3] NCCL INFO Channel 13/0 : 15[3] -> 12[0] via P2P/CUMEM
+0: jzxh159:1308402:1310258 [3] NCCL INFO Channel 12/0 : 3[3] -> 0[0] via P2P/CUMEM
+2: jzxh161:3283536:3284336 [0] NCCL INFO Channel 04/0 : 8[0] -> 4[0] [send] via NET/IB/0/GDRDMA
+3: jzxh162:3160745:3161552 [3] NCCL INFO Channel 14/0 : 15[3] -> 12[0] via P2P/CUMEM
+2: jzxh161:3283536:3284336 [0] NCCL INFO Channel 08/0 : 8[0] -> 4[0] [send] via NET/IB/0/GDRDMA
+2: jzxh161:3283538:3284335 [2] NCCL INFO Channel 11/0 : 10[2] -> 8[0] via P2P/CUMEM
+2: jzxh161:3283539:3284338 [3] NCCL INFO Channel 09/0 : 11[3] -> 8[0] via P2P/CUMEM
+2: jzxh161:3283536:3284336 [0] NCCL INFO Channel 12/0 : 8[0] -> 4[0] [send] via NET/IB/0/GDRDMA
+2: jzxh161:3283538:3284335 [2] NCCL INFO Channel 13/0 : 10[2] -> 8[0] via P2P/CUMEM
+0: jzxh159:1308402:1310258 [3] NCCL INFO Channel 13/0 : 3[3] -> 0[0] via P2P/CUMEM
+1: jzxh160:364774:365551 [3] NCCL INFO Channel 04/0 : 7[3] -> 4[0] via P2P/CUMEM
+2: jzxh161:3283539:3284338 [3] NCCL INFO Channel 12/0 : 11[3] -> 8[0] via P2P/CUMEM
+2: jzxh161:3283538:3284335 [2] NCCL INFO Channel 14/0 : 10[2] -> 8[0] via P2P/CUMEM
+2: jzxh161:3283539:3284338 [3] NCCL INFO Channel 13/0 : 11[3] -> 8[0] via P2P/CUMEM
+0: jzxh159:1308402:1310258 [3] NCCL INFO Channel 14/0 : 3[3] -> 0[0] via P2P/CUMEM
+1: jzxh160:364774:365551 [3] NCCL INFO Channel 05/0 : 7[3] -> 4[0] via P2P/CUMEM
+2: jzxh161:3283539:3284338 [3] NCCL INFO Channel 14/0 : 11[3] -> 8[0] via P2P/CUMEM
+1: jzxh160:364774:365551 [3] NCCL INFO Channel 06/0 : 7[3] -> 4[0] via P2P/CUMEM
+1: jzxh160:364774:365551 [3] NCCL INFO Channel 09/0 : 7[3] -> 4[0] via P2P/CUMEM
+1: jzxh160:364774:365551 [3] NCCL INFO Channel 12/0 : 7[3] -> 4[0] via P2P/CUMEM
+0: jzxh159:1308402:1310258 [3] NCCL INFO Channel 02/0 : 3[3] -> 1[1] via P2P/CUMEM
+1: jzxh160:364774:365551 [3] NCCL INFO Channel 13/0 : 7[3] -> 4[0] via P2P/CUMEM
+3: jzxh162:3160745:3161552 [3] NCCL INFO Channel 02/0 : 15[3] -> 13[1] via P2P/CUMEM
+0: jzxh159:1308402:1310258 [3] NCCL INFO Channel 03/0 : 3[3] -> 1[1] via P2P/CUMEM
+1: jzxh160:364774:365551 [3] NCCL INFO Channel 14/0 : 7[3] -> 4[0] via P2P/CUMEM
+3: jzxh162:3160745:3161552 [3] NCCL INFO Channel 03/0 : 15[3] -> 13[1] via P2P/CUMEM
+0: jzxh159:1308402:1310258 [3] NCCL INFO Channel 05/0 : 3[3] -> 1[1] via P2P/CUMEM
+3: jzxh162:3160745:3161552 [3] NCCL INFO Channel 05/0 : 15[3] -> 13[1] via P2P/CUMEM
+0: jzxh159:1308402:1310258 [3] NCCL INFO Channel 06/0 : 3[3] -> 1[1] via P2P/CUMEM
+3: jzxh162:3160745:3161552 [3] NCCL INFO Channel 06/0 : 15[3] -> 13[1] via P2P/CUMEM
+2: jzxh161:3283539:3284338 [3] NCCL INFO Channel 02/0 : 11[3] -> 9[1] via P2P/CUMEM
+0: jzxh159:1308402:1310258 [3] NCCL INFO Channel 10/0 : 3[3] -> 1[1] via P2P/CUMEM
+3: jzxh162:3160745:3161552 [3] NCCL INFO Channel 10/0 : 15[3] -> 13[1] via P2P/CUMEM
+2: jzxh161:3283539:3284338 [3] NCCL INFO Channel 03/0 : 11[3] -> 9[1] via P2P/CUMEM
+0: jzxh159:1308402:1310258 [3] NCCL INFO Channel 11/0 : 3[3] -> 1[1] via P2P/CUMEM
+3: jzxh162:3160745:3161552 [3] NCCL INFO Channel 11/0 : 15[3] -> 13[1] via P2P/CUMEM
+3: jzxh162:3160745:3161552 [3] NCCL INFO Channel 13/0 : 15[3] -> 13[1] via P2P/CUMEM
+2: jzxh161:3283539:3284338 [3] NCCL INFO Channel 05/0 : 11[3] -> 9[1] via P2P/CUMEM
+0: jzxh159:1308402:1310258 [3] NCCL INFO Channel 13/0 : 3[3] -> 1[1] via P2P/CUMEM
+0: jzxh159:1308402:1310258 [3] NCCL INFO Channel 14/0 : 3[3] -> 1[1] via P2P/CUMEM
+2: jzxh161:3283539:3284338 [3] NCCL INFO Channel 06/0 : 11[3] -> 9[1] via P2P/CUMEM
+3: jzxh162:3160745:3161552 [3] NCCL INFO Channel 14/0 : 15[3] -> 13[1] via P2P/CUMEM
+1: jzxh160:364774:365551 [3] NCCL INFO Channel 02/0 : 7[3] -> 5[1] via P2P/CUMEM
+0: jzxh159:1308402:1310258 [3] NCCL INFO Channel 00/0 : 3[3] -> 2[2] via P2P/CUMEM
+2: jzxh161:3283539:3284338 [3] NCCL INFO Channel 10/0 : 11[3] -> 9[1] via P2P/CUMEM
+3: jzxh162:3160745:3161552 [3] NCCL INFO Channel 00/0 : 15[3] -> 14[2] via P2P/CUMEM
+1: jzxh160:364774:365551 [3] NCCL INFO Channel 03/0 : 7[3] -> 5[1] via P2P/CUMEM
+0: jzxh159:1308402:1310258 [3] NCCL INFO Channel 04/0 : 3[3] -> 2[2] via P2P/CUMEM
+1: jzxh160:364774:365551 [3] NCCL INFO Channel 05/0 : 7[3] -> 5[1] via P2P/CUMEM
+2: jzxh161:3283539:3284338 [3] NCCL INFO Channel 11/0 : 11[3] -> 9[1] via P2P/CUMEM
+3: jzxh162:3160745:3161552 [3] NCCL INFO Channel 04/0 : 15[3] -> 14[2] via P2P/CUMEM
+0: jzxh159:1308402:1310258 [3] NCCL INFO Channel 07/0 : 3[3] -> 2[2] via P2P/CUMEM
+1: jzxh160:364774:365551 [3] NCCL INFO Channel 06/0 : 7[3] -> 5[1] via P2P/CUMEM
+2: jzxh161:3283539:3284338 [3] NCCL INFO Channel 13/0 : 11[3] -> 9[1] via P2P/CUMEM
+0: jzxh159:1308402:1310258 [3] NCCL INFO Channel 08/0 : 3[3] -> 2[2] via P2P/CUMEM
+1: jzxh160:364774:365551 [3] NCCL INFO Channel 10/0 : 7[3] -> 5[1] via P2P/CUMEM
+2: jzxh161:3283539:3284338 [3] NCCL INFO Channel 14/0 : 11[3] -> 9[1] via P2P/CUMEM
+3: jzxh162:3160745:3161552 [3] NCCL INFO Channel 07/0 : 15[3] -> 14[2] via P2P/CUMEM
+1: jzxh160:364774:365551 [3] NCCL INFO Channel 11/0 : 7[3] -> 5[1] via P2P/CUMEM
+0: jzxh159:1308402:1310258 [3] NCCL INFO Channel 12/0 : 3[3] -> 2[2] via P2P/CUMEM
+2: jzxh161:3283539:3284338 [3] NCCL INFO Channel 00/0 : 11[3] -> 10[2] via P2P/CUMEM
+3: jzxh162:3160745:3161552 [3] NCCL INFO Channel 08/0 : 15[3] -> 14[2] via P2P/CUMEM
+1: jzxh160:364774:365551 [3] NCCL INFO Channel 13/0 : 7[3] -> 5[1] via P2P/CUMEM
+0: jzxh159:1308402:1310258 [3] NCCL INFO Channel 15/0 : 3[3] -> 2[2] via P2P/CUMEM
+1: jzxh160:364774:365551 [3] NCCL INFO Channel 14/0 : 7[3] -> 5[1] via P2P/CUMEM
+2: jzxh161:3283539:3284338 [3] NCCL INFO Channel 04/0 : 11[3] -> 10[2] via P2P/CUMEM
+1: jzxh160:364774:365551 [3] NCCL INFO Channel 00/0 : 7[3] -> 6[2] via P2P/CUMEM
+3: jzxh162:3160745:3161552 [3] NCCL INFO Channel 12/0 : 15[3] -> 14[2] via P2P/CUMEM
+2: jzxh161:3283539:3284338 [3] NCCL INFO Channel 07/0 : 11[3] -> 10[2] via P2P/CUMEM
+1: jzxh160:364774:365551 [3] NCCL INFO Channel 04/0 : 7[3] -> 6[2] via P2P/CUMEM
+3: jzxh162:3160745:3161552 [3] NCCL INFO Channel 15/0 : 15[3] -> 14[2] via P2P/CUMEM
+2: jzxh161:3283539:3284338 [3] NCCL INFO Channel 08/0 : 11[3] -> 10[2] via P2P/CUMEM
+1: jzxh160:364774:365551 [3] NCCL INFO Channel 07/0 : 7[3] -> 6[2] via P2P/CUMEM
+3: jzxh162:3160744:3161553 [2] NCCL INFO Channel 00/0 : 14[2] -> 13[1] via P2P/CUMEM
+1: jzxh160:364774:365551 [3] NCCL INFO Channel 08/0 : 7[3] -> 6[2] via P2P/CUMEM
+2: jzxh161:3283539:3284338 [3] NCCL INFO Channel 12/0 : 11[3] -> 10[2] via P2P/CUMEM
+3: jzxh162:3160744:3161553 [2] NCCL INFO Channel 01/0 : 14[2] -> 13[1] via P2P/CUMEM
+0: jzxh159:1308401:1310257 [2] NCCL INFO Channel 00/0 : 2[2] -> 1[1] via P2P/CUMEM
+1: jzxh160:364774:365551 [3] NCCL INFO Channel 12/0 : 7[3] -> 6[2] via P2P/CUMEM
+3: jzxh162:3160744:3161553 [2] NCCL INFO Channel 04/0 : 14[2] -> 13[1] via P2P/CUMEM
+0: jzxh159:1308400:1310256 [1] NCCL INFO Channel 00/0 : 1[1] -> 0[0] via P2P/CUMEM
+0: jzxh159:1308401:1310257 [2] NCCL INFO Channel 01/0 : 2[2] -> 1[1] via P2P/CUMEM
+2: jzxh161:3283539:3284338 [3] NCCL INFO Channel 15/0 : 11[3] -> 10[2] via P2P/CUMEM
+3: jzxh162:3160744:3161553 [2] NCCL INFO Channel 07/0 : 14[2] -> 13[1] via P2P/CUMEM
+0: jzxh159:1308400:1310256 [1] NCCL INFO Channel 03/0 : 1[1] -> 0[0] via P2P/CUMEM
+0: jzxh159:1308401:1310257 [2] NCCL INFO Channel 04/0 : 2[2] -> 1[1] via P2P/CUMEM
+1: jzxh160:364774:365551 [3] NCCL INFO Channel 15/0 : 7[3] -> 6[2] via P2P/CUMEM
+3: jzxh162:3160743:3161551 [1] NCCL INFO Channel 00/0 : 13[1] -> 12[0] via P2P/CUMEM
+0: jzxh159:1308400:1310256 [1] NCCL INFO Channel 07/0 : 1[1] -> 0[0] via P2P/CUMEM
+0: jzxh159:1308401:1310257 [2] NCCL INFO Channel 07/0 : 2[2] -> 1[1] via P2P/CUMEM
+2: jzxh161:3283538:3284335 [2] NCCL INFO Channel 00/0 : 10[2] -> 9[1] via P2P/CUMEM
+3: jzxh162:3160744:3161553 [2] NCCL INFO Channel 08/0 : 14[2] -> 13[1] via P2P/CUMEM
+3: jzxh162:3160743:3161551 [1] NCCL INFO Channel 03/0 : 13[1] -> 12[0] via P2P/CUMEM
+0: jzxh159:1308400:1310256 [1] NCCL INFO Channel 08/0 : 1[1] -> 0[0] via P2P/CUMEM
+0: jzxh159:1308401:1310257 [2] NCCL INFO Channel 08/0 : 2[2] -> 1[1] via P2P/CUMEM
+2: jzxh161:3283538:3284335 [2] NCCL INFO Channel 01/0 : 10[2] -> 9[1] via P2P/CUMEM
+3: jzxh162:3160744:3161553 [2] NCCL INFO Channel 09/0 : 14[2] -> 13[1] via P2P/CUMEM
+2: jzxh161:3283538:3284335 [2] NCCL INFO Channel 04/0 : 10[2] -> 9[1] via P2P/CUMEM
+3: jzxh162:3160743:3161551 [1] NCCL INFO Channel 07/0 : 13[1] -> 12[0] via P2P/CUMEM
+1: jzxh160:364773:365553 [2] NCCL INFO Channel 00/0 : 6[2] -> 5[1] via P2P/CUMEM
+0: jzxh159:1308400:1310256 [1] NCCL INFO Channel 11/0 : 1[1] -> 0[0] via P2P/CUMEM
+0: jzxh159:1308401:1310257 [2] NCCL INFO Channel 09/0 : 2[2] -> 1[1] via P2P/CUMEM
+2: jzxh161:3283537:3284337 [1] NCCL INFO Channel 00/0 : 9[1] -> 8[0] via P2P/CUMEM
+3: jzxh162:3160744:3161553 [2] NCCL INFO Channel 12/0 : 14[2] -> 13[1] via P2P/CUMEM
+2: jzxh161:3283538:3284335 [2] NCCL INFO Channel 07/0 : 10[2] -> 9[1] via P2P/CUMEM
+3: jzxh162:3160743:3161551 [1] NCCL INFO Channel 08/0 : 13[1] -> 12[0] via P2P/CUMEM
+2: jzxh161:3283537:3284337 [1] NCCL INFO Channel 03/0 : 9[1] -> 8[0] via P2P/CUMEM
+0: jzxh159:1308400:1310256 [1] NCCL INFO Channel 15/0 : 1[1] -> 0[0] via P2P/CUMEM
+0: jzxh159:1308401:1310257 [2] NCCL INFO Channel 12/0 : 2[2] -> 1[1] via P2P/CUMEM
+1: jzxh160:364773:365553 [2] NCCL INFO Channel 01/0 : 6[2] -> 5[1] via P2P/CUMEM
+2: jzxh161:3283538:3284335 [2] NCCL INFO Channel 08/0 : 10[2] -> 9[1] via P2P/CUMEM
+2: jzxh161:3283537:3284337 [1] NCCL INFO Channel 07/0 : 9[1] -> 8[0] via P2P/CUMEM
+2: jzxh161:3283538:3284335 [2] NCCL INFO Channel 09/0 : 10[2] -> 9[1] via P2P/CUMEM
+3: jzxh162:3160744:3161553 [2] NCCL INFO Channel 15/0 : 14[2] -> 13[1] via P2P/CUMEM
+3: jzxh162:3160743:3161551 [1] NCCL INFO Channel 11/0 : 13[1] -> 12[0] via P2P/CUMEM
+0: jzxh159:1308401:1310257 [2] NCCL INFO Channel 15/0 : 2[2] -> 1[1] via P2P/CUMEM
+1: jzxh160:364773:365553 [2] NCCL INFO Channel 04/0 : 6[2] -> 5[1] via P2P/CUMEM
+2: jzxh161:3283537:3284337 [1] NCCL INFO Channel 08/0 : 9[1] -> 8[0] via P2P/CUMEM
+2: jzxh161:3283538:3284335 [2] NCCL INFO Channel 12/0 : 10[2] -> 9[1] via P2P/CUMEM
+3: jzxh162:3160743:3161551 [1] NCCL INFO Channel 15/0 : 13[1] -> 12[0] via P2P/CUMEM
+1: jzxh160:364772:365552 [1] NCCL INFO Channel 00/0 : 5[1] -> 4[0] via P2P/CUMEM
+2: jzxh161:3283537:3284337 [1] NCCL INFO Channel 11/0 : 9[1] -> 8[0] via P2P/CUMEM
+2: jzxh161:3283538:3284335 [2] NCCL INFO Channel 15/0 : 10[2] -> 9[1] via P2P/CUMEM
+1: jzxh160:364773:365553 [2] NCCL INFO Channel 07/0 : 6[2] -> 5[1] via P2P/CUMEM
+1: jzxh160:364772:365552 [1] NCCL INFO Channel 03/0 : 5[1] -> 4[0] via P2P/CUMEM
+2: jzxh161:3283537:3284337 [1] NCCL INFO Channel 15/0 : 9[1] -> 8[0] via P2P/CUMEM
+1: jzxh160:364773:365553 [2] NCCL INFO Channel 08/0 : 6[2] -> 5[1] via P2P/CUMEM
+1: jzxh160:364772:365552 [1] NCCL INFO Channel 07/0 : 5[1] -> 4[0] via P2P/CUMEM
+1: jzxh160:364773:365553 [2] NCCL INFO Channel 09/0 : 6[2] -> 5[1] via P2P/CUMEM
+1: jzxh160:364772:365552 [1] NCCL INFO Channel 08/0 : 5[1] -> 4[0] via P2P/CUMEM
+1: jzxh160:364773:365553 [2] NCCL INFO Channel 12/0 : 6[2] -> 5[1] via P2P/CUMEM
+1: jzxh160:364772:365552 [1] NCCL INFO Channel 11/0 : 5[1] -> 4[0] via P2P/CUMEM
+1: jzxh160:364773:365553 [2] NCCL INFO Channel 15/0 : 6[2] -> 5[1] via P2P/CUMEM
+1: jzxh160:364772:365552 [1] NCCL INFO Channel 15/0 : 5[1] -> 4[0] via P2P/CUMEM
+0: jzxh159:1308399:1310255 [0] NCCL INFO Connected all trees
+0: jzxh159:1308400:1310256 [1] NCCL INFO Connected all trees
+0: jzxh159:1308400:1310256 [1] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512
+0: jzxh159:1308400:1310256 [1] NCCL INFO 16 coll channels, 16 collnet channels, 0 nvls channels, 16 p2p channels, 2 p2p channels per peer
+0: jzxh159:1308402:1310258 [3] NCCL INFO Connected all trees
+0: jzxh159:1308401:1310257 [2] NCCL INFO Connected all trees
+0: jzxh159:1308399:1310255 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512
+0: jzxh159:1308402:1310258 [3] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512
+0: jzxh159:1308399:1310255 [0] NCCL INFO 16 coll channels, 16 collnet channels, 0 nvls channels, 16 p2p channels, 2 p2p channels per peer
+0: jzxh159:1308402:1310258 [3] NCCL INFO 16 coll channels, 16 collnet channels, 0 nvls channels, 16 p2p channels, 2 p2p channels per peer
+0: jzxh159:1308401:1310257 [2] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512
+0: jzxh159:1308401:1310257 [2] NCCL INFO 16 coll channels, 16 collnet channels, 0 nvls channels, 16 p2p channels, 2 p2p channels per peer
+1: jzxh160:364771:365554 [0] NCCL INFO Connected all trees
+3: jzxh162:3160742:3161550 [0] NCCL INFO Connected all trees
+3: jzxh162:3160743:3161551 [1] NCCL INFO Connected all trees
+2: jzxh161:3283538:3284335 [2] NCCL INFO Connected all trees
+2: jzxh161:3283539:3284338 [3] NCCL INFO Connected all trees
+3: jzxh162:3160745:3161552 [3] NCCL INFO Connected all trees
+3: jzxh162:3160743:3161551 [1] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512
+3: jzxh162:3160742:3161550 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512
+3: jzxh162:3160743:3161551 [1] NCCL INFO 16 coll channels, 16 collnet channels, 0 nvls channels, 16 p2p channels, 2 p2p channels per peer
+3: jzxh162:3160742:3161550 [0] NCCL INFO 16 coll channels, 16 collnet channels, 0 nvls channels, 16 p2p channels, 2 p2p channels per peer
+3: jzxh162:3160745:3161552 [3] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512
+3: jzxh162:3160745:3161552 [3] NCCL INFO 16 coll channels, 16 collnet channels, 0 nvls channels, 16 p2p channels, 2 p2p channels per peer
+3: jzxh162:3160744:3161553 [2] NCCL INFO Connected all trees
+3: jzxh162:3160744:3161553 [2] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512
+3: jzxh162:3160744:3161553 [2] NCCL INFO 16 coll channels, 16 collnet channels, 0 nvls channels, 16 p2p channels, 2 p2p channels per peer
+1: jzxh160:364772:365552 [1] NCCL INFO Connected all trees
+1: jzxh160:364771:365554 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512
+1: jzxh160:364774:365551 [3] NCCL INFO Connected all trees
+1: jzxh160:364771:365554 [0] NCCL INFO 16 coll channels, 16 collnet channels, 0 nvls channels, 16 p2p channels, 2 p2p channels per peer
+1: jzxh160:364772:365552 [1] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512
+1: jzxh160:364772:365552 [1] NCCL INFO 16 coll channels, 16 collnet channels, 0 nvls channels, 16 p2p channels, 2 p2p channels per peer
+1: jzxh160:364773:365553 [2] NCCL INFO Connected all trees
+1: jzxh160:364774:365551 [3] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512
+1: jzxh160:364774:365551 [3] NCCL INFO 16 coll channels, 16 collnet channels, 0 nvls channels, 16 p2p channels, 2 p2p channels per peer
+1: jzxh160:364773:365553 [2] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512
+1: jzxh160:364773:365553 [2] NCCL INFO 16 coll channels, 16 collnet channels, 0 nvls channels, 16 p2p channels, 2 p2p channels per peer
+2: jzxh161:3283538:3284335 [2] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512
+2: jzxh161:3283538:3284335 [2] NCCL INFO 16 coll channels, 16 collnet channels, 0 nvls channels, 16 p2p channels, 2 p2p channels per peer
+2: jzxh161:3283539:3284338 [3] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512
+2: jzxh161:3283539:3284338 [3] NCCL INFO 16 coll channels, 16 collnet channels, 0 nvls channels, 16 p2p channels, 2 p2p channels per peer
+2: jzxh161:3283536:3284336 [0] NCCL INFO Connected all trees
+2: jzxh161:3283537:3284337 [1] NCCL INFO Connected all trees
+2: jzxh161:3283536:3284336 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512
+2: jzxh161:3283536:3284336 [0] NCCL INFO 16 coll channels, 16 collnet channels, 0 nvls channels, 16 p2p channels, 2 p2p channels per peer
+2: jzxh161:3283537:3284337 [1] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512
+2: jzxh161:3283537:3284337 [1] NCCL INFO 16 coll channels, 16 collnet channels, 0 nvls channels, 16 p2p channels, 2 p2p channels per peer
+0: jzxh159:1308401:1310257 [2] NCCL INFO TUNER/Plugin: Plugin load returned 2 : libnccl-net.so: cannot open shared object file: No such file or directory : when loading libnccl-tuner.so
+0: jzxh159:1308401:1310257 [2] NCCL INFO TUNER/Plugin: Using internal tuner plugin.
+0: jzxh159:1308401:1310257 [2] NCCL INFO ncclCommInitRank comm 0x55cb335512a0 rank 2 nranks 16 cudaDev 2 nvmlDev 2 busId 9d000 commId 0x524c94fe7e5cd79a - Init COMPLETE
+0: jzxh159:1308400:1310256 [1] NCCL INFO TUNER/Plugin: Plugin load returned 2 : libnccl-net.so: cannot open shared object file: No such file or directory : when loading libnccl-tuner.so
+0: jzxh159:1308399:1310255 [0] NCCL INFO TUNER/Plugin: Plugin load returned 2 : libnccl-net.so: cannot open shared object file: No such file or directory : when loading libnccl-tuner.so
+0: jzxh159:1308402:1310258 [3] NCCL INFO TUNER/Plugin: Plugin load returned 2 : libnccl-net.so: cannot open shared object file: No such file or directory : when loading libnccl-tuner.so
+0: jzxh159:1308400:1310256 [1] NCCL INFO TUNER/Plugin: Using internal tuner plugin.
+0: jzxh159:1308399:1310255 [0] NCCL INFO TUNER/Plugin: Using internal tuner plugin.
+0: jzxh159:1308402:1310258 [3] NCCL INFO TUNER/Plugin: Using internal tuner plugin.
+0: jzxh159:1308400:1310256 [1] NCCL INFO ncclCommInitRank comm 0x5576720d8870 rank 1 nranks 16 cudaDev 1 nvmlDev 1 busId 2c000 commId 0x524c94fe7e5cd79a - Init COMPLETE
+0: jzxh159:1308399:1310255 [0] NCCL INFO ncclCommInitRank comm 0x5650a3b6e180 rank 0 nranks 16 cudaDev 0 nvmlDev 0 busId 1b000 commId 0x524c94fe7e5cd79a - Init COMPLETE
+0: jzxh159:1308402:1310258 [3] NCCL INFO ncclCommInitRank comm 0x55cdfc201020 rank 3 nranks 16 cudaDev 3 nvmlDev 3 busId ad000 commId 0x524c94fe7e5cd79a - Init COMPLETE
+3: jzxh162:3160745:3161552 [3] NCCL INFO TUNER/Plugin: Plugin load returned 2 : libnccl-net.so: cannot open shared object file: No such file or directory : when loading libnccl-tuner.so
+3: jzxh162:3160743:3161551 [1] NCCL INFO TUNER/Plugin: Plugin load returned 2 : libnccl-net.so: cannot open shared object file: No such file or directory : when loading libnccl-tuner.so
+3: jzxh162:3160742:3161550 [0] NCCL INFO TUNER/Plugin: Plugin load returned 2 : libnccl-net.so: cannot open shared object file: No such file or directory : when loading libnccl-tuner.so
+3: jzxh162:3160745:3161552 [3] NCCL INFO TUNER/Plugin: Using internal tuner plugin.
+3: jzxh162:3160743:3161551 [1] NCCL INFO TUNER/Plugin: Using internal tuner plugin.
+3: jzxh162:3160742:3161550 [0] NCCL INFO TUNER/Plugin: Using internal tuner plugin.
+3: jzxh162:3160745:3161552 [3] NCCL INFO ncclCommInitRank comm 0x55c152c5c3b0 rank 15 nranks 16 cudaDev 3 nvmlDev 3 busId ad000 commId 0x524c94fe7e5cd79a - Init COMPLETE
+3: jzxh162:3160742:3161550 [0] NCCL INFO ncclCommInitRank comm 0x5649a9566180 rank 12 nranks 16 cudaDev 0 nvmlDev 0 busId 1b000 commId 0x524c94fe7e5cd79a - Init COMPLETE
+3: jzxh162:3160743:3161551 [1] NCCL INFO ncclCommInitRank comm 0x55dd2bde7570 rank 13 nranks 16 cudaDev 1 nvmlDev 1 busId 2c000 commId 0x524c94fe7e5cd79a - Init COMPLETE
+3: jzxh162:3160744:3161553 [2] NCCL INFO TUNER/Plugin: Plugin load returned 2 : libnccl-net.so: cannot open shared object file: No such file or directory : when loading libnccl-tuner.so
+3: jzxh162:3160744:3161553 [2] NCCL INFO TUNER/Plugin: Using internal tuner plugin.
+3: jzxh162:3160744:3161553 [2] NCCL INFO ncclCommInitRank comm 0x56427a552c70 rank 14 nranks 16 cudaDev 2 nvmlDev 2 busId 9d000 commId 0x524c94fe7e5cd79a - Init COMPLETE
+3: jzxh162:3160744:3161582 [2] NCCL INFO Channel 08/1 : 14[2] -> 0[0] [send] via NET/IB/0(12)/GDRDMA/Shared
+3: jzxh162:3160744:3161582 [2] NCCL INFO Channel 09/1 : 14[2] -> 0[0] [send] via NET/IB/0(12)/GDRDMA/Shared
+0: jzxh159:1308401:1310287 [2] NCCL INFO Channel 00/1 : 2[2] -> 0[0] via P2P/CUMEM
+0: jzxh159:1308402:1310288 [3] NCCL INFO Channel 00/1 : 3[3] -> 0[0] via P2P/CUMEM
+0: jzxh159:1308400:1310289 [1] NCCL INFO Channel 00/1 : 1[1] -> 0[0] via P2P/CUMEM
+3: jzxh162:3160742:3161583 [0] NCCL INFO Channel 08/1 : 12[0] -> 0[0] [send] via NET/IB/0/GDRDMA/Shared
+0: jzxh159:1308399:1310290 [0] NCCL INFO Channel 08/1 : 15[3] -> 0[0] [receive] via NET/IB/0/GDRDMA/Shared
+3: jzxh162:3160745:3161584 [3] NCCL INFO Channel 08/1 : 15[3] -> 0[0] [send] via NET/IB/0(12)/GDRDMA/Shared
+3: jzxh162:3160742:3161583 [0] NCCL INFO Channel 09/1 : 12[0] -> 0[0] [send] via NET/IB/0/GDRDMA/Shared
+0: jzxh159:1308399:1310290 [0] NCCL INFO Channel 09/1 : 15[3] -> 0[0] [receive] via NET/IB/0/GDRDMA/Shared
+3: jzxh162:3160745:3161584 [3] NCCL INFO Channel 09/1 : 15[3] -> 0[0] [send] via NET/IB/0(12)/GDRDMA/Shared
+0: jzxh159:1308401:1310287 [2] NCCL INFO Channel 01/1 : 2[2] -> 0[0] via P2P/CUMEM
+0: jzxh159:1308402:1310288 [3] NCCL INFO Channel 01/1 : 3[3] -> 0[0] via P2P/CUMEM
+0: jzxh159:1308400:1310289 [1] NCCL INFO Channel 01/1 : 1[1] -> 0[0] via P2P/CUMEM
+0: jzxh159:1308399:1310290 [0] NCCL INFO Channel 08/1 : 14[2] -> 0[0] [receive] via NET/IB/0/GDRDMA/Shared
+3: jzxh162:3160743:3161585 [1] NCCL INFO Channel 08/1 : 13[1] -> 0[0] [send] via NET/IB/0(12)/GDRDMA/Shared
+3: jzxh162:3160743:3161585 [1] NCCL INFO Channel 09/1 : 13[1] -> 0[0] [send] via NET/IB/0(12)/GDRDMA/Shared
+0: jzxh159:1308399:1310290 [0] NCCL INFO Channel 09/1 : 14[2] -> 0[0] [receive] via NET/IB/0/GDRDMA/Shared
+0: jzxh159:1308399:1310290 [0] NCCL INFO Channel 08/1 : 13[1] -> 0[0] [receive] via NET/IB/0/GDRDMA/Shared
+0: jzxh159:1308399:1310290 [0] NCCL INFO Channel 09/1 : 13[1] -> 0[0] [receive] via NET/IB/0/GDRDMA/Shared
+0: jzxh159:1308399:1310290 [0] NCCL INFO Channel 08/1 : 12[0] -> 0[0] [receive] via NET/IB/0/GDRDMA/Shared
+0: jzxh159:1308399:1310290 [0] NCCL INFO Channel 09/1 : 12[0] -> 0[0] [receive] via NET/IB/0/GDRDMA/Shared
+0: jzxh159:1308399:1310290 [0] NCCL INFO Channel 04/1 : 11[3] -> 0[0] [receive] via NET/IB/0/GDRDMA/Shared
+0: jzxh159:1308399:1310290 [0] NCCL INFO Channel 05/1 : 11[3] -> 0[0] [receive] via NET/IB/0/GDRDMA/Shared
+1: jzxh160:364772:365552 [1] NCCL INFO TUNER/Plugin: Plugin load returned 2 : libnccl-net.so: cannot open shared object file: No such file or directory : when loading libnccl-tuner.so
+1: jzxh160:364772:365552 [1] NCCL INFO TUNER/Plugin: Using internal tuner plugin.
+1: jzxh160:364772:365552 [1] NCCL INFO ncclCommInitRank comm 0x556d3b785610 rank 5 nranks 16 cudaDev 1 nvmlDev 1 busId 2c000 commId 0x524c94fe7e5cd79a - Init COMPLETE
+1: jzxh160:364773:365553 [2] NCCL INFO TUNER/Plugin: Plugin load returned 2 : libnccl-net.so: cannot open shared object file: No such file or directory : when loading libnccl-tuner.so
+1: jzxh160:364774:365551 [3] NCCL INFO TUNER/Plugin: Plugin load returned 2 : libnccl-net.so: cannot open shared object file: No such file or directory : when loading libnccl-tuner.so
+1: jzxh160:364771:365554 [0] NCCL INFO TUNER/Plugin: Plugin load returned 2 : libnccl-net.so: cannot open shared object file: No such file or directory : when loading libnccl-tuner.so
+1: jzxh160:364773:365553 [2] NCCL INFO TUNER/Plugin: Using internal tuner plugin.
+1: jzxh160:364774:365551 [3] NCCL INFO TUNER/Plugin: Using internal tuner plugin.
+1: jzxh160:364771:365554 [0] NCCL INFO TUNER/Plugin: Using internal tuner plugin.
+1: jzxh160:364774:365551 [3] NCCL INFO ncclCommInitRank comm 0x557fffcd2ae0 rank 7 nranks 16 cudaDev 3 nvmlDev 3 busId ad000 commId 0x524c94fe7e5cd79a - Init COMPLETE
+2: jzxh161:3283537:3284337 [1] NCCL INFO TUNER/Plugin: Plugin load returned 2 : libnccl-net.so: cannot open shared object file: No such file or directory : when loading libnccl-tuner.so
+2: jzxh161:3283537:3284337 [1] NCCL INFO TUNER/Plugin: Using internal tuner plugin.
+2: jzxh161:3283537:3284337 [1] NCCL INFO ncclCommInitRank comm 0x55f84f114b10 rank 9 nranks 16 cudaDev 1 nvmlDev 1 busId 2c000 commId 0x524c94fe7e5cd79a - Init COMPLETE
+1: jzxh160:364773:365553 [2] NCCL INFO ncclCommInitRank comm 0x56520ef52d20 rank 6 nranks 16 cudaDev 2 nvmlDev 2 busId 9d000 commId 0x524c94fe7e5cd79a - Init COMPLETE
+1: jzxh160:364771:365554 [0] NCCL INFO ncclCommInitRank comm 0x55a894a256f0 rank 4 nranks 16 cudaDev 0 nvmlDev 0 busId 1b000 commId 0x524c94fe7e5cd79a - Init COMPLETE
+2: jzxh161:3283536:3284336 [0] NCCL INFO TUNER/Plugin: Plugin load returned 2 : libnccl-net.so: cannot open shared object file: No such file or directory : when loading libnccl-tuner.so
+2: jzxh161:3283536:3284336 [0] NCCL INFO TUNER/Plugin: Using internal tuner plugin.
+2: jzxh161:3283536:3284336 [0] NCCL INFO ncclCommInitRank comm 0x55c8d4c4af80 rank 8 nranks 16 cudaDev 0 nvmlDev 0 busId 1b000 commId 0x524c94fe7e5cd79a - Init COMPLETE
+2: jzxh161:3283539:3284338 [3] NCCL INFO TUNER/Plugin: Plugin load returned 2 : libnccl-net.so: cannot open shared object file: No such file or directory : when loading libnccl-tuner.so
+2: jzxh161:3283538:3284335 [2] NCCL INFO TUNER/Plugin: Plugin load returned 2 : libnccl-net.so: cannot open shared object file: No such file or directory : when loading libnccl-tuner.so
+2: jzxh161:3283538:3284335 [2] NCCL INFO TUNER/Plugin: Using internal tuner plugin.
+2: jzxh161:3283539:3284338 [3] NCCL INFO TUNER/Plugin: Using internal tuner plugin.
+2: jzxh161:3283538:3284335 [2] NCCL INFO ncclCommInitRank comm 0x5643a3152d00 rank 10 nranks 16 cudaDev 2 nvmlDev 2 busId 9d000 commId 0x524c94fe7e5cd79a - Init COMPLETE
+2: jzxh161:3283539:3284338 [3] NCCL INFO ncclCommInitRank comm 0x557efcb6a4d0 rank 11 nranks 16 cudaDev 3 nvmlDev 3 busId ad000 commId 0x524c94fe7e5cd79a - Init COMPLETE
+1: jzxh160:364771:365583 [0] NCCL INFO Channel 12/1 : 4[0] -> 0[0] [send] via NET/IB/0/GDRDMA/Shared
+1: jzxh160:364773:365584 [2] NCCL INFO Channel 12/1 : 6[2] -> 0[0] [send] via NET/IB/0(4)/GDRDMA/Shared
+1: jzxh160:364771:365583 [0] NCCL INFO Channel 13/1 : 4[0] -> 0[0] [send] via NET/IB/0/GDRDMA/Shared
+1: jzxh160:364773:365584 [2] NCCL INFO Channel 13/1 : 6[2] -> 0[0] [send] via NET/IB/0(4)/GDRDMA/Shared
+1: jzxh160:364774:365585 [3] NCCL INFO Channel 12/1 : 7[3] -> 0[0] [send] via NET/IB/0(4)/GDRDMA/Shared
+1: jzxh160:364774:365585 [3] NCCL INFO Channel 13/1 : 7[3] -> 0[0] [send] via NET/IB/0(4)/GDRDMA/Shared
+1: jzxh160:364772:365586 [1] NCCL INFO Channel 12/1 : 5[1] -> 0[0] [send] via NET/IB/0(4)/GDRDMA/Shared
+1: jzxh160:364772:365586 [1] NCCL INFO Channel 13/1 : 5[1] -> 0[0] [send] via NET/IB/0(4)/GDRDMA/Shared
+2: jzxh161:3283539:3284370 [3] NCCL INFO Channel 04/1 : 11[3] -> 0[0] [send] via NET/IB/0(8)/GDRDMA/Shared
+2: jzxh161:3283537:3284369 [1] NCCL INFO Channel 04/1 : 9[1] -> 0[0] [send] via NET/IB/0(8)/GDRDMA/Shared
+2: jzxh161:3283539:3284370 [3] NCCL INFO Channel 05/1 : 11[3] -> 0[0] [send] via NET/IB/0(8)/GDRDMA/Shared
+0: jzxh159:1308399:1310290 [0] NCCL INFO Channel 04/1 : 10[2] -> 0[0] [receive] via NET/IB/0/GDRDMA/Shared
+2: jzxh161:3283536:3284368 [0] NCCL INFO Channel 04/1 : 8[0] -> 0[0] [send] via NET/IB/0/GDRDMA/Shared
+0: jzxh159:1308399:1310290 [0] NCCL INFO Channel 05/1 : 10[2] -> 0[0] [receive] via NET/IB/0/GDRDMA/Shared
+2: jzxh161:3283537:3284369 [1] NCCL INFO Channel 05/1 : 9[1] -> 0[0] [send] via NET/IB/0(8)/GDRDMA/Shared
+2: jzxh161:3283536:3284368 [0] NCCL INFO Channel 05/1 : 8[0] -> 0[0] [send] via NET/IB/0/GDRDMA/Shared
+2: jzxh161:3283538:3284367 [2] NCCL INFO Channel 04/1 : 10[2] -> 0[0] [send] via NET/IB/0(8)/GDRDMA/Shared
+2: jzxh161:3283538:3284367 [2] NCCL INFO Channel 05/1 : 10[2] -> 0[0] [send] via NET/IB/0(8)/GDRDMA/Shared
+0: jzxh159:1308399:1310290 [0] NCCL INFO Channel 04/1 : 9[1] -> 0[0] [receive] via NET/IB/0/GDRDMA/Shared
+0: jzxh159:1308399:1310290 [0] NCCL INFO Channel 05/1 : 9[1] -> 0[0] [receive] via NET/IB/0/GDRDMA/Shared
+0: jzxh159:1308399:1310290 [0] NCCL INFO Channel 04/1 : 8[0] -> 0[0] [receive] via NET/IB/0/GDRDMA/Shared
+0: jzxh159:1308399:1310290 [0] NCCL INFO Channel 05/1 : 8[0] -> 0[0] [receive] via NET/IB/0/GDRDMA/Shared
+0: jzxh159:1308399:1310290 [0] NCCL INFO Channel 12/1 : 7[3] -> 0[0] [receive] via NET/IB/0/GDRDMA/Shared
+0: jzxh159:1308399:1310290 [0] NCCL INFO Channel 13/1 : 7[3] -> 0[0] [receive] via NET/IB/0/GDRDMA/Shared
+0: jzxh159:1308399:1310290 [0] NCCL INFO Channel 12/1 : 6[2] -> 0[0] [receive] via NET/IB/0/GDRDMA/Shared
+0: jzxh159:1308399:1310290 [0] NCCL INFO Channel 13/1 : 6[2] -> 0[0] [receive] via NET/IB/0/GDRDMA/Shared
+0: jzxh159:1308399:1310290 [0] NCCL INFO Channel 12/1 : 5[1] -> 0[0] [receive] via NET/IB/0/GDRDMA/Shared
+0: jzxh159:1308399:1310290 [0] NCCL INFO Channel 13/1 : 5[1] -> 0[0] [receive] via NET/IB/0/GDRDMA/Shared
+0: jzxh159:1308399:1310290 [0] NCCL INFO Channel 12/1 : 4[0] -> 0[0] [receive] via NET/IB/0/GDRDMA/Shared
+0: jzxh159:1308399:1310290 [0] NCCL INFO Channel 13/1 : 4[0] -> 0[0] [receive] via NET/IB/0/GDRDMA/Shared
+0: [2025-11-23 14:45:05,799] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:436] [PID:1308399] [RANK:0] gather_len_batches: [39614, 39614, 39614, 39614, 39614, 39614, 39614, 39614, 39614, 39614, 39614, 39614, 39614, 39614, 39614, 39614]
+0: [2025-11-23 14:45:06,098] [INFO] [axolotl.utils.trainer.calc_sample_packing_eff_est:495] [PID:1308399] [RANK:0] sample_packing_eff_est across ranks: [0.895937442779541, 0.895937442779541, 0.895937442779541, 0.895937442779541, 0.895937442779541, 0.895937442779541, 0.895937442779541, 0.895937442779541, 0.895937442779541, 0.895937442779541, 0.895937442779541, 0.895937442779541, 0.895937442779541, 0.895937442779541, 0.895937442779541, 0.895937442779541]
+0: [2025-11-23 14:45:06,115] [INFO] [axolotl.utils.data.sft._prepare_standard_dataset:127] [PID:1308399] [RANK:0] Maximum number of steps set at 1485
+3: Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.
+0: Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.
+1: Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.
+1: Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.
+2: Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.
+3: Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.
+1: Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.
+0: Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.
+2: Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.
+3: Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.
+0: Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.
+3: Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.
+0: Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.
+2: Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.
+2: Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.
+1: Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.
+0: [2025-11-23 14:45:13,869] [INFO] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_evaluation_loop:110] [PID:1308399] [RANK:0] Patched Trainer.evaluation_loop with nanmean loss calculation
+0: [2025-11-23 14:45:13,870] [INFO] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_maybe_log_save_evaluate:164] [PID:1308399] [RANK:0] Patched Trainer._maybe_log_save_evaluate with nanmean loss calculation
+2: 
Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]
Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]
Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]
Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]
Loading checkpoint shards:  50%|█████     | 1/2 [00:18<00:18, 18.55s/it]
Loading checkpoint shards:  50%|█████     | 1/2 [00:18<00:18, 18.55s/it]
Loading checkpoint shards:  50%|█████     | 1/2 [00:18<00:18, 18.55s/it]
Loading checkpoint shards:  50%|█████     | 1/2 [00:18<00:18, 18.55s/it]
Loading checkpoint shards: 100%|██████████| 2/2 [00:26<00:00, 12.31s/it]
Loading checkpoint shards: 100%|██████████| 2/2 [00:26<00:00, 13.24s/it]
+2: 
Loading checkpoint shards: 100%|██████████| 2/2 [00:26<00:00, 12.31s/it]
Loading checkpoint shards: 100%|██████████| 2/2 [00:26<00:00, 13.24s/it]
+1: 
Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]
Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]
Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]
Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]
Loading checkpoint shards:  50%|█████     | 1/2 [00:18<00:18, 18.55s/it]
Loading checkpoint shards:  50%|█████     | 1/2 [00:18<00:18, 18.55s/it]
Loading checkpoint shards:  50%|█████     | 1/2 [00:18<00:18, 18.55s/it]
Loading checkpoint shards:  50%|█████     | 1/2 [00:18<00:18, 18.55s/it]
Loading checkpoint shards: 100%|██████████| 2/2 [00:26<00:00, 12.31s/it]
Loading checkpoint shards: 100%|██████████| 2/2 [00:26<00:00, 12.31s/it]
Loading checkpoint shards: 100%|██████████| 2/2 [00:26<00:00, 13.24s/it]
+1: 
Loading checkpoint shards: 100%|██████████| 2/2 [00:26<00:00, 13.24s/it]
+1: 
Loading checkpoint shards: 100%|██████████| 2/2 [00:26<00:00, 12.31s/it]
Loading checkpoint shards: 100%|██████████| 2/2 [00:26<00:00, 13.24s/it]
+3: 
Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]
Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]
Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]
Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]
Loading checkpoint shards:  50%|█████     | 1/2 [00:18<00:18, 18.55s/it]
Loading checkpoint shards:  50%|█████     | 1/2 [00:18<00:18, 18.55s/it]
Loading checkpoint shards:  50%|█████     | 1/2 [00:18<00:18, 18.55s/it]
Loading checkpoint shards:  50%|█████     | 1/2 [00:18<00:18, 18.55s/it]
Loading checkpoint shards: 100%|██████████| 2/2 [00:26<00:00, 12.31s/it]
Loading checkpoint shards: 100%|██████████| 2/2 [00:26<00:00, 12.31s/it]
Loading checkpoint shards: 100%|██████████| 2/2 [00:26<00:00, 13.24s/it]
+0: 
Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]
Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]
Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]
Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]
Loading checkpoint shards:  50%|█████     | 1/2 [00:18<00:18, 18.55s/it]
Loading checkpoint shards:  50%|█████     | 1/2 [00:18<00:18, 18.55s/it]
Loading checkpoint shards:  50%|█████     | 1/2 [00:18<00:18, 18.55s/it]
Loading checkpoint shards:  50%|█████     | 1/2 [00:18<00:18, 18.62s/it]
Loading checkpoint shards: 100%|██████████| 2/2 [00:26<00:00, 12.31s/it]
Loading checkpoint shards: 100%|██████████| 2/2 [00:26<00:00, 13.25s/it]
+2: 
Loading checkpoint shards: 100%|██████████| 2/2 [00:26<00:00, 12.31s/it]
Loading checkpoint shards: 100%|██████████| 2/2 [00:26<00:00, 12.31s/it]
Loading checkpoint shards: 100%|██████████| 2/2 [00:26<00:00, 13.24s/it]
+2: 
Loading checkpoint shards: 100%|██████████| 2/2 [00:26<00:00, 13.24s/it]
+0: 
Loading checkpoint shards: 100%|██████████| 2/2 [00:26<00:00, 12.31s/it]
Loading checkpoint shards: 100%|██████████| 2/2 [00:26<00:00, 13.25s/it]
+3: 
Loading checkpoint shards: 100%|██████████| 2/2 [00:26<00:00, 12.31s/it]
Loading checkpoint shards: 100%|██████████| 2/2 [00:26<00:00, 13.24s/it]
+1: 
Loading checkpoint shards: 100%|██████████| 2/2 [00:26<00:00, 12.31s/it]
Loading checkpoint shards: 100%|██████████| 2/2 [00:26<00:00, 13.24s/it]
+3: 
Loading checkpoint shards: 100%|██████████| 2/2 [00:26<00:00, 13.24s/it]
+3: 
Loading checkpoint shards: 100%|██████████| 2/2 [00:26<00:00, 12.31s/it]
Loading checkpoint shards: 100%|██████████| 2/2 [00:26<00:00, 13.24s/it]
+0: 
Loading checkpoint shards: 100%|██████████| 2/2 [00:26<00:00, 12.31s/it]
Loading checkpoint shards: 100%|██████████| 2/2 [00:26<00:00, 13.25s/it]
+0: 
Loading checkpoint shards: 100%|██████████| 2/2 [00:26<00:00, 12.32s/it]
Loading checkpoint shards: 100%|██████████| 2/2 [00:26<00:00, 13.27s/it]
+0: [2025-11-23 14:45:46,236] [INFO] [axolotl.loaders.model._configure_embedding_dtypes:345] [PID:1308399] [RANK:0] Converting modules to torch.bfloat16
+0: [2025-11-23 14:46:47,316] [INFO] [axolotl.train.save_initial_configs:416] [PID:1308399] [RANK:0] Pre-saving tokenizer to /lustre/fswork/projects/rech/dgo/udv55np/ift/Nemotron-Super-49B-v1_5/gemma-3-4b/0.25...
+0: [2025-11-23 14:46:48,218] [INFO] [axolotl.train.save_initial_configs:419] [PID:1308399] [RANK:0] Pre-saving model config to /lustre/fswork/projects/rech/dgo/udv55np/ift/Nemotron-Super-49B-v1_5/gemma-3-4b/0.25...
+0: [2025-11-23 14:46:48,227] [INFO] [axolotl.train.save_initial_configs:423] [PID:1308399] [RANK:0] Pre-saving processor to /lustre/fswork/projects/rech/dgo/udv55np/ift/Nemotron-Super-49B-v1_5/gemma-3-4b/0.25...
+0: [2025-11-23 14:46:50,935] [INFO] [axolotl.train.execute_training:203] [PID:1308399] [RANK:0] Starting trainer...
+0: [2025-11-23 14:49:38,003] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:436] [PID:1308399] [RANK:0] gather_len_batches: [39614, 39614, 39614, 39614, 39614, 39614, 39614, 39614, 39614, 39614, 39614, 39614, 39614, 39614, 39614, 39614]
+0: Parameter Offload - Persistent parameters statistics: param_count = 479, numel = 768880
+0: jzxh159:1308399:1311116 [0] NCCL INFO Comm config Blocking set to 1
+0: jzxh159:1308401:1311125 [2] NCCL INFO Comm config Blocking set to 1
+0: jzxh159:1308402:1311129 [3] NCCL INFO Comm config Blocking set to 1
+0: jzxh159:1308399:1311133 [0] NCCL INFO Using non-device net plugin version 0
+0: jzxh159:1308399:1311133 [0] NCCL INFO Using network IB
+2: jzxh161:3283539:3285116 [3] NCCL INFO Comm config Blocking set to 1
+0: jzxh159:1308400:1311123 [1] NCCL INFO Comm config Blocking set to 1
+0: jzxh159:1308402:1311135 [3] NCCL INFO Using non-device net plugin version 0
+0: jzxh159:1308401:1311134 [2] NCCL INFO Using non-device net plugin version 0
+0: jzxh159:1308402:1311135 [3] NCCL INFO Using network IB
+0: jzxh159:1308401:1311134 [2] NCCL INFO Using network IB
+3: jzxh162:3160745:3162421 [3] NCCL INFO Comm config Blocking set to 1
+3: jzxh162:3160743:3162413 [1] NCCL INFO Comm config Blocking set to 1
+3: jzxh162:3160744:3162423 [2] NCCL INFO Comm config Blocking set to 1
+2: jzxh161:3283538:3285113 [2] NCCL INFO Comm config Blocking set to 1
+3: jzxh162:3160742:3162411 [0] NCCL INFO Comm config Blocking set to 1
+0: jzxh159:1308400:1311136 [1] NCCL INFO Using non-device net plugin version 0
+0: jzxh159:1308400:1311136 [1] NCCL INFO Using network IB
+2: jzxh161:3283539:3285120 [3] NCCL INFO Using non-device net plugin version 0
+2: jzxh161:3283539:3285120 [3] NCCL INFO Using network IB
+3: jzxh162:3160743:3162426 [1] NCCL INFO Using non-device net plugin version 0
+3: jzxh162:3160743:3162426 [1] NCCL INFO Using network IB
+3: jzxh162:3160745:3162425 [3] NCCL INFO Using non-device net plugin version 0
+3: jzxh162:3160745:3162425 [3] NCCL INFO Using network IB
+3: jzxh162:3160742:3162428 [0] NCCL INFO Using non-device net plugin version 0
+3: jzxh162:3160742:3162428 [0] NCCL INFO Using network IB
+3: jzxh162:3160744:3162427 [2] NCCL INFO Using non-device net plugin version 0
+3: jzxh162:3160744:3162427 [2] NCCL INFO Using network IB
+2: jzxh161:3283536:3285105 [0] NCCL INFO Comm config Blocking set to 1
+2: jzxh161:3283538:3285121 [2] NCCL INFO Using non-device net plugin version 0
+2: jzxh161:3283538:3285121 [2] NCCL INFO Using network IB
+2: jzxh161:3283537:3285108 [1] NCCL INFO Comm config Blocking set to 1
+2: jzxh161:3283537:3285123 [1] NCCL INFO Using non-device net plugin version 0
+2: jzxh161:3283537:3285123 [1] NCCL INFO Using network IB
+2: jzxh161:3283536:3285122 [0] NCCL INFO Using non-device net plugin version 0
+2: jzxh161:3283536:3285122 [0] NCCL INFO Using network IB
+1: jzxh160:364771:366425 [0] NCCL INFO Comm config Blocking set to 1
+1: jzxh160:364773:366432 [2] NCCL INFO Comm config Blocking set to 1
+1: jzxh160:364774:366436 [3] NCCL INFO Comm config Blocking set to 1
+1: jzxh160:364772:366427 [1] NCCL INFO Comm config Blocking set to 1
+1: jzxh160:364773:366439 [2] NCCL INFO Using non-device net plugin version 0
+1: jzxh160:364773:366439 [2] NCCL INFO Using network IB
+1: jzxh160:364774:366440 [3] NCCL INFO Using non-device net plugin version 0
+1: jzxh160:364774:366440 [3] NCCL INFO Using network IB
+1: jzxh160:364772:366441 [1] NCCL INFO Using non-device net plugin version 0
+1: jzxh160:364772:366441 [1] NCCL INFO Using network IB
+1: jzxh160:364771:366438 [0] NCCL INFO Using non-device net plugin version 0
+1: jzxh160:364771:366438 [0] NCCL INFO Using network IB
+2: jzxh161:3283538:3285121 [2] NCCL INFO ncclCommInitRank comm 0x14a1d01231c0 rank 10 nranks 16 cudaDev 2 nvmlDev 2 busId 9d000 commId 0x2b939d813fa1b763 - Init START
+2: jzxh161:3283539:3285120 [3] NCCL INFO ncclCommInitRank comm 0x14b15c115410 rank 11 nranks 16 cudaDev 3 nvmlDev 3 busId ad000 commId 0x2b939d813fa1b763 - Init START
+2: jzxh161:3283537:3285123 [1] NCCL INFO ncclCommInitRank comm 0x147064122c40 rank 9 nranks 16 cudaDev 1 nvmlDev 1 busId 2c000 commId 0x2b939d813fa1b763 - Init START
+2: jzxh161:3283536:3285122 [0] NCCL INFO ncclCommInitRank comm 0x14c064143000 rank 8 nranks 16 cudaDev 0 nvmlDev 0 busId 1b000 commId 0x2b939d813fa1b763 - Init START
+3: jzxh162:3160743:3162426 [1] NCCL INFO ncclCommInitRank comm 0x1481bc125340 rank 13 nranks 16 cudaDev 1 nvmlDev 1 busId 2c000 commId 0x2b939d813fa1b763 - Init START
+3: jzxh162:3160744:3162427 [2] NCCL INFO ncclCommInitRank comm 0x151768115ac0 rank 14 nranks 16 cudaDev 2 nvmlDev 2 busId 9d000 commId 0x2b939d813fa1b763 - Init START
+3: jzxh162:3160745:3162425 [3] NCCL INFO ncclCommInitRank comm 0x152d14114450 rank 15 nranks 16 cudaDev 3 nvmlDev 3 busId ad000 commId 0x2b939d813fa1b763 - Init START
+3: jzxh162:3160742:3162428 [0] NCCL INFO ncclCommInitRank comm 0x14c174126240 rank 12 nranks 16 cudaDev 0 nvmlDev 0 busId 1b000 commId 0x2b939d813fa1b763 - Init START
+1: jzxh160:364774:366440 [3] NCCL INFO ncclCommInitRank comm 0x1457001250c0 rank 7 nranks 16 cudaDev 3 nvmlDev 3 busId ad000 commId 0x2b939d813fa1b763 - Init START
+0: jzxh159:1308402:1311135 [3] NCCL INFO ncclCommInitRank comm 0x154364116f80 rank 3 nranks 16 cudaDev 3 nvmlDev 3 busId ad000 commId 0x2b939d813fa1b763 - Init START
+0: jzxh159:1308399:1311133 [0] NCCL INFO ncclCommInitRank comm 0x151f9411a7c0 rank 0 nranks 16 cudaDev 0 nvmlDev 0 busId 1b000 commId 0x2b939d813fa1b763 - Init START
+0: jzxh159:1308400:1311136 [1] NCCL INFO ncclCommInitRank comm 0x14c618124680 rank 1 nranks 16 cudaDev 1 nvmlDev 1 busId 2c000 commId 0x2b939d813fa1b763 - Init START
+0: jzxh159:1308401:1311134 [2] NCCL INFO ncclCommInitRank comm 0x1463c8123fc0 rank 2 nranks 16 cudaDev 2 nvmlDev 2 busId 9d000 commId 0x2b939d813fa1b763 - Init START
+1: jzxh160:364772:366441 [1] NCCL INFO ncclCommInitRank comm 0x144548117000 rank 5 nranks 16 cudaDev 1 nvmlDev 1 busId 2c000 commId 0x2b939d813fa1b763 - Init START
+1: jzxh160:364773:366439 [2] NCCL INFO ncclCommInitRank comm 0x149098133e00 rank 6 nranks 16 cudaDev 2 nvmlDev 2 busId 9d000 commId 0x2b939d813fa1b763 - Init START
+1: jzxh160:364771:366438 [0] NCCL INFO ncclCommInitRank comm 0x15301c1476c0 rank 4 nranks 16 cudaDev 0 nvmlDev 0 busId 1b000 commId 0x2b939d813fa1b763 - Init START
+0: jzxh159:1308400:1311136 [1] NCCL INFO Setting affinity for GPU 1 to ffff,ff000000,00000000,0000ffff,ff000000
+0: jzxh159:1308400:1311136 [1] NCCL INFO NVLS multicast support is not available on dev 1
+1: jzxh160:364774:366440 [3] NCCL INFO Setting affinity for GPU 3 to ffffff00,00000000,00000000,ffffff00,00000000,00000000
+1: jzxh160:364774:366440 [3] NCCL INFO NVLS multicast support is not available on dev 3
+1: jzxh160:364772:366441 [1] NCCL INFO Setting affinity for GPU 1 to ffff,ff000000,00000000,0000ffff,ff000000
+1: jzxh160:364772:366441 [1] NCCL INFO NVLS multicast support is not available on dev 1
+2: jzxh161:3283537:3285123 [1] NCCL INFO Setting affinity for GPU 1 to ffff,ff000000,00000000,0000ffff,ff000000
+2: jzxh161:3283537:3285123 [1] NCCL INFO NVLS multicast support is not available on dev 1
+0: jzxh159:1308399:1311133 [0] NCCL INFO Setting affinity for GPU 0 to ffffff,00000000,00000000,00ffffff
+0: jzxh159:1308399:1311133 [0] NCCL INFO NVLS multicast support is not available on dev 0
+1: jzxh160:364773:366439 [2] NCCL INFO Setting affinity for GPU 2 to ff,ffff0000,00000000,000000ff,ffff0000,00000000
+1: jzxh160:364773:366439 [2] NCCL INFO NVLS multicast support is not available on dev 2
+0: jzxh159:1308402:1311135 [3] NCCL INFO Setting affinity for GPU 3 to ffffff00,00000000,00000000,ffffff00,00000000,00000000
+0: jzxh159:1308402:1311135 [3] NCCL INFO NVLS multicast support is not available on dev 3
+1: jzxh160:364771:366438 [0] NCCL INFO Setting affinity for GPU 0 to ffffff,00000000,00000000,00ffffff
+1: jzxh160:364771:366438 [0] NCCL INFO NVLS multicast support is not available on dev 0
+0: jzxh159:1308401:1311134 [2] NCCL INFO Setting affinity for GPU 2 to ff,ffff0000,00000000,000000ff,ffff0000,00000000
+0: jzxh159:1308401:1311134 [2] NCCL INFO NVLS multicast support is not available on dev 2
+2: jzxh161:3283538:3285121 [2] NCCL INFO Setting affinity for GPU 2 to ff,ffff0000,00000000,000000ff,ffff0000,00000000
+2: jzxh161:3283538:3285121 [2] NCCL INFO NVLS multicast support is not available on dev 2
+3: jzxh162:3160742:3162428 [0] NCCL INFO Setting affinity for GPU 0 to ffffff,00000000,00000000,00ffffff
+3: jzxh162:3160742:3162428 [0] NCCL INFO NVLS multicast support is not available on dev 0
+2: jzxh161:3283536:3285122 [0] NCCL INFO Setting affinity for GPU 0 to ffffff,00000000,00000000,00ffffff
+2: jzxh161:3283536:3285122 [0] NCCL INFO NVLS multicast support is not available on dev 0
+2: jzxh161:3283539:3285120 [3] NCCL INFO Setting affinity for GPU 3 to ffffff00,00000000,00000000,ffffff00,00000000,00000000
+2: jzxh161:3283539:3285120 [3] NCCL INFO NVLS multicast support is not available on dev 3
+3: jzxh162:3160743:3162426 [1] NCCL INFO Setting affinity for GPU 1 to ffff,ff000000,00000000,0000ffff,ff000000
+3: jzxh162:3160743:3162426 [1] NCCL INFO NVLS multicast support is not available on dev 1
+3: jzxh162:3160745:3162425 [3] NCCL INFO Setting affinity for GPU 3 to ffffff00,00000000,00000000,ffffff00,00000000,00000000
+3: jzxh162:3160745:3162425 [3] NCCL INFO NVLS multicast support is not available on dev 3
+3: jzxh162:3160744:3162427 [2] NCCL INFO Setting affinity for GPU 2 to ff,ffff0000,00000000,000000ff,ffff0000,00000000
+3: jzxh162:3160744:3162427 [2] NCCL INFO NVLS multicast support is not available on dev 2
+0: jzxh159:1308402:1311135 [3] NCCL INFO comm 0x154364116f80 rank 3 nRanks 16 nNodes 4 localRanks 4 localRank 3 MNNVL 0
+0: jzxh159:1308401:1311134 [2] NCCL INFO comm 0x1463c8123fc0 rank 2 nRanks 16 nNodes 4 localRanks 4 localRank 2 MNNVL 0
+0: jzxh159:1308400:1311136 [1] NCCL INFO comm 0x14c618124680 rank 1 nRanks 16 nNodes 4 localRanks 4 localRank 1 MNNVL 0
+0: jzxh159:1308399:1311133 [0] NCCL INFO comm 0x151f9411a7c0 rank 0 nRanks 16 nNodes 4 localRanks 4 localRank 0 MNNVL 0
+0: jzxh159:1308399:1311133 [0] NCCL INFO Channel 00/16 :    0   1   2   3   4   5   6   7   8   9  10  11  12  13  14  15
+0: jzxh159:1308402:1311135 [3] NCCL INFO Trees [0] -1/-1/-1->3->2 [1] 0/-1/-1->3->2 [2] -1/-1/-1->3->1 [3] 1/11/-1->3->-1 [4] 2/-1/-1->3->0 [5] 0/-1/-1->3->1 [6] 1/-1/-1->3->0 [7] 2/11/-1->3->-1 [8] -1/-1/-1->3->2 [9] 0/-1/-1->3->2 [10] -1/-1/-1->3->1 [11] 1/-1/-1->3->7 [12] 2/-1/-1->3->0 [13] 0/-1/-1->3->1 [14] 1/-1/-1->3->0 [15] 2/-1/-1->3->7
+0: jzxh159:1308401:1311134 [2] NCCL INFO Trees [0] 3/-1/-1->2->1 [1] 3/-1/-1->2->1 [2] 0/10/-1->2->-1 [3] -1/-1/-1->2->0 [4] 1/-1/-1->2->3 [5] -1/-1/-1->2->0 [6] 0/10/-1->2->-1 [7] 1/-1/-1->2->3 [8] 3/-1/-1->2->1 [9] 3/-1/-1->2->1 [10] 0/-1/-1->2->6 [11] -1/-1/-1->2->0 [12] 1/-1/-1->2->3 [13] -1/-1/-1->2->0 [14] 0/-1/-1->2->6 [15] 1/-1/-1->2->3
+0: jzxh159:1308402:1311135 [3] NCCL INFO P2P Chunksize set to 131072
+0: jzxh159:1308399:1311133 [0] NCCL INFO Channel 01/16 :    0   3   2   5   4   7   6   9   8  11  10  13  12  15  14   1
+0: jzxh159:1308401:1311134 [2] NCCL INFO P2P Chunksize set to 131072
+0: jzxh159:1308400:1311136 [1] NCCL INFO Trees [0] 2/-1/-1->1->0 [1] 2/9/-1->1->-1 [2] 3/-1/-1->1->0 [3] 0/-1/-1->1->3 [4] -1/-1/-1->1->2 [5] 3/9/-1->1->-1 [6] -1/-1/-1->1->3 [7] 0/-1/-1->1->2 [8] 2/-1/-1->1->0 [9] 2/-1/-1->1->5 [10] 3/-1/-1->1->0 [11] 0/-1/-1->1->3 [12] -1/-1/-1->1->2 [13] 3/-1/-1->1->5 [14] -1/-1/-1->1->3 [15] 0/-1/-1->1->2
+0: jzxh159:1308400:1311136 [1] NCCL INFO P2P Chunksize set to 131072
+0: jzxh159:1308399:1311133 [0] NCCL INFO Channel 02/16 :    0   3   6   5   4   7  10   9   8  11  14  13  12  15   2   1
+0: jzxh159:1308399:1311133 [0] NCCL INFO Channel 03/16 :    0   1   2   7   4   5   6  11   8   9  10  15  12  13  14   3
+0: jzxh159:1308399:1311133 [0] NCCL INFO Channel 04/16 :    0   1   2   3   4   5   6   7   8   9  10  11  12  13  14  15
+0: jzxh159:1308399:1311133 [0] NCCL INFO Channel 05/16 :    0   3   2   5   4   7   6   9   8  11  10  13  12  15  14   1
+0: jzxh159:1308399:1311133 [0] NCCL INFO Channel 06/16 :    0   3   6   5   4   7  10   9   8  11  14  13  12  15   2   1
+0: jzxh159:1308399:1311133 [0] NCCL INFO Channel 07/16 :    0   1   2   7   4   5   6  11   8   9  10  15  12  13  14   3
+0: jzxh159:1308399:1311133 [0] NCCL INFO Channel 08/16 :    0   1   2   3   4   5   6   7   8   9  10  11  12  13  14  15
+0: jzxh159:1308399:1311133 [0] NCCL INFO Channel 09/16 :    0   3   2   5   4   7   6   9   8  11  10  13  12  15  14   1
+0: jzxh159:1308399:1311133 [0] NCCL INFO Channel 10/16 :    0   3   6   5   4   7  10   9   8  11  14  13  12  15   2   1
+0: jzxh159:1308399:1311133 [0] NCCL INFO Channel 11/16 :    0   1   2   7   4   5   6  11   8   9  10  15  12  13  14   3
+0: jzxh159:1308399:1311133 [0] NCCL INFO Channel 12/16 :    0   1   2   3   4   5   6   7   8   9  10  11  12  13  14  15
+0: jzxh159:1308399:1311133 [0] NCCL INFO Channel 13/16 :    0   3   2   5   4   7   6   9   8  11  10  13  12  15  14   1
+0: jzxh159:1308399:1311133 [0] NCCL INFO Channel 14/16 :    0   3   6   5   4   7  10   9   8  11  14  13  12  15   2   1
+0: jzxh159:1308399:1311133 [0] NCCL INFO Channel 15/16 :    0   1   2   7   4   5   6  11   8   9  10  15  12  13  14   3
+0: jzxh159:1308399:1311133 [0] NCCL INFO Trees [0] 1/8/-1->0->-1 [1] -1/-1/-1->0->3 [2] 1/-1/-1->0->2 [3] 2/-1/-1->0->1 [4] 3/8/-1->0->-1 [5] 2/-1/-1->0->3 [6] 3/-1/-1->0->2 [7] -1/-1/-1->0->1 [8] 1/-1/-1->0->4 [9] -1/-1/-1->0->3 [10] 1/-1/-1->0->2 [11] 2/-1/-1->0->1 [12] 3/-1/-1->0->4 [13] 2/-1/-1->0->3 [14] 3/-1/-1->0->2 [15] -1/-1/-1->0->1
+0: jzxh159:1308399:1311133 [0] NCCL INFO P2P Chunksize set to 131072
+1: jzxh160:364774:366440 [3] NCCL INFO comm 0x1457001250c0 rank 7 nRanks 16 nNodes 4 localRanks 4 localRank 3 MNNVL 0
+1: jzxh160:364773:366439 [2] NCCL INFO comm 0x149098133e00 rank 6 nRanks 16 nNodes 4 localRanks 4 localRank 2 MNNVL 0
+1: jzxh160:364772:366441 [1] NCCL INFO comm 0x144548117000 rank 5 nRanks 16 nNodes 4 localRanks 4 localRank 1 MNNVL 0
+1: jzxh160:364771:366438 [0] NCCL INFO comm 0x15301c1476c0 rank 4 nRanks 16 nNodes 4 localRanks 4 localRank 0 MNNVL 0
+3: jzxh162:3160745:3162425 [3] NCCL INFO comm 0x152d14114450 rank 15 nRanks 16 nNodes 4 localRanks 4 localRank 3 MNNVL 0
+3: jzxh162:3160744:3162427 [2] NCCL INFO comm 0x151768115ac0 rank 14 nRanks 16 nNodes 4 localRanks 4 localRank 2 MNNVL 0
+1: jzxh160:364773:366439 [2] NCCL INFO Trees [0] 7/-1/-1->6->5 [1] 7/-1/-1->6->5 [2] 4/-1/-1->6->10 [3] -1/-1/-1->6->4 [4] 5/-1/-1->6->7 [5] -1/-1/-1->6->4 [6] 4/-1/-1->6->10 [7] 5/-1/-1->6->7 [8] 7/-1/-1->6->5 [9] 7/-1/-1->6->5 [10] 4/10/2->6->14 [11] -1/-1/-1->6->4 [12] 5/-1/-1->6->7 [13] -1/-1/-1->6->4 [14] 4/10/2->6->14 [15] 5/-1/-1->6->7
+1: jzxh160:364774:366440 [3] NCCL INFO Trees [0] -1/-1/-1->7->6 [1] 4/-1/-1->7->6 [2] -1/-1/-1->7->5 [3] 5/-1/-1->7->11 [4] 6/-1/-1->7->4 [5] 4/-1/-1->7->5 [6] 5/-1/-1->7->4 [7] 6/-1/-1->7->11 [8] -1/-1/-1->7->6 [9] 4/-1/-1->7->6 [10] -1/-1/-1->7->5 [11] 5/11/3->7->15 [12] 6/-1/-1->7->4 [13] 4/-1/-1->7->5 [14] 5/-1/-1->7->4 [15] 6/11/3->7->15
+1: jzxh160:364773:366439 [2] NCCL INFO P2P Chunksize set to 131072
+1: jzxh160:364774:366440 [3] NCCL INFO P2P Chunksize set to 131072
+2: jzxh161:3283536:3285122 [0] NCCL INFO comm 0x14c064143000 rank 8 nRanks 16 nNodes 4 localRanks 4 localRank 0 MNNVL 0
+3: jzxh162:3160745:3162425 [3] NCCL INFO Trees [0] -1/-1/-1->15->14 [1] 12/-1/-1->15->14 [2] -1/-1/-1->15->13 [3] 13/-1/-1->15->11 [4] 14/-1/-1->15->12 [5] 12/-1/-1->15->13 [6] 13/-1/-1->15->12 [7] 14/-1/-1->15->11 [8] -1/-1/-1->15->14 [9] 12/-1/-1->15->14 [10] -1/-1/-1->15->13 [11] 13/7/-1->15->-1 [12] 14/-1/-1->15->12 [13] 12/-1/-1->15->13 [14] 13/-1/-1->15->12 [15] 14/7/-1->15->-1
+3: jzxh162:3160745:3162425 [3] NCCL INFO P2P Chunksize set to 131072
+3: jzxh162:3160744:3162427 [2] NCCL INFO Trees [0] 15/-1/-1->14->13 [1] 15/-1/-1->14->13 [2] 12/-1/-1->14->10 [3] -1/-1/-1->14->12 [4] 13/-1/-1->14->15 [5] -1/-1/-1->14->12 [6] 12/-1/-1->14->10 [7] 13/-1/-1->14->15 [8] 15/-1/-1->14->13 [9] 15/-1/-1->14->13 [10] 12/6/-1->14->-1 [11] -1/-1/-1->14->12 [12] 13/-1/-1->14->15 [13] -1/-1/-1->14->12 [14] 12/6/-1->14->-1 [15] 13/-1/-1->14->15
+3: jzxh162:3160744:3162427 [2] NCCL INFO P2P Chunksize set to 131072
+2: jzxh161:3283539:3285120 [3] NCCL INFO comm 0x14b15c115410 rank 11 nRanks 16 nNodes 4 localRanks 4 localRank 3 MNNVL 0
+2: jzxh161:3283537:3285123 [1] NCCL INFO comm 0x147064122c40 rank 9 nRanks 16 nNodes 4 localRanks 4 localRank 1 MNNVL 0
+2: jzxh161:3283538:3285121 [2] NCCL INFO comm 0x14a1d01231c0 rank 10 nRanks 16 nNodes 4 localRanks 4 localRank 2 MNNVL 0
+2: jzxh161:3283536:3285122 [0] NCCL INFO Trees [0] 9/4/12->8->0 [1] -1/-1/-1->8->11 [2] 9/-1/-1->8->10 [3] 10/-1/-1->8->9 [4] 11/4/12->8->0 [5] 10/-1/-1->8->11 [6] 11/-1/-1->8->10 [7] -1/-1/-1->8->9 [8] 9/-1/-1->8->4 [9] -1/-1/-1->8->11 [10] 9/-1/-1->8->10 [11] 10/-1/-1->8->9 [12] 11/-1/-1->8->4 [13] 10/-1/-1->8->11 [14] 11/-1/-1->8->10 [15] -1/-1/-1->8->9
+2: jzxh161:3283536:3285122 [0] NCCL INFO P2P Chunksize set to 131072
+3: jzxh162:3160743:3162426 [1] NCCL INFO comm 0x1481bc125340 rank 13 nRanks 16 nNodes 4 localRanks 4 localRank 1 MNNVL 0
+3: jzxh162:3160742:3162428 [0] NCCL INFO comm 0x14c174126240 rank 12 nRanks 16 nNodes 4 localRanks 4 localRank 0 MNNVL 0
+3: jzxh162:3160743:3162426 [1] NCCL INFO Trees [0] 14/-1/-1->13->12 [1] 14/-1/-1->13->9 [2] 15/-1/-1->13->12 [3] 12/-1/-1->13->15 [4] -1/-1/-1->13->14 [5] 15/-1/-1->13->9 [6] -1/-1/-1->13->15 [7] 12/-1/-1->13->14 [8] 14/-1/-1->13->12 [9] 14/5/-1->13->-1 [10] 15/-1/-1->13->12 [11] 12/-1/-1->13->15 [12] -1/-1/-1->13->14 [13] 15/5/-1->13->-1 [14] -1/-1/-1->13->15 [15] 12/-1/-1->13->14
+3: jzxh162:3160743:3162426 [1] NCCL INFO P2P Chunksize set to 131072
+3: jzxh162:3160742:3162428 [0] NCCL INFO Trees [0] 13/-1/-1->12->8 [1] -1/-1/-1->12->15 [2] 13/-1/-1->12->14 [3] 14/-1/-1->12->13 [4] 15/-1/-1->12->8 [5] 14/-1/-1->12->15 [6] 15/-1/-1->12->14 [7] -1/-1/-1->12->13 [8] 13/4/-1->12->-1 [9] -1/-1/-1->12->15 [10] 13/-1/-1->12->14 [11] 14/-1/-1->12->13 [12] 15/4/-1->12->-1 [13] 14/-1/-1->12->15 [14] 15/-1/-1->12->14 [15] -1/-1/-1->12->13
+3: jzxh162:3160742:3162428 [0] NCCL INFO P2P Chunksize set to 131072
+2: jzxh161:3283539:3285120 [3] NCCL INFO Trees [0] -1/-1/-1->11->10 [1] 8/-1/-1->11->10 [2] -1/-1/-1->11->9 [3] 9/7/15->11->3 [4] 10/-1/-1->11->8 [5] 8/-1/-1->11->9 [6] 9/-1/-1->11->8 [7] 10/7/15->11->3 [8] -1/-1/-1->11->10 [9] 8/-1/-1->11->10 [10] -1/-1/-1->11->9 [11] 9/-1/-1->11->7 [12] 10/-1/-1->11->8 [13] 8/-1/-1->11->9 [14] 9/-1/-1->11->8 [15] 10/-1/-1->11->7
+2: jzxh161:3283537:3285123 [1] NCCL INFO Trees [0] 10/-1/-1->9->8 [1] 10/5/13->9->1 [2] 11/-1/-1->9->8 [3] 8/-1/-1->9->11 [4] -1/-1/-1->9->10 [5] 11/5/13->9->1 [6] -1/-1/-1->9->11 [7] 8/-1/-1->9->10 [8] 10/-1/-1->9->8 [9] 10/-1/-1->9->5 [10] 11/-1/-1->9->8 [11] 8/-1/-1->9->11 [12] -1/-1/-1->9->10 [13] 11/-1/-1->9->5 [14] -1/-1/-1->9->11 [15] 8/-1/-1->9->10
+2: jzxh161:3283539:3285120 [3] NCCL INFO P2P Chunksize set to 131072
+2: jzxh161:3283537:3285123 [1] NCCL INFO P2P Chunksize set to 131072
+2: jzxh161:3283538:3285121 [2] NCCL INFO Trees [0] 11/-1/-1->10->9 [1] 11/-1/-1->10->9 [2] 8/6/14->10->2 [3] -1/-1/-1->10->8 [4] 9/-1/-1->10->11 [5] -1/-1/-1->10->8 [6] 8/6/14->10->2 [7] 9/-1/-1->10->11 [8] 11/-1/-1->10->9 [9] 11/-1/-1->10->9 [10] 8/-1/-1->10->6 [11] -1/-1/-1->10->8 [12] 9/-1/-1->10->11 [13] -1/-1/-1->10->8 [14] 8/-1/-1->10->6 [15] 9/-1/-1->10->11
+2: jzxh161:3283538:3285121 [2] NCCL INFO P2P Chunksize set to 131072
+1: jzxh160:364772:366441 [1] NCCL INFO Trees [0] 6/-1/-1->5->4 [1] 6/-1/-1->5->9 [2] 7/-1/-1->5->4 [3] 4/-1/-1->5->7 [4] -1/-1/-1->5->6 [5] 7/-1/-1->5->9 [6] -1/-1/-1->5->7 [7] 4/-1/-1->5->6 [8] 6/-1/-1->5->4 [9] 6/9/1->5->13 [10] 7/-1/-1->5->4 [11] 4/-1/-1->5->7 [12] -1/-1/-1->5->6 [13] 7/9/1->5->13 [14] -1/-1/-1->5->7 [15] 4/-1/-1->5->6
+1: jzxh160:364772:366441 [1] NCCL INFO P2P Chunksize set to 131072
+1: jzxh160:364771:366438 [0] NCCL INFO Trees [0] 5/-1/-1->4->8 [1] -1/-1/-1->4->7 [2] 5/-1/-1->4->6 [3] 6/-1/-1->4->5 [4] 7/-1/-1->4->8 [5] 6/-1/-1->4->7 [6] 7/-1/-1->4->6 [7] -1/-1/-1->4->5 [8] 5/8/0->4->12 [9] -1/-1/-1->4->7 [10] 5/-1/-1->4->6 [11] 6/-1/-1->4->5 [12] 7/8/0->4->12 [13] 6/-1/-1->4->7 [14] 7/-1/-1->4->6 [15] -1/-1/-1->4->5
+1: jzxh160:364771:366438 [0] NCCL INFO P2P Chunksize set to 131072
+2: jzxh161:3283537:3285123 [1] NCCL INFO Channel 00/0 : 9[1] -> 10[2] via P2P/CUMEM
+0: jzxh159:1308401:1311134 [2] NCCL INFO Channel 00/0 : 2[2] -> 3[3] via P2P/CUMEM
+0: jzxh159:1308401:1311134 [2] NCCL INFO Channel 04/0 : 2[2] -> 3[3] via P2P/CUMEM
+0: jzxh159:1308401:1311134 [2] NCCL INFO Channel 08/0 : 2[2] -> 3[3] via P2P/CUMEM
+1: jzxh160:364772:366441 [1] NCCL INFO Channel 00/0 : 5[1] -> 6[2] via P2P/CUMEM
+2: jzxh161:3283537:3285123 [1] NCCL INFO Channel 03/0 : 9[1] -> 10[2] via P2P/CUMEM
+3: jzxh162:3160744:3162427 [2] NCCL INFO Channel 00/0 : 14[2] -> 15[3] via P2P/CUMEM
+0: jzxh159:1308401:1311134 [2] NCCL INFO Channel 12/0 : 2[2] -> 3[3] via P2P/CUMEM
+1: jzxh160:364772:366441 [1] NCCL INFO Channel 03/0 : 5[1] -> 6[2] via P2P/CUMEM
+2: jzxh161:3283537:3285123 [1] NCCL INFO Channel 04/0 : 9[1] -> 10[2] via P2P/CUMEM
+3: jzxh162:3160744:3162427 [2] NCCL INFO Channel 04/0 : 14[2] -> 15[3] via P2P/CUMEM
+0: jzxh159:1308400:1311136 [1] NCCL INFO Channel 00/0 : 1[1] -> 2[2] via P2P/CUMEM
+1: jzxh160:364772:366441 [1] NCCL INFO Channel 04/0 : 5[1] -> 6[2] via P2P/CUMEM
+2: jzxh161:3283537:3285123 [1] NCCL INFO Channel 07/0 : 9[1] -> 10[2] via P2P/CUMEM
+0: jzxh159:1308400:1311136 [1] NCCL INFO Channel 03/0 : 1[1] -> 2[2] via P2P/CUMEM
+1: jzxh160:364772:366441 [1] NCCL INFO Channel 07/0 : 5[1] -> 6[2] via P2P/CUMEM
+2: jzxh161:3283537:3285123 [1] NCCL INFO Channel 08/0 : 9[1] -> 10[2] via P2P/CUMEM
+2: jzxh161:3283537:3285123 [1] NCCL INFO Channel 11/0 : 9[1] -> 10[2] via P2P/CUMEM
+2: jzxh161:3283537:3285123 [1] NCCL INFO Channel 12/0 : 9[1] -> 10[2] via P2P/CUMEM
+0: jzxh159:1308400:1311136 [1] NCCL INFO Channel 04/0 : 1[1] -> 2[2] via P2P/CUMEM
+2: jzxh161:3283537:3285123 [1] NCCL INFO Channel 15/0 : 9[1] -> 10[2] via P2P/CUMEM
+3: jzxh162:3160744:3162427 [2] NCCL INFO Channel 08/0 : 14[2] -> 15[3] via P2P/CUMEM
+1: jzxh160:364772:366441 [1] NCCL INFO Channel 08/0 : 5[1] -> 6[2] via P2P/CUMEM
+0: jzxh159:1308400:1311136 [1] NCCL INFO Channel 07/0 : 1[1] -> 2[2] via P2P/CUMEM
+0: jzxh159:1308400:1311136 [1] NCCL INFO Channel 08/0 : 1[1] -> 2[2] via P2P/CUMEM
+1: jzxh160:364773:366439 [2] NCCL INFO Channel 00/0 : 6[2] -> 7[3] via P2P/CUMEM
+1: jzxh160:364772:366441 [1] NCCL INFO Channel 11/0 : 5[1] -> 6[2] via P2P/CUMEM
+3: jzxh162:3160744:3162427 [2] NCCL INFO Channel 12/0 : 14[2] -> 15[3] via P2P/CUMEM
+0: jzxh159:1308400:1311136 [1] NCCL INFO Channel 11/0 : 1[1] -> 2[2] via P2P/CUMEM
+1: jzxh160:364772:366441 [1] NCCL INFO Channel 12/0 : 5[1] -> 6[2] via P2P/CUMEM
+0: jzxh159:1308400:1311136 [1] NCCL INFO Channel 12/0 : 1[1] -> 2[2] via P2P/CUMEM
+1: jzxh160:364773:366439 [2] NCCL INFO Channel 04/0 : 6[2] -> 7[3] via P2P/CUMEM
+2: jzxh161:3283538:3285121 [2] NCCL INFO Channel 00/0 : 10[2] -> 11[3] via P2P/CUMEM
+0: jzxh159:1308400:1311136 [1] NCCL INFO Channel 15/0 : 1[1] -> 2[2] via P2P/CUMEM
+1: jzxh160:364772:366441 [1] NCCL INFO Channel 15/0 : 5[1] -> 6[2] via P2P/CUMEM
+1: jzxh160:364773:366439 [2] NCCL INFO Channel 08/0 : 6[2] -> 7[3] via P2P/CUMEM
+0: jzxh159:1308399:1311133 [0] NCCL INFO Channel 00/0 : 15[3] -> 0[0] [receive] via NET/IB/0/GDRDMA
+3: jzxh162:3160743:3162426 [1] NCCL INFO Channel 00/0 : 13[1] -> 14[2] via P2P/CUMEM
+0: jzxh159:1308402:1311135 [3] NCCL INFO Channel 00/0 : 3[3] -> 4[0] [send] via NET/IB/0(0)/GDRDMA
+1: jzxh160:364773:366439 [2] NCCL INFO Channel 12/0 : 6[2] -> 7[3] via P2P/CUMEM
+2: jzxh161:3283538:3285121 [2] NCCL INFO Channel 04/0 : 10[2] -> 11[3] via P2P/CUMEM
+0: jzxh159:1308399:1311133 [0] NCCL INFO Channel 04/0 : 15[3] -> 0[0] [receive] via NET/IB/0/GDRDMA
+0: jzxh159:1308402:1311135 [3] NCCL INFO Channel 04/0 : 3[3] -> 4[0] [send] via NET/IB/0(0)/GDRDMA
+3: jzxh162:3160743:3162426 [1] NCCL INFO Channel 03/0 : 13[1] -> 14[2] via P2P/CUMEM
+0: jzxh159:1308399:1311133 [0] NCCL INFO Channel 08/0 : 15[3] -> 0[0] [receive] via NET/IB/0/GDRDMA
+0: jzxh159:1308402:1311135 [3] NCCL INFO Channel 08/0 : 3[3] -> 4[0] [send] via NET/IB/0(0)/GDRDMA
+2: jzxh161:3283538:3285121 [2] NCCL INFO Channel 08/0 : 10[2] -> 11[3] via P2P/CUMEM
+3: jzxh162:3160743:3162426 [1] NCCL INFO Channel 04/0 : 13[1] -> 14[2] via P2P/CUMEM
+0: jzxh159:1308399:1311133 [0] NCCL INFO Channel 12/0 : 15[3] -> 0[0] [receive] via NET/IB/0/GDRDMA
+0: jzxh159:1308399:1311133 [0] NCCL INFO Channel 00/0 : 0[0] -> 1[1] via P2P/CUMEM
+0: jzxh159:1308402:1311135 [3] NCCL INFO Channel 12/0 : 3[3] -> 4[0] [send] via NET/IB/0(0)/GDRDMA
+2: jzxh161:3283538:3285121 [2] NCCL INFO Channel 12/0 : 10[2] -> 11[3] via P2P/CUMEM
+3: jzxh162:3160743:3162426 [1] NCCL INFO Channel 07/0 : 13[1] -> 14[2] via P2P/CUMEM
+3: jzxh162:3160743:3162426 [1] NCCL INFO Channel 08/0 : 13[1] -> 14[2] via P2P/CUMEM
+0: jzxh159:1308399:1311133 [0] NCCL INFO Channel 03/0 : 0[0] -> 1[1] via P2P/CUMEM
+3: jzxh162:3160743:3162426 [1] NCCL INFO Channel 11/0 : 13[1] -> 14[2] via P2P/CUMEM
+0: jzxh159:1308399:1311133 [0] NCCL INFO Channel 04/0 : 0[0] -> 1[1] via P2P/CUMEM
+0: jzxh159:1308399:1311133 [0] NCCL INFO Channel 07/0 : 0[0] -> 1[1] via P2P/CUMEM
+3: jzxh162:3160743:3162426 [1] NCCL INFO Channel 12/0 : 13[1] -> 14[2] via P2P/CUMEM
+3: jzxh162:3160743:3162426 [1] NCCL INFO Channel 15/0 : 13[1] -> 14[2] via P2P/CUMEM
+0: jzxh159:1308399:1311133 [0] NCCL INFO Channel 08/0 : 0[0] -> 1[1] via P2P/CUMEM
+0: jzxh159:1308399:1311133 [0] NCCL INFO Channel 11/0 : 0[0] -> 1[1] via P2P/CUMEM
+1: jzxh160:364771:366438 [0] NCCL INFO Channel 00/0 : 3[3] -> 4[0] [receive] via NET/IB/0/GDRDMA
+2: jzxh161:3283539:3285120 [3] NCCL INFO Channel 00/0 : 11[3] -> 12[0] [send] via NET/IB/0(8)/GDRDMA
+0: jzxh159:1308399:1311133 [0] NCCL INFO Channel 12/0 : 0[0] -> 1[1] via P2P/CUMEM
+1: jzxh160:364774:366440 [3] NCCL INFO Channel 00/0 : 7[3] -> 8[0] [send] via NET/IB/0(4)/GDRDMA
+2: jzxh161:3283536:3285122 [0] NCCL INFO Channel 00/0 : 7[3] -> 8[0] [receive] via NET/IB/0/GDRDMA
+1: jzxh160:364771:366438 [0] NCCL INFO Channel 04/0 : 3[3] -> 4[0] [receive] via NET/IB/0/GDRDMA
+2: jzxh161:3283539:3285120 [3] NCCL INFO Channel 04/0 : 11[3] -> 12[0] [send] via NET/IB/0(8)/GDRDMA
+1: jzxh160:364774:366440 [3] NCCL INFO Channel 04/0 : 7[3] -> 8[0] [send] via NET/IB/0(4)/GDRDMA
+2: jzxh161:3283536:3285122 [0] NCCL INFO Channel 04/0 : 7[3] -> 8[0] [receive] via NET/IB/0/GDRDMA
+1: jzxh160:364771:366438 [0] NCCL INFO Channel 08/0 : 3[3] -> 4[0] [receive] via NET/IB/0/GDRDMA
+0: jzxh159:1308399:1311133 [0] NCCL INFO Channel 15/0 : 0[0] -> 1[1] via P2P/CUMEM
+1: jzxh160:364774:366440 [3] NCCL INFO Channel 08/0 : 7[3] -> 8[0] [send] via NET/IB/0(4)/GDRDMA
+2: jzxh161:3283539:3285120 [3] NCCL INFO Channel 08/0 : 11[3] -> 12[0] [send] via NET/IB/0(8)/GDRDMA
+1: jzxh160:364771:366438 [0] NCCL INFO Channel 12/0 : 3[3] -> 4[0] [receive] via NET/IB/0/GDRDMA
+2: jzxh161:3283536:3285122 [0] NCCL INFO Channel 08/0 : 7[3] -> 8[0] [receive] via NET/IB/0/GDRDMA
+1: jzxh160:364771:366438 [0] NCCL INFO Channel 00/0 : 4[0] -> 5[1] via P2P/CUMEM
+2: jzxh161:3283539:3285120 [3] NCCL INFO Channel 12/0 : 11[3] -> 12[0] [send] via NET/IB/0(8)/GDRDMA
+1: jzxh160:364774:366440 [3] NCCL INFO Channel 12/0 : 7[3] -> 8[0] [send] via NET/IB/0(4)/GDRDMA
+2: jzxh161:3283536:3285122 [0] NCCL INFO Channel 12/0 : 7[3] -> 8[0] [receive] via NET/IB/0/GDRDMA
+2: jzxh161:3283536:3285122 [0] NCCL INFO Channel 00/0 : 8[0] -> 9[1] via P2P/CUMEM
+1: jzxh160:364771:366438 [0] NCCL INFO Channel 03/0 : 4[0] -> 5[1] via P2P/CUMEM
+3: jzxh162:3160745:3162425 [3] NCCL INFO Channel 00/0 : 15[3] -> 0[0] [send] via NET/IB/0(12)/GDRDMA
+2: jzxh161:3283536:3285122 [0] NCCL INFO Channel 03/0 : 8[0] -> 9[1] via P2P/CUMEM
+3: jzxh162:3160742:3162428 [0] NCCL INFO Channel 00/0 : 11[3] -> 12[0] [receive] via NET/IB/0/GDRDMA
+3: jzxh162:3160745:3162425 [3] NCCL INFO Channel 04/0 : 15[3] -> 0[0] [send] via NET/IB/0(12)/GDRDMA
+1: jzxh160:364771:366438 [0] NCCL INFO Channel 04/0 : 4[0] -> 5[1] via P2P/CUMEM
+1: jzxh160:364771:366438 [0] NCCL INFO Channel 07/0 : 4[0] -> 5[1] via P2P/CUMEM
+2: jzxh161:3283536:3285122 [0] NCCL INFO Channel 04/0 : 8[0] -> 9[1] via P2P/CUMEM
+3: jzxh162:3160742:3162428 [0] NCCL INFO Channel 04/0 : 11[3] -> 12[0] [receive] via NET/IB/0/GDRDMA
+3: jzxh162:3160745:3162425 [3] NCCL INFO Channel 08/0 : 15[3] -> 0[0] [send] via NET/IB/0(12)/GDRDMA
+3: jzxh162:3160742:3162428 [0] NCCL INFO Channel 08/0 : 11[3] -> 12[0] [receive] via NET/IB/0/GDRDMA
+3: jzxh162:3160745:3162425 [3] NCCL INFO Channel 12/0 : 15[3] -> 0[0] [send] via NET/IB/0(12)/GDRDMA
+3: jzxh162:3160742:3162428 [0] NCCL INFO Channel 12/0 : 11[3] -> 12[0] [receive] via NET/IB/0/GDRDMA
+3: jzxh162:3160742:3162428 [0] NCCL INFO Channel 00/0 : 12[0] -> 13[1] via P2P/CUMEM
+0: jzxh159:1308399:1311133 [0] NCCL INFO Channel 01/0 : 0[0] -> 3[3] via P2P/CUMEM
+1: jzxh160:364771:366438 [0] NCCL INFO Channel 08/0 : 4[0] -> 5[1] via P2P/CUMEM
+2: jzxh161:3283536:3285122 [0] NCCL INFO Channel 07/0 : 8[0] -> 9[1] via P2P/CUMEM
+0: jzxh159:1308399:1311133 [0] NCCL INFO Channel 02/0 : 0[0] -> 3[3] via P2P/CUMEM
+1: jzxh160:364771:366438 [0] NCCL INFO Channel 11/0 : 4[0] -> 5[1] via P2P/CUMEM
+2: jzxh161:3283536:3285122 [0] NCCL INFO Channel 08/0 : 8[0] -> 9[1] via P2P/CUMEM
+3: jzxh162:3160742:3162428 [0] NCCL INFO Channel 03/0 : 12[0] -> 13[1] via P2P/CUMEM
+0: jzxh159:1308399:1311133 [0] NCCL INFO Channel 05/0 : 0[0] -> 3[3] via P2P/CUMEM
+1: jzxh160:364771:366438 [0] NCCL INFO Channel 12/0 : 4[0] -> 5[1] via P2P/CUMEM
+2: jzxh161:3283536:3285122 [0] NCCL INFO Channel 11/0 : 8[0] -> 9[1] via P2P/CUMEM
+1: jzxh160:364771:366438 [0] NCCL INFO Channel 15/0 : 4[0] -> 5[1] via P2P/CUMEM
+0: jzxh159:1308399:1311133 [0] NCCL INFO Channel 06/0 : 0[0] -> 3[3] via P2P/CUMEM
+3: jzxh162:3160742:3162428 [0] NCCL INFO Channel 04/0 : 12[0] -> 13[1] via P2P/CUMEM
+0: jzxh159:1308399:1311133 [0] NCCL INFO Channel 09/0 : 0[0] -> 3[3] via P2P/CUMEM
+2: jzxh161:3283536:3285122 [0] NCCL INFO Channel 12/0 : 8[0] -> 9[1] via P2P/CUMEM
+2: jzxh161:3283536:3285122 [0] NCCL INFO Channel 15/0 : 8[0] -> 9[1] via P2P/CUMEM
+1: jzxh160:364771:366438 [0] NCCL INFO Channel 01/0 : 4[0] -> 7[3] via P2P/CUMEM
+3: jzxh162:3160742:3162428 [0] NCCL INFO Channel 07/0 : 12[0] -> 13[1] via P2P/CUMEM
+0: jzxh159:1308399:1311133 [0] NCCL INFO Channel 10/0 : 0[0] -> 3[3] via P2P/CUMEM
+1: jzxh160:364771:366438 [0] NCCL INFO Channel 02/0 : 4[0] -> 7[3] via P2P/CUMEM
+2: jzxh161:3283536:3285122 [0] NCCL INFO Channel 01/0 : 8[0] -> 11[3] via P2P/CUMEM
+3: jzxh162:3160742:3162428 [0] NCCL INFO Channel 08/0 : 12[0] -> 13[1] via P2P/CUMEM
+0: jzxh159:1308399:1311133 [0] NCCL INFO Channel 13/0 : 0[0] -> 3[3] via P2P/CUMEM
+2: jzxh161:3283536:3285122 [0] NCCL INFO Channel 02/0 : 8[0] -> 11[3] via P2P/CUMEM
+1: jzxh160:364771:366438 [0] NCCL INFO Channel 05/0 : 4[0] -> 7[3] via P2P/CUMEM
+3: jzxh162:3160742:3162428 [0] NCCL INFO Channel 11/0 : 12[0] -> 13[1] via P2P/CUMEM
+2: jzxh161:3283536:3285122 [0] NCCL INFO Channel 05/0 : 8[0] -> 11[3] via P2P/CUMEM
+0: jzxh159:1308399:1311133 [0] NCCL INFO Channel 14/0 : 0[0] -> 3[3] via P2P/CUMEM
+1: jzxh160:364771:366438 [0] NCCL INFO Channel 06/0 : 4[0] -> 7[3] via P2P/CUMEM
+3: jzxh162:3160742:3162428 [0] NCCL INFO Channel 12/0 : 12[0] -> 13[1] via P2P/CUMEM
+2: jzxh161:3283536:3285122 [0] NCCL INFO Channel 06/0 : 8[0] -> 11[3] via P2P/CUMEM
+3: jzxh162:3160742:3162428 [0] NCCL INFO Channel 15/0 : 12[0] -> 13[1] via P2P/CUMEM
+1: jzxh160:364771:366438 [0] NCCL INFO Channel 09/0 : 4[0] -> 7[3] via P2P/CUMEM
+2: jzxh161:3283536:3285122 [0] NCCL INFO Channel 09/0 : 8[0] -> 11[3] via P2P/CUMEM
+2: jzxh161:3283536:3285122 [0] NCCL INFO Channel 10/0 : 8[0] -> 11[3] via P2P/CUMEM
+1: jzxh160:364771:366438 [0] NCCL INFO Channel 10/0 : 4[0] -> 7[3] via P2P/CUMEM
+2: jzxh161:3283536:3285122 [0] NCCL INFO Channel 13/0 : 8[0] -> 11[3] via P2P/CUMEM
+1: jzxh160:364771:366438 [0] NCCL INFO Channel 13/0 : 4[0] -> 7[3] via P2P/CUMEM
+3: jzxh162:3160742:3162428 [0] NCCL INFO Channel 01/0 : 12[0] -> 15[3] via P2P/CUMEM
+2: jzxh161:3283536:3285122 [0] NCCL INFO Channel 14/0 : 8[0] -> 11[3] via P2P/CUMEM
+1: jzxh160:364771:366438 [0] NCCL INFO Channel 14/0 : 4[0] -> 7[3] via P2P/CUMEM
+3: jzxh162:3160742:3162428 [0] NCCL INFO Channel 02/0 : 12[0] -> 15[3] via P2P/CUMEM
+0: jzxh159:1308401:1311134 [2] NCCL INFO Channel 02/0 : 15[3] -> 2[2] [receive] via NET/IB/2/GDRDMA
+0: jzxh159:1308402:1311135 [3] NCCL INFO Channel 02/0 : 3[3] -> 6[2] [send] via NET/IB/2(2)/GDRDMA
+0: jzxh159:1308401:1311134 [2] NCCL INFO Channel 06/0 : 15[3] -> 2[2] [receive] via NET/IB/2/GDRDMA
+0: jzxh159:1308402:1311135 [3] NCCL INFO Channel 06/0 : 3[3] -> 6[2] [send] via NET/IB/2(2)/GDRDMA
+0: jzxh159:1308401:1311134 [2] NCCL INFO Channel 10/0 : 15[3] -> 2[2] [receive] via NET/IB/2/GDRDMA
+0: jzxh159:1308402:1311135 [3] NCCL INFO Channel 10/0 : 3[3] -> 6[2] [send] via NET/IB/2(2)/GDRDMA
+0: jzxh159:1308401:1311134 [2] NCCL INFO Channel 14/0 : 15[3] -> 2[2] [receive] via NET/IB/2/GDRDMA
+0: jzxh159:1308402:1311135 [3] NCCL INFO Channel 14/0 : 3[3] -> 6[2] [send] via NET/IB/2(2)/GDRDMA
+3: jzxh162:3160742:3162428 [0] NCCL INFO Channel 05/0 : 12[0] -> 15[3] via P2P/CUMEM
+3: jzxh162:3160742:3162428 [0] NCCL INFO Channel 06/0 : 12[0] -> 15[3] via P2P/CUMEM
+3: jzxh162:3160742:3162428 [0] NCCL INFO Channel 09/0 : 12[0] -> 15[3] via P2P/CUMEM
+3: jzxh162:3160742:3162428 [0] NCCL INFO Channel 10/0 : 12[0] -> 15[3] via P2P/CUMEM
+3: jzxh162:3160742:3162428 [0] NCCL INFO Channel 13/0 : 12[0] -> 15[3] via P2P/CUMEM
+3: jzxh162:3160742:3162428 [0] NCCL INFO Channel 14/0 : 12[0] -> 15[3] via P2P/CUMEM
+1: jzxh160:364773:366439 [2] NCCL INFO Channel 02/0 : 3[3] -> 6[2] [receive] via NET/IB/2/GDRDMA
+1: jzxh160:364774:366440 [3] NCCL INFO Channel 02/0 : 7[3] -> 10[2] [send] via NET/IB/2(6)/GDRDMA
+2: jzxh161:3283538:3285121 [2] NCCL INFO Channel 02/0 : 7[3] -> 10[2] [receive] via NET/IB/2/GDRDMA
+2: jzxh161:3283539:3285120 [3] NCCL INFO Channel 02/0 : 11[3] -> 14[2] [send] via NET/IB/2(10)/GDRDMA
+1: jzxh160:364773:366439 [2] NCCL INFO Channel 06/0 : 3[3] -> 6[2] [receive] via NET/IB/2/GDRDMA
+1: jzxh160:364774:366440 [3] NCCL INFO Channel 06/0 : 7[3] -> 10[2] [send] via NET/IB/2(6)/GDRDMA
+1: jzxh160:364773:366439 [2] NCCL INFO Channel 10/0 : 3[3] -> 6[2] [receive] via NET/IB/2/GDRDMA
+1: jzxh160:364774:366440 [3] NCCL INFO Channel 10/0 : 7[3] -> 10[2] [send] via NET/IB/2(6)/GDRDMA
+1: jzxh160:364773:366439 [2] NCCL INFO Channel 14/0 : 3[3] -> 6[2] [receive] via NET/IB/2/GDRDMA
+1: jzxh160:364774:366440 [3] NCCL INFO Channel 14/0 : 7[3] -> 10[2] [send] via NET/IB/2(6)/GDRDMA
+2: jzxh161:3283538:3285121 [2] NCCL INFO Channel 06/0 : 7[3] -> 10[2] [receive] via NET/IB/2/GDRDMA
+2: jzxh161:3283539:3285120 [3] NCCL INFO Channel 06/0 : 11[3] -> 14[2] [send] via NET/IB/2(10)/GDRDMA
+2: jzxh161:3283538:3285121 [2] NCCL INFO Channel 10/0 : 7[3] -> 10[2] [receive] via NET/IB/2/GDRDMA
+2: jzxh161:3283539:3285120 [3] NCCL INFO Channel 10/0 : 11[3] -> 14[2] [send] via NET/IB/2(10)/GDRDMA
+0: jzxh159:1308400:1311136 [1] NCCL INFO Channel 01/0 : 14[2] -> 1[1] [receive] via NET/IB/1/GDRDMA
+0: jzxh159:1308401:1311134 [2] NCCL INFO Channel 01/0 : 2[2] -> 5[1] [send] via NET/IB/1(1)/GDRDMA
+2: jzxh161:3283538:3285121 [2] NCCL INFO Channel 14/0 : 7[3] -> 10[2] [receive] via NET/IB/2/GDRDMA
+0: jzxh159:1308400:1311136 [1] NCCL INFO Channel 05/0 : 14[2] -> 1[1] [receive] via NET/IB/1/GDRDMA
+2: jzxh161:3283539:3285120 [3] NCCL INFO Channel 14/0 : 11[3] -> 14[2] [send] via NET/IB/2(10)/GDRDMA
+0: jzxh159:1308401:1311134 [2] NCCL INFO Channel 05/0 : 2[2] -> 5[1] [send] via NET/IB/1(1)/GDRDMA
+0: jzxh159:1308400:1311136 [1] NCCL INFO Channel 09/0 : 14[2] -> 1[1] [receive] via NET/IB/1/GDRDMA
+0: jzxh159:1308401:1311134 [2] NCCL INFO Channel 09/0 : 2[2] -> 5[1] [send] via NET/IB/1(1)/GDRDMA
+0: jzxh159:1308400:1311136 [1] NCCL INFO Channel 13/0 : 14[2] -> 1[1] [receive] via NET/IB/1/GDRDMA
+0: jzxh159:1308401:1311134 [2] NCCL INFO Channel 13/0 : 2[2] -> 5[1] [send] via NET/IB/1(1)/GDRDMA
+3: jzxh162:3160744:3162427 [2] NCCL INFO Channel 02/0 : 11[3] -> 14[2] [receive] via NET/IB/2/GDRDMA
+3: jzxh162:3160745:3162425 [3] NCCL INFO Channel 02/0 : 15[3] -> 2[2] [send] via NET/IB/2(14)/GDRDMA
+3: jzxh162:3160744:3162427 [2] NCCL INFO Channel 06/0 : 11[3] -> 14[2] [receive] via NET/IB/2/GDRDMA
+3: jzxh162:3160745:3162425 [3] NCCL INFO Channel 06/0 : 15[3] -> 2[2] [send] via NET/IB/2(14)/GDRDMA
+3: jzxh162:3160744:3162427 [2] NCCL INFO Channel 10/0 : 11[3] -> 14[2] [receive] via NET/IB/2/GDRDMA
+3: jzxh162:3160745:3162425 [3] NCCL INFO Channel 10/0 : 15[3] -> 2[2] [send] via NET/IB/2(14)/GDRDMA
+3: jzxh162:3160744:3162427 [2] NCCL INFO Channel 14/0 : 11[3] -> 14[2] [receive] via NET/IB/2/GDRDMA
+3: jzxh162:3160745:3162425 [3] NCCL INFO Channel 14/0 : 15[3] -> 2[2] [send] via NET/IB/2(14)/GDRDMA
+1: jzxh160:364772:366441 [1] NCCL INFO Channel 01/0 : 2[2] -> 5[1] [receive] via NET/IB/1/GDRDMA
+1: jzxh160:364773:366439 [2] NCCL INFO Channel 01/0 : 6[2] -> 9[1] [send] via NET/IB/1(5)/GDRDMA
+1: jzxh160:364772:366441 [1] NCCL INFO Channel 05/0 : 2[2] -> 5[1] [receive] via NET/IB/1/GDRDMA
+1: jzxh160:364773:366439 [2] NCCL INFO Channel 05/0 : 6[2] -> 9[1] [send] via NET/IB/1(5)/GDRDMA
+2: jzxh161:3283537:3285123 [1] NCCL INFO Channel 01/0 : 6[2] -> 9[1] [receive] via NET/IB/1/GDRDMA
+2: jzxh161:3283538:3285121 [2] NCCL INFO Channel 01/0 : 10[2] -> 13[1] [send] via NET/IB/1(9)/GDRDMA
+2: jzxh161:3283537:3285123 [1] NCCL INFO Channel 05/0 : 6[2] -> 9[1] [receive] via NET/IB/1/GDRDMA
+2: jzxh161:3283538:3285121 [2] NCCL INFO Channel 05/0 : 10[2] -> 13[1] [send] via NET/IB/1(9)/GDRDMA
+2: jzxh161:3283537:3285123 [1] NCCL INFO Channel 09/0 : 6[2] -> 9[1] [receive] via NET/IB/1/GDRDMA
+1: jzxh160:364772:366441 [1] NCCL INFO Channel 09/0 : 2[2] -> 5[1] [receive] via NET/IB/1/GDRDMA
+2: jzxh161:3283538:3285121 [2] NCCL INFO Channel 09/0 : 10[2] -> 13[1] [send] via NET/IB/1(9)/GDRDMA
+1: jzxh160:364773:366439 [2] NCCL INFO Channel 09/0 : 6[2] -> 9[1] [send] via NET/IB/1(5)/GDRDMA
+2: jzxh161:3283537:3285123 [1] NCCL INFO Channel 13/0 : 6[2] -> 9[1] [receive] via NET/IB/1/GDRDMA
+1: jzxh160:364772:366441 [1] NCCL INFO Channel 13/0 : 2[2] -> 5[1] [receive] via NET/IB/1/GDRDMA
+2: jzxh161:3283538:3285121 [2] NCCL INFO Channel 13/0 : 10[2] -> 13[1] [send] via NET/IB/1(9)/GDRDMA
+1: jzxh160:364773:366439 [2] NCCL INFO Channel 13/0 : 6[2] -> 9[1] [send] via NET/IB/1(5)/GDRDMA
+1: jzxh160:364772:366441 [1] NCCL INFO Channel 01/0 : 5[1] -> 4[0] via P2P/CUMEM
+1: jzxh160:364772:366441 [1] NCCL INFO Channel 02/0 : 5[1] -> 4[0] via P2P/CUMEM
+2: jzxh161:3283537:3285123 [1] NCCL INFO Channel 01/0 : 9[1] -> 8[0] via P2P/CUMEM
+3: jzxh162:3160743:3162426 [1] NCCL INFO Channel 01/0 : 10[2] -> 13[1] [receive] via NET/IB/1/GDRDMA
+2: jzxh161:3283537:3285123 [1] NCCL INFO Channel 02/0 : 9[1] -> 8[0] via P2P/CUMEM
+1: jzxh160:364772:366441 [1] NCCL INFO Channel 05/0 : 5[1] -> 4[0] via P2P/CUMEM
+3: jzxh162:3160744:3162427 [2] NCCL INFO Channel 01/0 : 14[2] -> 1[1] [send] via NET/IB/1(13)/GDRDMA
+3: jzxh162:3160743:3162426 [1] NCCL INFO Channel 05/0 : 10[2] -> 13[1] [receive] via NET/IB/1/GDRDMA
+3: jzxh162:3160744:3162427 [2] NCCL INFO Channel 05/0 : 14[2] -> 1[1] [send] via NET/IB/1(13)/GDRDMA
+1: jzxh160:364772:366441 [1] NCCL INFO Channel 06/0 : 5[1] -> 4[0] via P2P/CUMEM
+2: jzxh161:3283537:3285123 [1] NCCL INFO Channel 05/0 : 9[1] -> 8[0] via P2P/CUMEM
+3: jzxh162:3160743:3162426 [1] NCCL INFO Channel 09/0 : 10[2] -> 13[1] [receive] via NET/IB/1/GDRDMA
+3: jzxh162:3160744:3162427 [2] NCCL INFO Channel 09/0 : 14[2] -> 1[1] [send] via NET/IB/1(13)/GDRDMA
+3: jzxh162:3160743:3162426 [1] NCCL INFO Channel 13/0 : 10[2] -> 13[1] [receive] via NET/IB/1/GDRDMA
+1: jzxh160:364772:366441 [1] NCCL INFO Channel 09/0 : 5[1] -> 4[0] via P2P/CUMEM
+2: jzxh161:3283537:3285123 [1] NCCL INFO Channel 06/0 : 9[1] -> 8[0] via P2P/CUMEM
+3: jzxh162:3160744:3162427 [2] NCCL INFO Channel 13/0 : 14[2] -> 1[1] [send] via NET/IB/1(13)/GDRDMA
+1: jzxh160:364772:366441 [1] NCCL INFO Channel 10/0 : 5[1] -> 4[0] via P2P/CUMEM
+1: jzxh160:364772:366441 [1] NCCL INFO Channel 13/0 : 5[1] -> 4[0] via P2P/CUMEM
+2: jzxh161:3283537:3285123 [1] NCCL INFO Channel 09/0 : 9[1] -> 8[0] via P2P/CUMEM
+2: jzxh161:3283537:3285123 [1] NCCL INFO Channel 10/0 : 9[1] -> 8[0] via P2P/CUMEM
+2: jzxh161:3283537:3285123 [1] NCCL INFO Channel 13/0 : 9[1] -> 8[0] via P2P/CUMEM
+2: jzxh161:3283537:3285123 [1] NCCL INFO Channel 14/0 : 9[1] -> 8[0] via P2P/CUMEM
+1: jzxh160:364772:366441 [1] NCCL INFO Channel 14/0 : 5[1] -> 4[0] via P2P/CUMEM
+0: jzxh159:1308400:1311136 [1] NCCL INFO Channel 01/0 : 1[1] -> 0[0] via P2P/CUMEM
+3: jzxh162:3160743:3162426 [1] NCCL INFO Channel 01/0 : 13[1] -> 12[0] via P2P/CUMEM
+0: jzxh159:1308400:1311136 [1] NCCL INFO Channel 02/0 : 1[1] -> 0[0] via P2P/CUMEM
+3: jzxh162:3160743:3162426 [1] NCCL INFO Channel 02/0 : 13[1] -> 12[0] via P2P/CUMEM
+0: jzxh159:1308400:1311136 [1] NCCL INFO Channel 05/0 : 1[1] -> 0[0] via P2P/CUMEM
+3: jzxh162:3160743:3162426 [1] NCCL INFO Channel 05/0 : 13[1] -> 12[0] via P2P/CUMEM
+0: jzxh159:1308400:1311136 [1] NCCL INFO Channel 06/0 : 1[1] -> 0[0] via P2P/CUMEM
+3: jzxh162:3160743:3162426 [1] NCCL INFO Channel 06/0 : 13[1] -> 12[0] via P2P/CUMEM
+0: jzxh159:1308400:1311136 [1] NCCL INFO Channel 09/0 : 1[1] -> 0[0] via P2P/CUMEM
+0: jzxh159:1308400:1311136 [1] NCCL INFO Channel 10/0 : 1[1] -> 0[0] via P2P/CUMEM
+3: jzxh162:3160743:3162426 [1] NCCL INFO Channel 09/0 : 13[1] -> 12[0] via P2P/CUMEM
+0: jzxh159:1308400:1311136 [1] NCCL INFO Channel 13/0 : 1[1] -> 0[0] via P2P/CUMEM
+3: jzxh162:3160743:3162426 [1] NCCL INFO Channel 10/0 : 13[1] -> 12[0] via P2P/CUMEM
+3: jzxh162:3160743:3162426 [1] NCCL INFO Channel 13/0 : 13[1] -> 12[0] via P2P/CUMEM
+0: jzxh159:1308400:1311136 [1] NCCL INFO Channel 14/0 : 1[1] -> 0[0] via P2P/CUMEM
+3: jzxh162:3160743:3162426 [1] NCCL INFO Channel 14/0 : 13[1] -> 12[0] via P2P/CUMEM
+3: jzxh162:3160745:3162425 [3] NCCL INFO Channel 03/0 : 10[2] -> 15[3] [receive] via NET/IB/3/GDRDMA
+3: jzxh162:3160744:3162427 [2] NCCL INFO Channel 03/0 : 14[2] -> 3[3] [send] via NET/IB/3(15)/GDRDMA
+3: jzxh162:3160745:3162425 [3] NCCL INFO Channel 07/0 : 10[2] -> 15[3] [receive] via NET/IB/3/GDRDMA
+3: jzxh162:3160744:3162427 [2] NCCL INFO Channel 07/0 : 14[2] -> 3[3] [send] via NET/IB/3(15)/GDRDMA
+3: jzxh162:3160745:3162425 [3] NCCL INFO Channel 11/0 : 10[2] -> 15[3] [receive] via NET/IB/3/GDRDMA
+3: jzxh162:3160744:3162427 [2] NCCL INFO Channel 11/0 : 14[2] -> 3[3] [send] via NET/IB/3(15)/GDRDMA
+3: jzxh162:3160745:3162425 [3] NCCL INFO Channel 15/0 : 10[2] -> 15[3] [receive] via NET/IB/3/GDRDMA
+3: jzxh162:3160744:3162427 [2] NCCL INFO Channel 15/0 : 14[2] -> 3[3] [send] via NET/IB/3(15)/GDRDMA
+1: jzxh160:364773:366439 [2] NCCL INFO Channel 03/0 : 6[2] -> 11[3] [send] via NET/IB/3(7)/GDRDMA
+0: jzxh159:1308401:1311134 [2] NCCL INFO Channel 03/0 : 2[2] -> 7[3] [send] via NET/IB/3(3)/GDRDMA
+0: jzxh159:1308402:1311135 [3] NCCL INFO Channel 03/0 : 14[2] -> 3[3] [receive] via NET/IB/3/GDRDMA
+0: jzxh159:1308401:1311134 [2] NCCL INFO Channel 07/0 : 2[2] -> 7[3] [send] via NET/IB/3(3)/GDRDMA
+0: jzxh159:1308402:1311135 [3] NCCL INFO Channel 07/0 : 14[2] -> 3[3] [receive] via NET/IB/3/GDRDMA
+0: jzxh159:1308401:1311134 [2] NCCL INFO Channel 11/0 : 2[2] -> 7[3] [send] via NET/IB/3(3)/GDRDMA
+0: jzxh159:1308402:1311135 [3] NCCL INFO Channel 11/0 : 14[2] -> 3[3] [receive] via NET/IB/3/GDRDMA
+1: jzxh160:364774:366440 [3] NCCL INFO Channel 03/0 : 2[2] -> 7[3] [receive] via NET/IB/3/GDRDMA
+1: jzxh160:364773:366439 [2] NCCL INFO Channel 07/0 : 6[2] -> 11[3] [send] via NET/IB/3(7)/GDRDMA
+1: jzxh160:364774:366440 [3] NCCL INFO Channel 07/0 : 2[2] -> 7[3] [receive] via NET/IB/3/GDRDMA
+0: jzxh159:1308401:1311134 [2] NCCL INFO Channel 15/0 : 2[2] -> 7[3] [send] via NET/IB/3(3)/GDRDMA
+1: jzxh160:364773:366439 [2] NCCL INFO Channel 11/0 : 6[2] -> 11[3] [send] via NET/IB/3(7)/GDRDMA
+1: jzxh160:364774:366440 [3] NCCL INFO Channel 11/0 : 2[2] -> 7[3] [receive] via NET/IB/3/GDRDMA
+1: jzxh160:364773:366439 [2] NCCL INFO Channel 15/0 : 6[2] -> 11[3] [send] via NET/IB/3(7)/GDRDMA
+0: jzxh159:1308402:1311135 [3] NCCL INFO Channel 15/0 : 14[2] -> 3[3] [receive] via NET/IB/3/GDRDMA
+1: jzxh160:364774:366440 [3] NCCL INFO Channel 15/0 : 2[2] -> 7[3] [receive] via NET/IB/3/GDRDMA
+0: jzxh159:1308402:1311135 [3] NCCL INFO Channel 03/0 : 3[3] -> 0[0] via P2P/CUMEM
+1: jzxh160:364774:366440 [3] NCCL INFO Channel 03/0 : 7[3] -> 4[0] via P2P/CUMEM
+0: jzxh159:1308402:1311135 [3] NCCL INFO Channel 07/0 : 3[3] -> 0[0] via P2P/CUMEM
+1: jzxh160:364774:366440 [3] NCCL INFO Channel 07/0 : 7[3] -> 4[0] via P2P/CUMEM
+0: jzxh159:1308402:1311135 [3] NCCL INFO Channel 11/0 : 3[3] -> 0[0] via P2P/CUMEM
+1: jzxh160:364774:366440 [3] NCCL INFO Channel 11/0 : 7[3] -> 4[0] via P2P/CUMEM
+0: jzxh159:1308402:1311135 [3] NCCL INFO Channel 15/0 : 3[3] -> 0[0] via P2P/CUMEM
+1: jzxh160:364774:366440 [3] NCCL INFO Channel 15/0 : 7[3] -> 4[0] via P2P/CUMEM
+3: jzxh162:3160744:3162427 [2] NCCL INFO Channel 02/0 : 14[2] -> 13[1] via P2P/CUMEM
+0: jzxh159:1308401:1311134 [2] NCCL INFO Channel 02/0 : 2[2] -> 1[1] via P2P/CUMEM
+1: jzxh160:364774:366440 [3] NCCL INFO Channel 01/0 : 7[3] -> 6[2] via P2P/CUMEM
+3: jzxh162:3160744:3162427 [2] NCCL INFO Channel 06/0 : 14[2] -> 13[1] via P2P/CUMEM
+0: jzxh159:1308402:1311135 [3] NCCL INFO Channel 01/0 : 3[3] -> 2[2] via P2P/CUMEM
+2: jzxh161:3283539:3285120 [3] NCCL INFO Channel 03/0 : 6[2] -> 11[3] [receive] via NET/IB/3/GDRDMA
+2: jzxh161:3283538:3285121 [2] NCCL INFO Channel 03/0 : 10[2] -> 15[3] [send] via NET/IB/3(11)/GDRDMA
+0: jzxh159:1308401:1311134 [2] NCCL INFO Channel 06/0 : 2[2] -> 1[1] via P2P/CUMEM
+1: jzxh160:364774:366440 [3] NCCL INFO Channel 05/0 : 7[3] -> 6[2] via P2P/CUMEM
+3: jzxh162:3160744:3162427 [2] NCCL INFO Channel 10/0 : 14[2] -> 13[1] via P2P/CUMEM
+2: jzxh161:3283539:3285120 [3] NCCL INFO Channel 07/0 : 6[2] -> 11[3] [receive] via NET/IB/3/GDRDMA
+2: jzxh161:3283538:3285121 [2] NCCL INFO Channel 07/0 : 10[2] -> 15[3] [send] via NET/IB/3(11)/GDRDMA
+2: jzxh161:3283539:3285120 [3] NCCL INFO Channel 11/0 : 6[2] -> 11[3] [receive] via NET/IB/3/GDRDMA
+2: jzxh161:3283538:3285121 [2] NCCL INFO Channel 11/0 : 10[2] -> 15[3] [send] via NET/IB/3(11)/GDRDMA
+2: jzxh161:3283539:3285120 [3] NCCL INFO Channel 15/0 : 6[2] -> 11[3] [receive] via NET/IB/3/GDRDMA
+2: jzxh161:3283538:3285121 [2] NCCL INFO Channel 15/0 : 10[2] -> 15[3] [send] via NET/IB/3(11)/GDRDMA
+1: jzxh160:364774:366440 [3] NCCL INFO Channel 09/0 : 7[3] -> 6[2] via P2P/CUMEM
+0: jzxh159:1308402:1311135 [3] NCCL INFO Channel 05/0 : 3[3] -> 2[2] via P2P/CUMEM
+0: jzxh159:1308401:1311134 [2] NCCL INFO Channel 10/0 : 2[2] -> 1[1] via P2P/CUMEM
+3: jzxh162:3160744:3162427 [2] NCCL INFO Channel 14/0 : 14[2] -> 13[1] via P2P/CUMEM
+2: jzxh161:3283539:3285120 [3] NCCL INFO Channel 03/0 : 11[3] -> 8[0] via P2P/CUMEM
+3: jzxh162:3160745:3162425 [3] NCCL INFO Channel 03/0 : 15[3] -> 12[0] via P2P/CUMEM
+1: jzxh160:364774:366440 [3] NCCL INFO Channel 13/0 : 7[3] -> 6[2] via P2P/CUMEM
+0: jzxh159:1308401:1311134 [2] NCCL INFO Channel 14/0 : 2[2] -> 1[1] via P2P/CUMEM
+0: jzxh159:1308402:1311135 [3] NCCL INFO Channel 09/0 : 3[3] -> 2[2] via P2P/CUMEM
+2: jzxh161:3283539:3285120 [3] NCCL INFO Channel 07/0 : 11[3] -> 8[0] via P2P/CUMEM
+3: jzxh162:3160745:3162425 [3] NCCL INFO Channel 07/0 : 15[3] -> 12[0] via P2P/CUMEM
+2: jzxh161:3283539:3285120 [3] NCCL INFO Channel 11/0 : 11[3] -> 8[0] via P2P/CUMEM
+3: jzxh162:3160745:3162425 [3] NCCL INFO Channel 11/0 : 15[3] -> 12[0] via P2P/CUMEM
+0: jzxh159:1308402:1311135 [3] NCCL INFO Channel 13/0 : 3[3] -> 2[2] via P2P/CUMEM
+2: jzxh161:3283539:3285120 [3] NCCL INFO Channel 15/0 : 11[3] -> 8[0] via P2P/CUMEM
+3: jzxh162:3160745:3162425 [3] NCCL INFO Channel 15/0 : 15[3] -> 12[0] via P2P/CUMEM
+2: jzxh161:3283538:3285121 [2] NCCL INFO Channel 02/0 : 10[2] -> 9[1] via P2P/CUMEM
+3: jzxh162:3160745:3162425 [3] NCCL INFO Channel 01/0 : 15[3] -> 14[2] via P2P/CUMEM
+1: jzxh160:364773:366439 [2] NCCL INFO Channel 02/0 : 6[2] -> 5[1] via P2P/CUMEM
+2: jzxh161:3283539:3285120 [3] NCCL INFO Channel 01/0 : 11[3] -> 10[2] via P2P/CUMEM
+3: jzxh162:3160745:3162425 [3] NCCL INFO Channel 05/0 : 15[3] -> 14[2] via P2P/CUMEM
+2: jzxh161:3283538:3285121 [2] NCCL INFO Channel 06/0 : 10[2] -> 9[1] via P2P/CUMEM
+1: jzxh160:364773:366439 [2] NCCL INFO Channel 06/0 : 6[2] -> 5[1] via P2P/CUMEM
+2: jzxh161:3283539:3285120 [3] NCCL INFO Channel 05/0 : 11[3] -> 10[2] via P2P/CUMEM
+3: jzxh162:3160745:3162425 [3] NCCL INFO Channel 09/0 : 15[3] -> 14[2] via P2P/CUMEM
+2: jzxh161:3283538:3285121 [2] NCCL INFO Channel 10/0 : 10[2] -> 9[1] via P2P/CUMEM
+1: jzxh160:364773:366439 [2] NCCL INFO Channel 10/0 : 6[2] -> 5[1] via P2P/CUMEM
+2: jzxh161:3283539:3285120 [3] NCCL INFO Channel 09/0 : 11[3] -> 10[2] via P2P/CUMEM
+3: jzxh162:3160745:3162425 [3] NCCL INFO Channel 13/0 : 15[3] -> 14[2] via P2P/CUMEM
+1: jzxh160:364773:366439 [2] NCCL INFO Channel 14/0 : 6[2] -> 5[1] via P2P/CUMEM
+2: jzxh161:3283538:3285121 [2] NCCL INFO Channel 14/0 : 10[2] -> 9[1] via P2P/CUMEM
+2: jzxh161:3283539:3285120 [3] NCCL INFO Channel 13/0 : 11[3] -> 10[2] via P2P/CUMEM
+2: jzxh161:3283536:3285122 [0] NCCL INFO Connected all rings
+2: jzxh161:3283536:3285122 [0] NCCL INFO Channel 02/0 : 8[0] -> 9[1] via P2P/CUMEM
+2: jzxh161:3283536:3285122 [0] NCCL INFO Channel 10/0 : 8[0] -> 9[1] via P2P/CUMEM
+0: jzxh159:1308401:1311134 [2] NCCL INFO Connected all rings
+0: jzxh159:1308399:1311133 [0] NCCL INFO Connected all rings
+0: jzxh159:1308402:1311135 [3] NCCL INFO Connected all rings
+0: jzxh159:1308400:1311136 [1] NCCL INFO Connected all rings
+0: jzxh159:1308399:1311133 [0] NCCL INFO Channel 02/0 : 0[0] -> 1[1] via P2P/CUMEM
+1: jzxh160:364774:366440 [3] NCCL INFO Connected all rings
+1: jzxh160:364773:366439 [2] NCCL INFO Connected all rings
+1: jzxh160:364771:366438 [0] NCCL INFO Connected all rings
+3: jzxh162:3160742:3162428 [0] NCCL INFO Connected all rings
+1: jzxh160:364772:366441 [1] NCCL INFO Connected all rings
+3: jzxh162:3160742:3162428 [0] NCCL INFO Channel 02/0 : 12[0] -> 13[1] via P2P/CUMEM
+3: jzxh162:3160743:3162426 [1] NCCL INFO Connected all rings
+1: jzxh160:364771:366438 [0] NCCL INFO Channel 02/0 : 4[0] -> 5[1] via P2P/CUMEM
+3: jzxh162:3160745:3162425 [3] NCCL INFO Connected all rings
+3: jzxh162:3160744:3162427 [2] NCCL INFO Connected all rings
+0: jzxh159:1308399:1311133 [0] NCCL INFO Channel 10/0 : 0[0] -> 1[1] via P2P/CUMEM
+1: jzxh160:364771:366438 [0] NCCL INFO Channel 10/0 : 4[0] -> 5[1] via P2P/CUMEM
+3: jzxh162:3160742:3162428 [0] NCCL INFO Channel 10/0 : 12[0] -> 13[1] via P2P/CUMEM
+2: jzxh161:3283537:3285123 [1] NCCL INFO Connected all rings
+2: jzxh161:3283539:3285120 [3] NCCL INFO Connected all rings
+2: jzxh161:3283538:3285121 [2] NCCL INFO Connected all rings
+0: jzxh159:1308400:1311136 [1] NCCL INFO Channel 01/0 : 1[1] -> 2[2] via P2P/CUMEM
+0: jzxh159:1308401:1311134 [2] NCCL INFO Channel 01/0 : 2[2] -> 3[3] via P2P/CUMEM
+1: jzxh160:364772:366441 [1] NCCL INFO Channel 01/0 : 5[1] -> 6[2] via P2P/CUMEM
+3: jzxh162:3160744:3162427 [2] NCCL INFO Channel 01/0 : 14[2] -> 15[3] via P2P/CUMEM
+0: jzxh159:1308400:1311136 [1] NCCL INFO Channel 09/0 : 1[1] -> 2[2] via P2P/CUMEM
+2: jzxh161:3283537:3285123 [1] NCCL INFO Channel 01/0 : 9[1] -> 10[2] via P2P/CUMEM
+3: jzxh162:3160743:3162426 [1] NCCL INFO Channel 01/0 : 13[1] -> 14[2] via P2P/CUMEM
+3: jzxh162:3160743:3162426 [1] NCCL INFO Channel 09/0 : 13[1] -> 14[2] via P2P/CUMEM
+3: jzxh162:3160744:3162427 [2] NCCL INFO Channel 07/0 : 14[2] -> 15[3] via P2P/CUMEM
+0: jzxh159:1308401:1311134 [2] NCCL INFO Channel 07/0 : 2[2] -> 3[3] via P2P/CUMEM
+1: jzxh160:364773:366439 [2] NCCL INFO Channel 01/0 : 6[2] -> 7[3] via P2P/CUMEM
+2: jzxh161:3283537:3285123 [1] NCCL INFO Channel 09/0 : 9[1] -> 10[2] via P2P/CUMEM
+3: jzxh162:3160744:3162427 [2] NCCL INFO Channel 09/0 : 14[2] -> 15[3] via P2P/CUMEM
+0: jzxh159:1308399:1311133 [0] NCCL INFO Channel 02/0 : 0[0] -> 2[2] via P2P/CUMEM
+0: jzxh159:1308401:1311134 [2] NCCL INFO Channel 09/0 : 2[2] -> 3[3] via P2P/CUMEM
+1: jzxh160:364773:366439 [2] NCCL INFO Channel 07/0 : 6[2] -> 7[3] via P2P/CUMEM
+2: jzxh161:3283538:3285121 [2] NCCL INFO Channel 01/0 : 10[2] -> 11[3] via P2P/CUMEM
+3: jzxh162:3160742:3162428 [0] NCCL INFO Channel 02/0 : 12[0] -> 14[2] via P2P/CUMEM
+3: jzxh162:3160744:3162427 [2] NCCL INFO Channel 15/0 : 14[2] -> 15[3] via P2P/CUMEM
+1: jzxh160:364772:366441 [1] NCCL INFO Channel 09/0 : 5[1] -> 6[2] via P2P/CUMEM
+1: jzxh160:364773:366439 [2] NCCL INFO Channel 09/0 : 6[2] -> 7[3] via P2P/CUMEM
+1: jzxh160:364773:366439 [2] NCCL INFO Channel 15/0 : 6[2] -> 7[3] via P2P/CUMEM
+0: jzxh159:1308399:1311133 [0] NCCL INFO Channel 03/0 : 0[0] -> 2[2] via P2P/CUMEM
+2: jzxh161:3283538:3285121 [2] NCCL INFO Channel 07/0 : 10[2] -> 11[3] via P2P/CUMEM
+0: jzxh159:1308401:1311134 [2] NCCL INFO Channel 15/0 : 2[2] -> 3[3] via P2P/CUMEM
+1: jzxh160:364772:366441 [1] NCCL INFO Channel 02/0 : 5[1] -> 7[3] via P2P/CUMEM
+3: jzxh162:3160742:3162428 [0] NCCL INFO Channel 03/0 : 12[0] -> 14[2] via P2P/CUMEM
+2: jzxh161:3283536:3285122 [0] NCCL INFO Channel 02/0 : 8[0] -> 10[2] via P2P/CUMEM
+3: jzxh162:3160743:3162426 [1] NCCL INFO Channel 02/0 : 13[1] -> 15[3] via P2P/CUMEM
+0: jzxh159:1308399:1311133 [0] NCCL INFO Channel 05/0 : 0[0] -> 2[2] via P2P/CUMEM
+2: jzxh161:3283538:3285121 [2] NCCL INFO Channel 09/0 : 10[2] -> 11[3] via P2P/CUMEM
+0: jzxh159:1308400:1311136 [1] NCCL INFO Channel 02/0 : 1[1] -> 3[3] via P2P/CUMEM
+0: jzxh159:1308399:1311133 [0] NCCL INFO Channel 06/0 : 0[0] -> 2[2] via P2P/CUMEM
+1: jzxh160:364772:366441 [1] NCCL INFO Channel 03/0 : 5[1] -> 7[3] via P2P/CUMEM
+2: jzxh161:3283536:3285122 [0] NCCL INFO Channel 03/0 : 8[0] -> 10[2] via P2P/CUMEM
+2: jzxh161:3283538:3285121 [2] NCCL INFO Channel 15/0 : 10[2] -> 11[3] via P2P/CUMEM
+3: jzxh162:3160743:3162426 [1] NCCL INFO Channel 03/0 : 13[1] -> 15[3] via P2P/CUMEM
+3: jzxh162:3160742:3162428 [0] NCCL INFO Channel 05/0 : 12[0] -> 14[2] via P2P/CUMEM
+2: jzxh161:3283536:3285122 [0] NCCL INFO Channel 05/0 : 8[0] -> 10[2] via P2P/CUMEM
+2: jzxh161:3283537:3285123 [1] NCCL INFO Channel 02/0 : 9[1] -> 11[3] via P2P/CUMEM
+2: jzxh161:3283536:3285122 [0] NCCL INFO Channel 06/0 : 8[0] -> 10[2] via P2P/CUMEM
+0: jzxh159:1308399:1311133 [0] NCCL INFO Channel 10/0 : 0[0] -> 2[2] via P2P/CUMEM
+0: jzxh159:1308400:1311136 [1] NCCL INFO Channel 03/0 : 1[1] -> 3[3] via P2P/CUMEM
+1: jzxh160:364772:366441 [1] NCCL INFO Channel 05/0 : 5[1] -> 7[3] via P2P/CUMEM
+3: jzxh162:3160743:3162426 [1] NCCL INFO Channel 05/0 : 13[1] -> 15[3] via P2P/CUMEM
+1: jzxh160:364771:366438 [0] NCCL INFO Channel 02/0 : 4[0] -> 6[2] via P2P/CUMEM
+2: jzxh161:3283536:3285122 [0] NCCL INFO Channel 10/0 : 8[0] -> 10[2] via P2P/CUMEM
+3: jzxh162:3160742:3162428 [0] NCCL INFO Channel 06/0 : 12[0] -> 14[2] via P2P/CUMEM
+0: jzxh159:1308399:1311133 [0] NCCL INFO Channel 11/0 : 0[0] -> 2[2] via P2P/CUMEM
+2: jzxh161:3283537:3285123 [1] NCCL INFO Channel 03/0 : 9[1] -> 11[3] via P2P/CUMEM
+0: jzxh159:1308400:1311136 [1] NCCL INFO Channel 05/0 : 1[1] -> 3[3] via P2P/CUMEM
+1: jzxh160:364772:366441 [1] NCCL INFO Channel 06/0 : 5[1] -> 7[3] via P2P/CUMEM
+2: jzxh161:3283536:3285122 [0] NCCL INFO Channel 11/0 : 8[0] -> 10[2] via P2P/CUMEM
+3: jzxh162:3160743:3162426 [1] NCCL INFO Channel 06/0 : 13[1] -> 15[3] via P2P/CUMEM
+2: jzxh161:3283537:3285123 [1] NCCL INFO Channel 05/0 : 9[1] -> 11[3] via P2P/CUMEM
+3: jzxh162:3160742:3162428 [0] NCCL INFO Channel 10/0 : 12[0] -> 14[2] via P2P/CUMEM
+1: jzxh160:364771:366438 [0] NCCL INFO Channel 03/0 : 4[0] -> 6[2] via P2P/CUMEM
+0: jzxh159:1308399:1311133 [0] NCCL INFO Channel 13/0 : 0[0] -> 2[2] via P2P/CUMEM
+1: jzxh160:364772:366441 [1] NCCL INFO Channel 10/0 : 5[1] -> 7[3] via P2P/CUMEM
+3: jzxh162:3160742:3162428 [0] NCCL INFO Channel 11/0 : 12[0] -> 14[2] via P2P/CUMEM
+2: jzxh161:3283536:3285122 [0] NCCL INFO Channel 13/0 : 8[0] -> 10[2] via P2P/CUMEM
+0: jzxh159:1308400:1311136 [1] NCCL INFO Channel 06/0 : 1[1] -> 3[3] via P2P/CUMEM
+1: jzxh160:364771:366438 [0] NCCL INFO Channel 05/0 : 4[0] -> 6[2] via P2P/CUMEM
+3: jzxh162:3160743:3162426 [1] NCCL INFO Channel 10/0 : 13[1] -> 15[3] via P2P/CUMEM
+0: jzxh159:1308399:1311133 [0] NCCL INFO Channel 14/0 : 0[0] -> 2[2] via P2P/CUMEM
+2: jzxh161:3283537:3285123 [1] NCCL INFO Channel 06/0 : 9[1] -> 11[3] via P2P/CUMEM
+0: jzxh159:1308400:1311136 [1] NCCL INFO Channel 10/0 : 1[1] -> 3[3] via P2P/CUMEM
+1: jzxh160:364772:366441 [1] NCCL INFO Channel 11/0 : 5[1] -> 7[3] via P2P/CUMEM
+3: jzxh162:3160743:3162426 [1] NCCL INFO Channel 11/0 : 13[1] -> 15[3] via P2P/CUMEM
+1: jzxh160:364771:366438 [0] NCCL INFO Channel 06/0 : 4[0] -> 6[2] via P2P/CUMEM
+2: jzxh161:3283536:3285122 [0] NCCL INFO Channel 14/0 : 8[0] -> 10[2] via P2P/CUMEM
+2: jzxh161:3283537:3285123 [1] NCCL INFO Channel 10/0 : 9[1] -> 11[3] via P2P/CUMEM
+3: jzxh162:3160742:3162428 [0] NCCL INFO Channel 13/0 : 12[0] -> 14[2] via P2P/CUMEM
+0: jzxh159:1308400:1311136 [1] NCCL INFO Channel 11/0 : 1[1] -> 3[3] via P2P/CUMEM
+1: jzxh160:364771:366438 [0] NCCL INFO Channel 10/0 : 4[0] -> 6[2] via P2P/CUMEM
+3: jzxh162:3160742:3162428 [0] NCCL INFO Channel 14/0 : 12[0] -> 14[2] via P2P/CUMEM
+2: jzxh161:3283537:3285123 [1] NCCL INFO Channel 11/0 : 9[1] -> 11[3] via P2P/CUMEM
+3: jzxh162:3160743:3162426 [1] NCCL INFO Channel 13/0 : 13[1] -> 15[3] via P2P/CUMEM
+2: jzxh161:3283537:3285123 [1] NCCL INFO Channel 13/0 : 9[1] -> 11[3] via P2P/CUMEM
+0: jzxh159:1308400:1311136 [1] NCCL INFO Channel 13/0 : 1[1] -> 3[3] via P2P/CUMEM
+1: jzxh160:364771:366438 [0] NCCL INFO Channel 11/0 : 4[0] -> 6[2] via P2P/CUMEM
+1: jzxh160:364771:366438 [0] NCCL INFO Channel 13/0 : 4[0] -> 6[2] via P2P/CUMEM
+2: jzxh161:3283537:3285123 [1] NCCL INFO Channel 14/0 : 9[1] -> 11[3] via P2P/CUMEM
+3: jzxh162:3160743:3162426 [1] NCCL INFO Channel 14/0 : 13[1] -> 15[3] via P2P/CUMEM
+0: jzxh159:1308400:1311136 [1] NCCL INFO Channel 14/0 : 1[1] -> 3[3] via P2P/CUMEM
+1: jzxh160:364771:366438 [0] NCCL INFO Channel 14/0 : 4[0] -> 6[2] via P2P/CUMEM
+3: jzxh162:3160742:3162428 [0] NCCL INFO Channel 04/0 : 12[0] -> 15[3] via P2P/CUMEM
+0: jzxh159:1308399:1311133 [0] NCCL INFO Channel 04/0 : 0[0] -> 3[3] via P2P/CUMEM
+3: jzxh162:3160744:3162427 [2] NCCL INFO Channel 02/0 : 10[2] -> 14[2] [receive] via NET/IB/2/GDRDMA
+3: jzxh162:3160744:3162427 [2] NCCL INFO Channel 06/0 : 10[2] -> 14[2] [receive] via NET/IB/2/GDRDMA
+3: jzxh162:3160742:3162428 [0] NCCL INFO Channel 12/0 : 12[0] -> 15[3] via P2P/CUMEM
+1: jzxh160:364771:366438 [0] NCCL INFO Channel 04/0 : 4[0] -> 7[3] via P2P/CUMEM
+0: jzxh159:1308400:1311136 [1] NCCL INFO Channel 09/0 : 1[1] -> 5[1] [send] via NET/IB/1/GDRDMA
+0: jzxh159:1308401:1311134 [2] NCCL INFO Channel 10/0 : 2[2] -> 6[2] [send] via NET/IB/2/GDRDMA
+0: jzxh159:1308401:1311134 [2] NCCL INFO Channel 14/0 : 2[2] -> 6[2] [send] via NET/IB/2/GDRDMA
+0: jzxh159:1308400:1311136 [1] NCCL INFO Channel 13/0 : 1[1] -> 5[1] [send] via NET/IB/1/GDRDMA
+1: jzxh160:364773:366439 [2] NCCL INFO Channel 10/0 : 2[2] -> 6[2] [receive] via NET/IB/2/GDRDMA
+1: jzxh160:364772:366441 [1] NCCL INFO Channel 13/0 : 5[1] -> 7[3] via P2P/CUMEM
+2: jzxh161:3283537:3285123 [1] NCCL INFO Channel 01/0 : 5[1] -> 9[1] [receive] via NET/IB/1/GDRDMA
+3: jzxh162:3160743:3162426 [1] NCCL INFO Channel 01/0 : 9[1] -> 13[1] [receive] via NET/IB/1/GDRDMA
+1: jzxh160:364773:366439 [2] NCCL INFO Channel 14/0 : 2[2] -> 6[2] [receive] via NET/IB/2/GDRDMA
+1: jzxh160:364771:366438 [0] NCCL INFO Channel 12/0 : 4[0] -> 7[3] via P2P/CUMEM
+3: jzxh162:3160743:3162426 [1] NCCL INFO Channel 05/0 : 9[1] -> 13[1] [receive] via NET/IB/1/GDRDMA
+0: jzxh159:1308399:1311133 [0] NCCL INFO Channel 12/0 : 0[0] -> 3[3] via P2P/CUMEM
+1: jzxh160:364773:366439 [2] NCCL INFO Channel 02/0 : 6[2] -> 10[2] [send] via NET/IB/2/GDRDMA
+1: jzxh160:364772:366441 [1] NCCL INFO Channel 14/0 : 5[1] -> 7[3] via P2P/CUMEM
+1: jzxh160:364773:366439 [2] NCCL INFO Channel 06/0 : 6[2] -> 10[2] [send] via NET/IB/2/GDRDMA
+1: jzxh160:364773:366439 [2] NCCL INFO Channel 10/0 : 6[2] -> 10[2] [send] via NET/IB/2/GDRDMA
+2: jzxh161:3283537:3285123 [1] NCCL INFO Channel 05/0 : 5[1] -> 9[1] [receive] via NET/IB/1/GDRDMA
+0: jzxh159:1308401:1311134 [2] NCCL INFO Channel 02/0 : 10[2] -> 2[2] [receive] via NET/IB/2/GDRDMA
+1: jzxh160:364773:366439 [2] NCCL INFO Channel 14/0 : 6[2] -> 10[2] [send] via NET/IB/2/GDRDMA
+2: jzxh161:3283537:3285123 [1] NCCL INFO Channel 09/0 : 5[1] -> 9[1] [receive] via NET/IB/1/GDRDMA
+2: jzxh161:3283537:3285123 [1] NCCL INFO Channel 13/0 : 5[1] -> 9[1] [receive] via NET/IB/1/GDRDMA
+2: jzxh161:3283536:3285122 [0] NCCL INFO Channel 04/0 : 8[0] -> 11[3] via P2P/CUMEM
+2: jzxh161:3283537:3285123 [1] NCCL INFO Channel 01/0 : 9[1] -> 13[1] [send] via NET/IB/1/GDRDMA
+3: jzxh162:3160745:3162425 [3] NCCL INFO Channel 03/0 : 11[3] -> 15[3] [receive] via NET/IB/3/GDRDMA
+3: jzxh162:3160742:3162428 [0] NCCL INFO Channel 00/0 : 8[0] -> 12[0] [receive] via NET/IB/0/GDRDMA
+0: jzxh159:1308402:1311135 [3] NCCL INFO Channel 11/0 : 3[3] -> 7[3] [send] via NET/IB/3/GDRDMA
+0: jzxh159:1308402:1311135 [3] NCCL INFO Channel 15/0 : 3[3] -> 7[3] [send] via NET/IB/3/GDRDMA
+2: jzxh161:3283537:3285123 [1] NCCL INFO Channel 05/0 : 9[1] -> 13[1] [send] via NET/IB/1/GDRDMA
+3: jzxh162:3160745:3162425 [3] NCCL INFO Channel 07/0 : 11[3] -> 15[3] [receive] via NET/IB/3/GDRDMA
+3: jzxh162:3160742:3162428 [0] NCCL INFO Channel 04/0 : 8[0] -> 12[0] [receive] via NET/IB/0/GDRDMA
+0: jzxh159:1308401:1311134 [2] NCCL INFO Channel 06/0 : 10[2] -> 2[2] [receive] via NET/IB/2/GDRDMA
+2: jzxh161:3283538:3285121 [2] NCCL INFO Channel 02/0 : 6[2] -> 10[2] [receive] via NET/IB/2/GDRDMA
+3: jzxh162:3160743:3162426 [1] NCCL INFO Channel 09/0 : 5[1] -> 13[1] [receive] via NET/IB/1/GDRDMA
+2: jzxh161:3283536:3285122 [0] NCCL INFO Channel 12/0 : 8[0] -> 11[3] via P2P/CUMEM
+2: jzxh161:3283538:3285121 [2] NCCL INFO Channel 06/0 : 6[2] -> 10[2] [receive] via NET/IB/2/GDRDMA
+2: jzxh161:3283538:3285121 [2] NCCL INFO Channel 10/0 : 6[2] -> 10[2] [receive] via NET/IB/2/GDRDMA
+2: jzxh161:3283538:3285121 [2] NCCL INFO Channel 14/0 : 6[2] -> 10[2] [receive] via NET/IB/2/GDRDMA
+1: jzxh160:364772:366441 [1] NCCL INFO Channel 09/0 : 1[1] -> 5[1] [receive] via NET/IB/1/GDRDMA
+2: jzxh161:3283538:3285121 [2] NCCL INFO Channel 02/0 : 10[2] -> 14[2] [send] via NET/IB/2/GDRDMA
+0: jzxh159:1308401:1311134 [2] NCCL INFO Channel 02/0 : 2[2] -> 10[2] [send] via NET/IB/2/GDRDMA
+0: jzxh159:1308399:1311133 [0] NCCL INFO Channel 08/0 : 0[0] -> 4[0] [send] via NET/IB/0/GDRDMA
+0: jzxh159:1308401:1311134 [2] NCCL INFO Channel 06/0 : 2[2] -> 10[2] [send] via NET/IB/2/GDRDMA
+0: jzxh159:1308399:1311133 [0] NCCL INFO Channel 12/0 : 0[0] -> 4[0] [send] via NET/IB/0/GDRDMA
+1: jzxh160:364772:366441 [1] NCCL INFO Channel 13/0 : 1[1] -> 5[1] [receive] via NET/IB/1/GDRDMA
+2: jzxh161:3283538:3285121 [2] NCCL INFO Channel 06/0 : 10[2] -> 14[2] [send] via NET/IB/2/GDRDMA
+1: jzxh160:364772:366441 [1] NCCL INFO Channel 01/0 : 5[1] -> 9[1] [send] via NET/IB/1/GDRDMA
+1: jzxh160:364773:366439 [2] NCCL INFO Channel 10/0 : 14[2] -> 6[2] [receive] via NET/IB/2/GDRDMA
+1: jzxh160:364772:366441 [1] NCCL INFO Channel 05/0 : 5[1] -> 9[1] [send] via NET/IB/1/GDRDMA
+1: jzxh160:364773:366439 [2] NCCL INFO Channel 14/0 : 14[2] -> 6[2] [receive] via NET/IB/2/GDRDMA
+2: jzxh161:3283539:3285120 [3] NCCL INFO Channel 03/0 : 7[3] -> 11[3] [receive] via NET/IB/3/GDRDMA
+2: jzxh161:3283536:3285122 [0] NCCL INFO Channel 00/0 : 4[0] -> 8[0] [receive] via NET/IB/0/GDRDMA
+2: jzxh161:3283539:3285120 [3] NCCL INFO Channel 07/0 : 7[3] -> 11[3] [receive] via NET/IB/3/GDRDMA
+2: jzxh161:3283536:3285122 [0] NCCL INFO Channel 04/0 : 4[0] -> 8[0] [receive] via NET/IB/0/GDRDMA
+2: jzxh161:3283539:3285120 [3] NCCL INFO Channel 11/0 : 7[3] -> 11[3] [receive] via NET/IB/3/GDRDMA
+2: jzxh161:3283538:3285121 [2] NCCL INFO Channel 02/0 : 2[2] -> 10[2] [receive] via NET/IB/2/GDRDMA
+2: jzxh161:3283536:3285122 [0] NCCL INFO Channel 08/0 : 4[0] -> 8[0] [receive] via NET/IB/0/GDRDMA
+2: jzxh161:3283539:3285120 [3] NCCL INFO Channel 15/0 : 7[3] -> 11[3] [receive] via NET/IB/3/GDRDMA
+0: jzxh159:1308400:1311136 [1] NCCL INFO Channel 01/0 : 9[1] -> 1[1] [receive] via NET/IB/1/GDRDMA
+2: jzxh161:3283538:3285121 [2] NCCL INFO Channel 06/0 : 2[2] -> 10[2] [receive] via NET/IB/2/GDRDMA
+1: jzxh160:364772:366441 [1] NCCL INFO Channel 09/0 : 5[1] -> 9[1] [send] via NET/IB/1/GDRDMA
+2: jzxh161:3283536:3285122 [0] NCCL INFO Channel 12/0 : 4[0] -> 8[0] [receive] via NET/IB/0/GDRDMA
+1: jzxh160:364773:366439 [2] NCCL INFO Channel 10/0 : 6[2] -> 14[2] [send] via NET/IB/2/GDRDMA
+2: jzxh161:3283539:3285120 [3] NCCL INFO Channel 03/0 : 11[3] -> 15[3] [send] via NET/IB/3/GDRDMA
+1: jzxh160:364772:366441 [1] NCCL INFO Channel 13/0 : 5[1] -> 9[1] [send] via NET/IB/1/GDRDMA
+2: jzxh161:3283538:3285121 [2] NCCL INFO Channel 02/0 : 10[2] -> 2[2] [send] via NET/IB/2/GDRDMA
+1: jzxh160:364773:366439 [2] NCCL INFO Channel 14/0 : 6[2] -> 14[2] [send] via NET/IB/2/GDRDMA
+2: jzxh161:3283536:3285122 [0] NCCL INFO Channel 00/0 : 8[0] -> 12[0] [send] via NET/IB/0/GDRDMA
+0: jzxh159:1308401:1311134 [2] NCCL INFO Channel 10/0 : 6[2] -> 2[2] [receive] via NET/IB/2/GDRDMA
+1: jzxh160:364774:366440 [3] NCCL INFO Channel 11/0 : 3[3] -> 7[3] [receive] via NET/IB/3/GDRDMA
+2: jzxh161:3283539:3285120 [3] NCCL INFO Channel 07/0 : 11[3] -> 15[3] [send] via NET/IB/3/GDRDMA
+0: jzxh159:1308400:1311136 [1] NCCL INFO Channel 05/0 : 9[1] -> 1[1] [receive] via NET/IB/1/GDRDMA
+1: jzxh160:364771:366438 [0] NCCL INFO Channel 08/0 : 0[0] -> 4[0] [receive] via NET/IB/0/GDRDMA
+2: jzxh161:3283538:3285121 [2] NCCL INFO Channel 06/0 : 10[2] -> 2[2] [send] via NET/IB/2/GDRDMA
+1: jzxh160:364774:366440 [3] NCCL INFO Channel 15/0 : 3[3] -> 7[3] [receive] via NET/IB/3/GDRDMA
+1: jzxh160:364771:366438 [0] NCCL INFO Channel 12/0 : 0[0] -> 4[0] [receive] via NET/IB/0/GDRDMA
+2: jzxh161:3283536:3285122 [0] NCCL INFO Channel 04/0 : 8[0] -> 12[0] [send] via NET/IB/0/GDRDMA
+1: jzxh160:364774:366440 [3] NCCL INFO Channel 03/0 : 7[3] -> 11[3] [send] via NET/IB/3/GDRDMA
+2: jzxh161:3283538:3285121 [2] NCCL INFO Channel 02/0 : 14[2] -> 10[2] [receive] via NET/IB/2/GDRDMA
+2: jzxh161:3283537:3285123 [1] NCCL INFO Channel 01/0 : 1[1] -> 9[1] [receive] via NET/IB/1/GDRDMA
+3: jzxh162:3160745:3162425 [3] NCCL INFO Channel 11/0 : 7[3] -> 15[3] [receive] via NET/IB/3/GDRDMA
+2: jzxh161:3283538:3285121 [2] NCCL INFO Channel 06/0 : 14[2] -> 10[2] [receive] via NET/IB/2/GDRDMA
+2: jzxh161:3283537:3285123 [1] NCCL INFO Channel 05/0 : 1[1] -> 9[1] [receive] via NET/IB/1/GDRDMA
+0: jzxh159:1308400:1311136 [1] NCCL INFO Channel 01/0 : 1[1] -> 9[1] [send] via NET/IB/1/GDRDMA
+0: jzxh159:1308401:1311134 [2] NCCL INFO Channel 14/0 : 6[2] -> 2[2] [receive] via NET/IB/2/GDRDMA
+0: jzxh159:1308400:1311136 [1] NCCL INFO Channel 05/0 : 1[1] -> 9[1] [send] via NET/IB/1/GDRDMA
+0: jzxh159:1308402:1311135 [3] NCCL INFO Channel 03/0 : 11[3] -> 3[3] [receive] via NET/IB/3/GDRDMA
+1: jzxh160:364771:366438 [0] NCCL INFO Channel 00/0 : 4[0] -> 8[0] [send] via NET/IB/0/GDRDMA
+1: jzxh160:364774:366440 [3] NCCL INFO Channel 07/0 : 7[3] -> 11[3] [send] via NET/IB/3/GDRDMA
+3: jzxh162:3160745:3162425 [3] NCCL INFO Channel 15/0 : 7[3] -> 15[3] [receive] via NET/IB/3/GDRDMA
+1: jzxh160:364772:366441 [1] NCCL INFO Channel 09/0 : 13[1] -> 5[1] [receive] via NET/IB/1/GDRDMA
+3: jzxh162:3160745:3162425 [3] NCCL INFO Channel 11/0 : 15[3] -> 7[3] [send] via NET/IB/3/GDRDMA
+1: jzxh160:364774:366440 [3] NCCL INFO Channel 11/0 : 7[3] -> 11[3] [send] via NET/IB/3/GDRDMA
+1: jzxh160:364771:366438 [0] NCCL INFO Channel 04/0 : 4[0] -> 8[0] [send] via NET/IB/0/GDRDMA
+3: jzxh162:3160745:3162425 [3] NCCL INFO Channel 15/0 : 15[3] -> 7[3] [send] via NET/IB/3/GDRDMA
+1: jzxh160:364772:366441 [1] NCCL INFO Channel 13/0 : 13[1] -> 5[1] [receive] via NET/IB/1/GDRDMA
+0: jzxh159:1308402:1311135 [3] NCCL INFO Channel 07/0 : 11[3] -> 3[3] [receive] via NET/IB/3/GDRDMA
+0: jzxh159:1308399:1311133 [0] NCCL INFO Channel 00/0 : 8[0] -> 0[0] [receive] via NET/IB/0/GDRDMA
+1: jzxh160:364774:366440 [3] NCCL INFO Channel 15/0 : 7[3] -> 11[3] [send] via NET/IB/3/GDRDMA
+1: jzxh160:364772:366441 [1] NCCL INFO Channel 09/0 : 5[1] -> 13[1] [send] via NET/IB/1/GDRDMA
+1: jzxh160:364771:366438 [0] NCCL INFO Channel 08/0 : 4[0] -> 8[0] [send] via NET/IB/0/GDRDMA
+1: jzxh160:364772:366441 [1] NCCL INFO Channel 13/0 : 5[1] -> 13[1] [send] via NET/IB/1/GDRDMA
+1: jzxh160:364771:366438 [0] NCCL INFO Channel 12/0 : 4[0] -> 8[0] [send] via NET/IB/0/GDRDMA
+2: jzxh161:3283538:3285121 [2] NCCL INFO Channel 02/0 : 10[2] -> 6[2] [send] via NET/IB/2/GDRDMA
+2: jzxh161:3283537:3285123 [1] NCCL INFO Channel 01/0 : 9[1] -> 1[1] [send] via NET/IB/1/GDRDMA
+2: jzxh161:3283537:3285123 [1] NCCL INFO Channel 05/0 : 9[1] -> 1[1] [send] via NET/IB/1/GDRDMA
+2: jzxh161:3283538:3285121 [2] NCCL INFO Channel 06/0 : 10[2] -> 6[2] [send] via NET/IB/2/GDRDMA
+2: jzxh161:3283538:3285121 [2] NCCL INFO Channel 10/0 : 10[2] -> 6[2] [send] via NET/IB/2/GDRDMA
+0: jzxh159:1308400:1311136 [1] NCCL INFO Channel 09/0 : 5[1] -> 1[1] [receive] via NET/IB/1/GDRDMA
+0: jzxh159:1308402:1311135 [3] NCCL INFO Channel 03/0 : 3[3] -> 11[3] [send] via NET/IB/3/GDRDMA
+0: jzxh159:1308399:1311133 [0] NCCL INFO Channel 04/0 : 8[0] -> 0[0] [receive] via NET/IB/0/GDRDMA
+0: jzxh159:1308402:1311135 [3] NCCL INFO Channel 07/0 : 3[3] -> 11[3] [send] via NET/IB/3/GDRDMA
+0: jzxh159:1308400:1311136 [1] NCCL INFO Channel 13/0 : 5[1] -> 1[1] [receive] via NET/IB/1/GDRDMA
+0: jzxh159:1308399:1311133 [0] NCCL INFO Channel 00/0 : 0[0] -> 8[0] [send] via NET/IB/0/GDRDMA
+3: jzxh162:3160743:3162426 [1] NCCL INFO Channel 13/0 : 5[1] -> 13[1] [receive] via NET/IB/1/GDRDMA
+3: jzxh162:3160744:3162427 [2] NCCL INFO Channel 10/0 : 6[2] -> 14[2] [receive] via NET/IB/2/GDRDMA
+3: jzxh162:3160743:3162426 [1] NCCL INFO Channel 09/0 : 13[1] -> 5[1] [send] via NET/IB/1/GDRDMA
+0: jzxh159:1308399:1311133 [0] NCCL INFO Channel 04/0 : 0[0] -> 8[0] [send] via NET/IB/0/GDRDMA
+3: jzxh162:3160744:3162427 [2] NCCL INFO Channel 14/0 : 6[2] -> 14[2] [receive] via NET/IB/2/GDRDMA
+3: jzxh162:3160743:3162426 [1] NCCL INFO Channel 13/0 : 13[1] -> 5[1] [send] via NET/IB/1/GDRDMA
+3: jzxh162:3160744:3162427 [2] NCCL INFO Channel 10/0 : 14[2] -> 6[2] [send] via NET/IB/2/GDRDMA
+1: jzxh160:364774:366440 [3] NCCL INFO Channel 11/0 : 15[3] -> 7[3] [receive] via NET/IB/3/GDRDMA
+2: jzxh161:3283538:3285121 [2] NCCL INFO Channel 14/0 : 10[2] -> 6[2] [send] via NET/IB/2/GDRDMA
+1: jzxh160:364774:366440 [3] NCCL INFO Channel 15/0 : 15[3] -> 7[3] [receive] via NET/IB/3/GDRDMA
+2: jzxh161:3283539:3285120 [3] NCCL INFO Channel 03/0 : 3[3] -> 11[3] [receive] via NET/IB/3/GDRDMA
+1: jzxh160:364774:366440 [3] NCCL INFO Channel 11/0 : 7[3] -> 15[3] [send] via NET/IB/3/GDRDMA
+2: jzxh161:3283539:3285120 [3] NCCL INFO Channel 07/0 : 3[3] -> 11[3] [receive] via NET/IB/3/GDRDMA
+2: jzxh161:3283537:3285123 [1] NCCL INFO Channel 01/0 : 13[1] -> 9[1] [receive] via NET/IB/1/GDRDMA
+2: jzxh161:3283539:3285120 [3] NCCL INFO Channel 03/0 : 11[3] -> 3[3] [send] via NET/IB/3/GDRDMA
+0: jzxh159:1308402:1311135 [3] NCCL INFO Channel 11/0 : 7[3] -> 3[3] [receive] via NET/IB/3/GDRDMA
+3: jzxh162:3160744:3162427 [2] NCCL INFO Channel 14/0 : 14[2] -> 6[2] [send] via NET/IB/2/GDRDMA
+3: jzxh162:3160743:3162426 [1] NCCL INFO Channel 01/0 : 13[1] -> 9[1] [send] via NET/IB/1/GDRDMA
+3: jzxh162:3160743:3162426 [1] NCCL INFO Channel 05/0 : 13[1] -> 9[1] [send] via NET/IB/1/GDRDMA
+3: jzxh162:3160744:3162427 [2] NCCL INFO Channel 02/0 : 14[2] -> 10[2] [send] via NET/IB/2/GDRDMA
+0: jzxh159:1308402:1311135 [3] NCCL INFO Channel 15/0 : 7[3] -> 3[3] [receive] via NET/IB/3/GDRDMA
+2: jzxh161:3283537:3285123 [1] NCCL INFO Channel 05/0 : 13[1] -> 9[1] [receive] via NET/IB/1/GDRDMA
+2: jzxh161:3283539:3285120 [3] NCCL INFO Channel 07/0 : 11[3] -> 3[3] [send] via NET/IB/3/GDRDMA
+2: jzxh161:3283537:3285123 [1] NCCL INFO Channel 01/0 : 9[1] -> 5[1] [send] via NET/IB/1/GDRDMA
+2: jzxh161:3283537:3285123 [1] NCCL INFO Channel 05/0 : 9[1] -> 5[1] [send] via NET/IB/1/GDRDMA
+2: jzxh161:3283537:3285123 [1] NCCL INFO Channel 09/0 : 9[1] -> 5[1] [send] via NET/IB/1/GDRDMA
+2: jzxh161:3283539:3285120 [3] NCCL INFO Channel 03/0 : 15[3] -> 11[3] [receive] via NET/IB/3/GDRDMA
+2: jzxh161:3283537:3285123 [1] NCCL INFO Channel 13/0 : 9[1] -> 5[1] [send] via NET/IB/1/GDRDMA
+2: jzxh161:3283539:3285120 [3] NCCL INFO Channel 07/0 : 15[3] -> 11[3] [receive] via NET/IB/3/GDRDMA
+1: jzxh160:364774:366440 [3] NCCL INFO Channel 15/0 : 7[3] -> 15[3] [send] via NET/IB/3/GDRDMA
+1: jzxh160:364772:366441 [1] NCCL INFO Channel 01/0 : 9[1] -> 5[1] [receive] via NET/IB/1/GDRDMA
+1: jzxh160:364772:366441 [1] NCCL INFO Channel 05/0 : 9[1] -> 5[1] [receive] via NET/IB/1/GDRDMA
+1: jzxh160:364772:366441 [1] NCCL INFO Channel 09/0 : 9[1] -> 5[1] [receive] via NET/IB/1/GDRDMA
+1: jzxh160:364774:366440 [3] NCCL INFO Channel 03/0 : 11[3] -> 7[3] [receive] via NET/IB/3/GDRDMA
+1: jzxh160:364773:366439 [2] NCCL INFO Channel 02/0 : 10[2] -> 6[2] [receive] via NET/IB/2/GDRDMA
+1: jzxh160:364772:366441 [1] NCCL INFO Channel 13/0 : 9[1] -> 5[1] [receive] via NET/IB/1/GDRDMA
+3: jzxh162:3160744:3162427 [2] NCCL INFO Channel 06/0 : 14[2] -> 10[2] [send] via NET/IB/2/GDRDMA
+3: jzxh162:3160745:3162425 [3] NCCL INFO Channel 03/0 : 15[3] -> 11[3] [send] via NET/IB/3/GDRDMA
+1: jzxh160:364774:366440 [3] NCCL INFO Channel 07/0 : 11[3] -> 7[3] [receive] via NET/IB/3/GDRDMA
+3: jzxh162:3160745:3162425 [3] NCCL INFO Channel 07/0 : 15[3] -> 11[3] [send] via NET/IB/3/GDRDMA
+1: jzxh160:364773:366439 [2] NCCL INFO Channel 06/0 : 10[2] -> 6[2] [receive] via NET/IB/2/GDRDMA
+3: jzxh162:3160744:3162427 [2] NCCL INFO Channel 02/0 : 14[2] -> 12[0] via P2P/CUMEM
+1: jzxh160:364772:366441 [1] NCCL INFO Channel 09/0 : 5[1] -> 1[1] [send] via NET/IB/1/GDRDMA
+0: jzxh159:1308401:1311134 [2] NCCL INFO Channel 02/0 : 2[2] -> 0[0] via P2P/CUMEM
+0: jzxh159:1308402:1311135 [3] NCCL INFO Channel 01/0 : 3[3] -> 0[0] via P2P/CUMEM
+1: jzxh160:364774:366440 [3] NCCL INFO Channel 11/0 : 11[3] -> 7[3] [receive] via NET/IB/3/GDRDMA
+1: jzxh160:364773:366439 [2] NCCL INFO Channel 10/0 : 10[2] -> 6[2] [receive] via NET/IB/2/GDRDMA
+2: jzxh161:3283539:3285120 [3] NCCL INFO Channel 03/0 : 11[3] -> 7[3] [send] via NET/IB/3/GDRDMA
+3: jzxh162:3160742:3162428 [0] NCCL INFO Channel 08/0 : 4[0] -> 12[0] [receive] via NET/IB/0/GDRDMA
+3: jzxh162:3160744:3162427 [2] NCCL INFO Channel 03/0 : 14[2] -> 12[0] via P2P/CUMEM
+1: jzxh160:364772:366441 [1] NCCL INFO Channel 13/0 : 5[1] -> 1[1] [send] via NET/IB/1/GDRDMA
+2: jzxh161:3283539:3285120 [3] NCCL INFO Channel 07/0 : 11[3] -> 7[3] [send] via NET/IB/3/GDRDMA
+3: jzxh162:3160745:3162425 [3] NCCL INFO Channel 01/0 : 15[3] -> 12[0] via P2P/CUMEM
+3: jzxh162:3160742:3162428 [0] NCCL INFO Channel 12/0 : 4[0] -> 12[0] [receive] via NET/IB/0/GDRDMA
+1: jzxh160:364774:366440 [3] NCCL INFO Channel 15/0 : 11[3] -> 7[3] [receive] via NET/IB/3/GDRDMA
+2: jzxh161:3283539:3285120 [3] NCCL INFO Channel 11/0 : 11[3] -> 7[3] [send] via NET/IB/3/GDRDMA
+1: jzxh160:364773:366439 [2] NCCL INFO Channel 14/0 : 10[2] -> 6[2] [receive] via NET/IB/2/GDRDMA
+2: jzxh161:3283539:3285120 [3] NCCL INFO Channel 15/0 : 11[3] -> 7[3] [send] via NET/IB/3/GDRDMA
+1: jzxh160:364774:366440 [3] NCCL INFO Channel 11/0 : 7[3] -> 3[3] [send] via NET/IB/3/GDRDMA
+2: jzxh161:3283536:3285122 [0] NCCL INFO Channel 00/0 : 0[0] -> 8[0] [receive] via NET/IB/0/GDRDMA
+1: jzxh160:364771:366438 [0] NCCL INFO Channel 08/0 : 12[0] -> 4[0] [receive] via NET/IB/0/GDRDMA
+1: jzxh160:364773:366439 [2] NCCL INFO Channel 10/0 : 6[2] -> 2[2] [send] via NET/IB/2/GDRDMA
+1: jzxh160:364774:366440 [3] NCCL INFO Channel 15/0 : 7[3] -> 3[3] [send] via NET/IB/3/GDRDMA
+1: jzxh160:364773:366439 [2] NCCL INFO Channel 14/0 : 6[2] -> 2[2] [send] via NET/IB/2/GDRDMA
+1: jzxh160:364771:366438 [0] NCCL INFO Channel 12/0 : 12[0] -> 4[0] [receive] via NET/IB/0/GDRDMA
+1: jzxh160:364771:366438 [0] NCCL INFO Channel 08/0 : 4[0] -> 12[0] [send] via NET/IB/0/GDRDMA
+0: jzxh159:1308399:1311133 [0] NCCL INFO Channel 08/0 : 4[0] -> 0[0] [receive] via NET/IB/0/GDRDMA
+1: jzxh160:364771:366438 [0] NCCL INFO Channel 12/0 : 4[0] -> 12[0] [send] via NET/IB/0/GDRDMA
+1: jzxh160:364773:366439 [2] NCCL INFO Channel 02/0 : 6[2] -> 4[0] via P2P/CUMEM
+1: jzxh160:364774:366440 [3] NCCL INFO Channel 01/0 : 7[3] -> 4[0] via P2P/CUMEM
+3: jzxh162:3160742:3162428 [0] NCCL INFO Channel 08/0 : 12[0] -> 4[0] [send] via NET/IB/0/GDRDMA
+0: jzxh159:1308399:1311133 [0] NCCL INFO Channel 12/0 : 4[0] -> 0[0] [receive] via NET/IB/0/GDRDMA
+1: jzxh160:364771:366438 [0] NCCL INFO Channel 00/0 : 8[0] -> 4[0] [receive] via NET/IB/0/GDRDMA
+3: jzxh162:3160742:3162428 [0] NCCL INFO Channel 12/0 : 12[0] -> 4[0] [send] via NET/IB/0/GDRDMA
+3: jzxh162:3160742:3162428 [0] NCCL INFO Channel 00/0 : 12[0] -> 8[0] [send] via NET/IB/0/GDRDMA
+2: jzxh161:3283536:3285122 [0] NCCL INFO Channel 04/0 : 0[0] -> 8[0] [receive] via NET/IB/0/GDRDMA
+2: jzxh161:3283536:3285122 [0] NCCL INFO Channel 00/0 : 8[0] -> 0[0] [send] via NET/IB/0/GDRDMA
+2: jzxh161:3283538:3285121 [2] NCCL INFO Channel 02/0 : 10[2] -> 8[0] via P2P/CUMEM
+2: jzxh161:3283539:3285120 [3] NCCL INFO Channel 01/0 : 11[3] -> 8[0] via P2P/CUMEM
+2: jzxh161:3283536:3285122 [0] NCCL INFO Channel 04/0 : 8[0] -> 0[0] [send] via NET/IB/0/GDRDMA
+2: jzxh161:3283536:3285122 [0] NCCL INFO Channel 00/0 : 12[0] -> 8[0] [receive] via NET/IB/0/GDRDMA
+2: jzxh161:3283539:3285120 [3] NCCL INFO Channel 04/0 : 11[3] -> 8[0] via P2P/CUMEM
+2: jzxh161:3283536:3285122 [0] NCCL INFO Channel 04/0 : 12[0] -> 8[0] [receive] via NET/IB/0/GDRDMA
+0: jzxh159:1308401:1311134 [2] NCCL INFO Channel 03/0 : 2[2] -> 0[0] via P2P/CUMEM
+0: jzxh159:1308402:1311135 [3] NCCL INFO Channel 04/0 : 3[3] -> 0[0] via P2P/CUMEM
+1: jzxh160:364771:366438 [0] NCCL INFO Channel 04/0 : 8[0] -> 4[0] [receive] via NET/IB/0/GDRDMA
+3: jzxh162:3160744:3162427 [2] NCCL INFO Channel 05/0 : 14[2] -> 12[0] via P2P/CUMEM
+3: jzxh162:3160742:3162428 [0] NCCL INFO Channel 04/0 : 12[0] -> 8[0] [send] via NET/IB/0/GDRDMA
+1: jzxh160:364774:366440 [3] NCCL INFO Channel 04/0 : 7[3] -> 4[0] via P2P/CUMEM
+3: jzxh162:3160745:3162425 [3] NCCL INFO Channel 04/0 : 15[3] -> 12[0] via P2P/CUMEM
+1: jzxh160:364771:366438 [0] NCCL INFO Channel 08/0 : 8[0] -> 4[0] [receive] via NET/IB/0/GDRDMA
+1: jzxh160:364771:366438 [0] NCCL INFO Channel 12/0 : 8[0] -> 4[0] [receive] via NET/IB/0/GDRDMA
+3: jzxh162:3160744:3162427 [2] NCCL INFO Channel 06/0 : 14[2] -> 12[0] via P2P/CUMEM
+0: jzxh159:1308401:1311134 [2] NCCL INFO Channel 05/0 : 2[2] -> 0[0] via P2P/CUMEM
+3: jzxh162:3160745:3162425 [3] NCCL INFO Channel 05/0 : 15[3] -> 12[0] via P2P/CUMEM
+2: jzxh161:3283538:3285121 [2] NCCL INFO Channel 03/0 : 10[2] -> 8[0] via P2P/CUMEM
+2: jzxh161:3283536:3285122 [0] NCCL INFO Channel 00/0 : 8[0] -> 4[0] [send] via NET/IB/0/GDRDMA
+2: jzxh161:3283536:3285122 [0] NCCL INFO Channel 04/0 : 8[0] -> 4[0] [send] via NET/IB/0/GDRDMA
+2: jzxh161:3283536:3285122 [0] NCCL INFO Channel 08/0 : 8[0] -> 4[0] [send] via NET/IB/0/GDRDMA
+2: jzxh161:3283536:3285122 [0] NCCL INFO Channel 12/0 : 8[0] -> 4[0] [send] via NET/IB/0/GDRDMA
+1: jzxh160:364771:366438 [0] NCCL INFO Channel 08/0 : 4[0] -> 0[0] [send] via NET/IB/0/GDRDMA
+1: jzxh160:364773:366439 [2] NCCL INFO Channel 03/0 : 6[2] -> 4[0] via P2P/CUMEM
+1: jzxh160:364771:366438 [0] NCCL INFO Channel 12/0 : 4[0] -> 0[0] [send] via NET/IB/0/GDRDMA
+1: jzxh160:364774:366440 [3] NCCL INFO Channel 05/0 : 7[3] -> 4[0] via P2P/CUMEM
+2: jzxh161:3283538:3285121 [2] NCCL INFO Channel 05/0 : 10[2] -> 8[0] via P2P/CUMEM
+2: jzxh161:3283539:3285120 [3] NCCL INFO Channel 05/0 : 11[3] -> 8[0] via P2P/CUMEM
+0: jzxh159:1308402:1311135 [3] NCCL INFO Channel 05/0 : 3[3] -> 0[0] via P2P/CUMEM
+1: jzxh160:364773:366439 [2] NCCL INFO Channel 05/0 : 6[2] -> 4[0] via P2P/CUMEM
+3: jzxh162:3160744:3162427 [2] NCCL INFO Channel 10/0 : 14[2] -> 12[0] via P2P/CUMEM
+2: jzxh161:3283538:3285121 [2] NCCL INFO Channel 06/0 : 10[2] -> 8[0] via P2P/CUMEM
+2: jzxh161:3283539:3285120 [3] NCCL INFO Channel 06/0 : 11[3] -> 8[0] via P2P/CUMEM
+0: jzxh159:1308401:1311134 [2] NCCL INFO Channel 06/0 : 2[2] -> 0[0] via P2P/CUMEM
+1: jzxh160:364774:366440 [3] NCCL INFO Channel 06/0 : 7[3] -> 4[0] via P2P/CUMEM
+3: jzxh162:3160745:3162425 [3] NCCL INFO Channel 06/0 : 15[3] -> 12[0] via P2P/CUMEM
+0: jzxh159:1308402:1311135 [3] NCCL INFO Channel 06/0 : 3[3] -> 0[0] via P2P/CUMEM
+1: jzxh160:364773:366439 [2] NCCL INFO Channel 06/0 : 6[2] -> 4[0] via P2P/CUMEM
+2: jzxh161:3283539:3285120 [3] NCCL INFO Channel 09/0 : 11[3] -> 8[0] via P2P/CUMEM
+3: jzxh162:3160745:3162425 [3] NCCL INFO Channel 09/0 : 15[3] -> 12[0] via P2P/CUMEM
+3: jzxh162:3160744:3162427 [2] NCCL INFO Channel 11/0 : 14[2] -> 12[0] via P2P/CUMEM
+0: jzxh159:1308401:1311134 [2] NCCL INFO Channel 10/0 : 2[2] -> 0[0] via P2P/CUMEM
+1: jzxh160:364774:366440 [3] NCCL INFO Channel 09/0 : 7[3] -> 4[0] via P2P/CUMEM
+2: jzxh161:3283538:3285121 [2] NCCL INFO Channel 10/0 : 10[2] -> 8[0] via P2P/CUMEM
+0: jzxh159:1308402:1311135 [3] NCCL INFO Channel 09/0 : 3[3] -> 0[0] via P2P/CUMEM
+0: jzxh159:1308401:1311134 [2] NCCL INFO Channel 11/0 : 2[2] -> 0[0] via P2P/CUMEM
+2: jzxh161:3283539:3285120 [3] NCCL INFO Channel 12/0 : 11[3] -> 8[0] via P2P/CUMEM
+3: jzxh162:3160745:3162425 [3] NCCL INFO Channel 12/0 : 15[3] -> 12[0] via P2P/CUMEM
+2: jzxh161:3283538:3285121 [2] NCCL INFO Channel 11/0 : 10[2] -> 8[0] via P2P/CUMEM
+1: jzxh160:364773:366439 [2] NCCL INFO Channel 10/0 : 6[2] -> 4[0] via P2P/CUMEM
+1: jzxh160:364774:366440 [3] NCCL INFO Channel 12/0 : 7[3] -> 4[0] via P2P/CUMEM
+3: jzxh162:3160744:3162427 [2] NCCL INFO Channel 13/0 : 14[2] -> 12[0] via P2P/CUMEM
+0: jzxh159:1308402:1311135 [3] NCCL INFO Channel 12/0 : 3[3] -> 0[0] via P2P/CUMEM
+2: jzxh161:3283538:3285121 [2] NCCL INFO Channel 13/0 : 10[2] -> 8[0] via P2P/CUMEM
+0: jzxh159:1308401:1311134 [2] NCCL INFO Channel 13/0 : 2[2] -> 0[0] via P2P/CUMEM
+3: jzxh162:3160744:3162427 [2] NCCL INFO Channel 14/0 : 14[2] -> 12[0] via P2P/CUMEM
+3: jzxh162:3160745:3162425 [3] NCCL INFO Channel 13/0 : 15[3] -> 12[0] via P2P/CUMEM
+1: jzxh160:364773:366439 [2] NCCL INFO Channel 11/0 : 6[2] -> 4[0] via P2P/CUMEM
+2: jzxh161:3283539:3285120 [3] NCCL INFO Channel 13/0 : 11[3] -> 8[0] via P2P/CUMEM
+1: jzxh160:364774:366440 [3] NCCL INFO Channel 13/0 : 7[3] -> 4[0] via P2P/CUMEM
+0: jzxh159:1308401:1311134 [2] NCCL INFO Channel 14/0 : 2[2] -> 0[0] via P2P/CUMEM
+2: jzxh161:3283539:3285120 [3] NCCL INFO Channel 14/0 : 11[3] -> 8[0] via P2P/CUMEM
+0: jzxh159:1308402:1311135 [3] NCCL INFO Channel 13/0 : 3[3] -> 0[0] via P2P/CUMEM
+2: jzxh161:3283538:3285121 [2] NCCL INFO Channel 14/0 : 10[2] -> 8[0] via P2P/CUMEM
+3: jzxh162:3160745:3162425 [3] NCCL INFO Channel 14/0 : 15[3] -> 12[0] via P2P/CUMEM
+1: jzxh160:364773:366439 [2] NCCL INFO Channel 13/0 : 6[2] -> 4[0] via P2P/CUMEM
+0: jzxh159:1308402:1311135 [3] NCCL INFO Channel 14/0 : 3[3] -> 0[0] via P2P/CUMEM
+1: jzxh160:364774:366440 [3] NCCL INFO Channel 14/0 : 7[3] -> 4[0] via P2P/CUMEM
+1: jzxh160:364773:366439 [2] NCCL INFO Channel 14/0 : 6[2] -> 4[0] via P2P/CUMEM
+3: jzxh162:3160745:3162425 [3] NCCL INFO Channel 02/0 : 15[3] -> 13[1] via P2P/CUMEM
+2: jzxh161:3283539:3285120 [3] NCCL INFO Channel 02/0 : 11[3] -> 9[1] via P2P/CUMEM
+0: jzxh159:1308402:1311135 [3] NCCL INFO Channel 02/0 : 3[3] -> 1[1] via P2P/CUMEM
+3: jzxh162:3160745:3162425 [3] NCCL INFO Channel 03/0 : 15[3] -> 13[1] via P2P/CUMEM
+2: jzxh161:3283539:3285120 [3] NCCL INFO Channel 03/0 : 11[3] -> 9[1] via P2P/CUMEM
+1: jzxh160:364774:366440 [3] NCCL INFO Channel 02/0 : 7[3] -> 5[1] via P2P/CUMEM
+0: jzxh159:1308402:1311135 [3] NCCL INFO Channel 03/0 : 3[3] -> 1[1] via P2P/CUMEM
+3: jzxh162:3160745:3162425 [3] NCCL INFO Channel 05/0 : 15[3] -> 13[1] via P2P/CUMEM
+2: jzxh161:3283539:3285120 [3] NCCL INFO Channel 05/0 : 11[3] -> 9[1] via P2P/CUMEM
+0: jzxh159:1308402:1311135 [3] NCCL INFO Channel 05/0 : 3[3] -> 1[1] via P2P/CUMEM
+1: jzxh160:364774:366440 [3] NCCL INFO Channel 03/0 : 7[3] -> 5[1] via P2P/CUMEM
+2: jzxh161:3283539:3285120 [3] NCCL INFO Channel 06/0 : 11[3] -> 9[1] via P2P/CUMEM
+0: jzxh159:1308402:1311135 [3] NCCL INFO Channel 06/0 : 3[3] -> 1[1] via P2P/CUMEM
+1: jzxh160:364774:366440 [3] NCCL INFO Channel 05/0 : 7[3] -> 5[1] via P2P/CUMEM
+1: jzxh160:364774:366440 [3] NCCL INFO Channel 06/0 : 7[3] -> 5[1] via P2P/CUMEM
+2: jzxh161:3283539:3285120 [3] NCCL INFO Channel 10/0 : 11[3] -> 9[1] via P2P/CUMEM
+0: jzxh159:1308402:1311135 [3] NCCL INFO Channel 10/0 : 3[3] -> 1[1] via P2P/CUMEM
+2: jzxh161:3283539:3285120 [3] NCCL INFO Channel 11/0 : 11[3] -> 9[1] via P2P/CUMEM
+0: jzxh159:1308402:1311135 [3] NCCL INFO Channel 11/0 : 3[3] -> 1[1] via P2P/CUMEM
+1: jzxh160:364774:366440 [3] NCCL INFO Channel 10/0 : 7[3] -> 5[1] via P2P/CUMEM
+2: jzxh161:3283539:3285120 [3] NCCL INFO Channel 13/0 : 11[3] -> 9[1] via P2P/CUMEM
+3: jzxh162:3160745:3162425 [3] NCCL INFO Channel 06/0 : 15[3] -> 13[1] via P2P/CUMEM
+1: jzxh160:364774:366440 [3] NCCL INFO Channel 11/0 : 7[3] -> 5[1] via P2P/CUMEM
+0: jzxh159:1308402:1311135 [3] NCCL INFO Channel 13/0 : 3[3] -> 1[1] via P2P/CUMEM
+2: jzxh161:3283539:3285120 [3] NCCL INFO Channel 14/0 : 11[3] -> 9[1] via P2P/CUMEM
+3: jzxh162:3160745:3162425 [3] NCCL INFO Channel 10/0 : 15[3] -> 13[1] via P2P/CUMEM
+1: jzxh160:364774:366440 [3] NCCL INFO Channel 13/0 : 7[3] -> 5[1] via P2P/CUMEM
+0: jzxh159:1308402:1311135 [3] NCCL INFO Channel 14/0 : 3[3] -> 1[1] via P2P/CUMEM
+3: jzxh162:3160745:3162425 [3] NCCL INFO Channel 11/0 : 15[3] -> 13[1] via P2P/CUMEM
+1: jzxh160:364774:366440 [3] NCCL INFO Channel 14/0 : 7[3] -> 5[1] via P2P/CUMEM
+2: jzxh161:3283539:3285120 [3] NCCL INFO Channel 00/0 : 11[3] -> 10[2] via P2P/CUMEM
+0: jzxh159:1308402:1311135 [3] NCCL INFO Channel 00/0 : 3[3] -> 2[2] via P2P/CUMEM
+3: jzxh162:3160745:3162425 [3] NCCL INFO Channel 13/0 : 15[3] -> 13[1] via P2P/CUMEM
+1: jzxh160:364774:366440 [3] NCCL INFO Channel 00/0 : 7[3] -> 6[2] via P2P/CUMEM
+3: jzxh162:3160745:3162425 [3] NCCL INFO Channel 14/0 : 15[3] -> 13[1] via P2P/CUMEM
+2: jzxh161:3283539:3285120 [3] NCCL INFO Channel 04/0 : 11[3] -> 10[2] via P2P/CUMEM
+0: jzxh159:1308402:1311135 [3] NCCL INFO Channel 04/0 : 3[3] -> 2[2] via P2P/CUMEM
+3: jzxh162:3160745:3162425 [3] NCCL INFO Channel 00/0 : 15[3] -> 14[2] via P2P/CUMEM
+1: jzxh160:364774:366440 [3] NCCL INFO Channel 04/0 : 7[3] -> 6[2] via P2P/CUMEM
+2: jzxh161:3283539:3285120 [3] NCCL INFO Channel 07/0 : 11[3] -> 10[2] via P2P/CUMEM
+0: jzxh159:1308402:1311135 [3] NCCL INFO Channel 07/0 : 3[3] -> 2[2] via P2P/CUMEM
+3: jzxh162:3160745:3162425 [3] NCCL INFO Channel 04/0 : 15[3] -> 14[2] via P2P/CUMEM
+1: jzxh160:364774:366440 [3] NCCL INFO Channel 07/0 : 7[3] -> 6[2] via P2P/CUMEM
+2: jzxh161:3283539:3285120 [3] NCCL INFO Channel 08/0 : 11[3] -> 10[2] via P2P/CUMEM
+0: jzxh159:1308402:1311135 [3] NCCL INFO Channel 08/0 : 3[3] -> 2[2] via P2P/CUMEM
+3: jzxh162:3160745:3162425 [3] NCCL INFO Channel 07/0 : 15[3] -> 14[2] via P2P/CUMEM
+2: jzxh161:3283539:3285120 [3] NCCL INFO Channel 12/0 : 11[3] -> 10[2] via P2P/CUMEM
+1: jzxh160:364774:366440 [3] NCCL INFO Channel 08/0 : 7[3] -> 6[2] via P2P/CUMEM
+3: jzxh162:3160745:3162425 [3] NCCL INFO Channel 08/0 : 15[3] -> 14[2] via P2P/CUMEM
+0: jzxh159:1308402:1311135 [3] NCCL INFO Channel 12/0 : 3[3] -> 2[2] via P2P/CUMEM
+2: jzxh161:3283539:3285120 [3] NCCL INFO Channel 15/0 : 11[3] -> 10[2] via P2P/CUMEM
+1: jzxh160:364774:366440 [3] NCCL INFO Channel 12/0 : 7[3] -> 6[2] via P2P/CUMEM
+0: jzxh159:1308402:1311135 [3] NCCL INFO Channel 15/0 : 3[3] -> 2[2] via P2P/CUMEM
+3: jzxh162:3160745:3162425 [3] NCCL INFO Channel 12/0 : 15[3] -> 14[2] via P2P/CUMEM
+1: jzxh160:364774:366440 [3] NCCL INFO Channel 15/0 : 7[3] -> 6[2] via P2P/CUMEM
+2: jzxh161:3283538:3285121 [2] NCCL INFO Channel 00/0 : 10[2] -> 9[1] via P2P/CUMEM
+3: jzxh162:3160745:3162425 [3] NCCL INFO Channel 15/0 : 15[3] -> 14[2] via P2P/CUMEM
+0: jzxh159:1308401:1311134 [2] NCCL INFO Channel 00/0 : 2[2] -> 1[1] via P2P/CUMEM
+2: jzxh161:3283538:3285121 [2] NCCL INFO Channel 01/0 : 10[2] -> 9[1] via P2P/CUMEM
+0: jzxh159:1308401:1311134 [2] NCCL INFO Channel 01/0 : 2[2] -> 1[1] via P2P/CUMEM
+1: jzxh160:364773:366439 [2] NCCL INFO Channel 00/0 : 6[2] -> 5[1] via P2P/CUMEM
+2: jzxh161:3283538:3285121 [2] NCCL INFO Channel 04/0 : 10[2] -> 9[1] via P2P/CUMEM
+0: jzxh159:1308400:1311136 [1] NCCL INFO Channel 00/0 : 1[1] -> 0[0] via P2P/CUMEM
+0: jzxh159:1308401:1311134 [2] NCCL INFO Channel 04/0 : 2[2] -> 1[1] via P2P/CUMEM
+1: jzxh160:364773:366439 [2] NCCL INFO Channel 01/0 : 6[2] -> 5[1] via P2P/CUMEM
+2: jzxh161:3283537:3285123 [1] NCCL INFO Channel 00/0 : 9[1] -> 8[0] via P2P/CUMEM
+2: jzxh161:3283537:3285123 [1] NCCL INFO Channel 03/0 : 9[1] -> 8[0] via P2P/CUMEM
+2: jzxh161:3283538:3285121 [2] NCCL INFO Channel 07/0 : 10[2] -> 9[1] via P2P/CUMEM
+1: jzxh160:364773:366439 [2] NCCL INFO Channel 04/0 : 6[2] -> 5[1] via P2P/CUMEM
+3: jzxh162:3160744:3162427 [2] NCCL INFO Channel 00/0 : 14[2] -> 13[1] via P2P/CUMEM
+0: jzxh159:1308400:1311136 [1] NCCL INFO Channel 03/0 : 1[1] -> 0[0] via P2P/CUMEM
+0: jzxh159:1308401:1311134 [2] NCCL INFO Channel 07/0 : 2[2] -> 1[1] via P2P/CUMEM
+1: jzxh160:364772:366441 [1] NCCL INFO Channel 00/0 : 5[1] -> 4[0] via P2P/CUMEM
+3: jzxh162:3160743:3162426 [1] NCCL INFO Channel 00/0 : 13[1] -> 12[0] via P2P/CUMEM
+3: jzxh162:3160744:3162427 [2] NCCL INFO Channel 01/0 : 14[2] -> 13[1] via P2P/CUMEM
+2: jzxh161:3283538:3285121 [2] NCCL INFO Channel 08/0 : 10[2] -> 9[1] via P2P/CUMEM
+2: jzxh161:3283537:3285123 [1] NCCL INFO Channel 07/0 : 9[1] -> 8[0] via P2P/CUMEM
+0: jzxh159:1308400:1311136 [1] NCCL INFO Channel 07/0 : 1[1] -> 0[0] via P2P/CUMEM
+1: jzxh160:364773:366439 [2] NCCL INFO Channel 07/0 : 6[2] -> 5[1] via P2P/CUMEM
+1: jzxh160:364772:366441 [1] NCCL INFO Channel 03/0 : 5[1] -> 4[0] via P2P/CUMEM
+0: jzxh159:1308401:1311134 [2] NCCL INFO Channel 08/0 : 2[2] -> 1[1] via P2P/CUMEM
+3: jzxh162:3160743:3162426 [1] NCCL INFO Channel 03/0 : 13[1] -> 12[0] via P2P/CUMEM
+2: jzxh161:3283537:3285123 [1] NCCL INFO Channel 08/0 : 9[1] -> 8[0] via P2P/CUMEM
+2: jzxh161:3283538:3285121 [2] NCCL INFO Channel 09/0 : 10[2] -> 9[1] via P2P/CUMEM
+1: jzxh160:364772:366441 [1] NCCL INFO Channel 07/0 : 5[1] -> 4[0] via P2P/CUMEM
+3: jzxh162:3160744:3162427 [2] NCCL INFO Channel 04/0 : 14[2] -> 13[1] via P2P/CUMEM
+3: jzxh162:3160743:3162426 [1] NCCL INFO Channel 07/0 : 13[1] -> 12[0] via P2P/CUMEM
+0: jzxh159:1308401:1311134 [2] NCCL INFO Channel 09/0 : 2[2] -> 1[1] via P2P/CUMEM
+1: jzxh160:364773:366439 [2] NCCL INFO Channel 08/0 : 6[2] -> 5[1] via P2P/CUMEM
+2: jzxh161:3283538:3285121 [2] NCCL INFO Channel 12/0 : 10[2] -> 9[1] via P2P/CUMEM
+3: jzxh162:3160744:3162427 [2] NCCL INFO Channel 07/0 : 14[2] -> 13[1] via P2P/CUMEM
+0: jzxh159:1308400:1311136 [1] NCCL INFO Channel 08/0 : 1[1] -> 0[0] via P2P/CUMEM
+2: jzxh161:3283537:3285123 [1] NCCL INFO Channel 11/0 : 9[1] -> 8[0] via P2P/CUMEM
+3: jzxh162:3160743:3162426 [1] NCCL INFO Channel 08/0 : 13[1] -> 12[0] via P2P/CUMEM
+1: jzxh160:364773:366439 [2] NCCL INFO Channel 09/0 : 6[2] -> 5[1] via P2P/CUMEM
+1: jzxh160:364772:366441 [1] NCCL INFO Channel 08/0 : 5[1] -> 4[0] via P2P/CUMEM
+0: jzxh159:1308400:1311136 [1] NCCL INFO Channel 11/0 : 1[1] -> 0[0] via P2P/CUMEM
+0: jzxh159:1308401:1311134 [2] NCCL INFO Channel 12/0 : 2[2] -> 1[1] via P2P/CUMEM
+3: jzxh162:3160744:3162427 [2] NCCL INFO Channel 08/0 : 14[2] -> 13[1] via P2P/CUMEM
+3: jzxh162:3160743:3162426 [1] NCCL INFO Channel 11/0 : 13[1] -> 12[0] via P2P/CUMEM
+2: jzxh161:3283537:3285123 [1] NCCL INFO Channel 15/0 : 9[1] -> 8[0] via P2P/CUMEM
+2: jzxh161:3283538:3285121 [2] NCCL INFO Channel 15/0 : 10[2] -> 9[1] via P2P/CUMEM
+1: jzxh160:364772:366441 [1] NCCL INFO Channel 11/0 : 5[1] -> 4[0] via P2P/CUMEM
+1: jzxh160:364773:366439 [2] NCCL INFO Channel 12/0 : 6[2] -> 5[1] via P2P/CUMEM
+3: jzxh162:3160744:3162427 [2] NCCL INFO Channel 09/0 : 14[2] -> 13[1] via P2P/CUMEM
+3: jzxh162:3160743:3162426 [1] NCCL INFO Channel 15/0 : 13[1] -> 12[0] via P2P/CUMEM
+0: jzxh159:1308401:1311134 [2] NCCL INFO Channel 15/0 : 2[2] -> 1[1] via P2P/CUMEM
+0: jzxh159:1308400:1311136 [1] NCCL INFO Channel 15/0 : 1[1] -> 0[0] via P2P/CUMEM
+1: jzxh160:364773:366439 [2] NCCL INFO Channel 15/0 : 6[2] -> 5[1] via P2P/CUMEM
+3: jzxh162:3160744:3162427 [2] NCCL INFO Channel 12/0 : 14[2] -> 13[1] via P2P/CUMEM
+1: jzxh160:364772:366441 [1] NCCL INFO Channel 15/0 : 5[1] -> 4[0] via P2P/CUMEM
+3: jzxh162:3160744:3162427 [2] NCCL INFO Channel 15/0 : 14[2] -> 13[1] via P2P/CUMEM
+0: jzxh159:1308399:1311133 [0] NCCL INFO Connected all trees
+0: jzxh159:1308402:1311135 [3] NCCL INFO Connected all trees
+0: jzxh159:1308400:1311136 [1] NCCL INFO Connected all trees
+0: jzxh159:1308402:1311135 [3] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512
+0: jzxh159:1308402:1311135 [3] NCCL INFO 16 coll channels, 16 collnet channels, 0 nvls channels, 16 p2p channels, 2 p2p channels per peer
+0: jzxh159:1308401:1311134 [2] NCCL INFO Connected all trees
+0: jzxh159:1308400:1311136 [1] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512
+0: jzxh159:1308400:1311136 [1] NCCL INFO 16 coll channels, 16 collnet channels, 0 nvls channels, 16 p2p channels, 2 p2p channels per peer
+0: jzxh159:1308401:1311134 [2] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512
+0: jzxh159:1308401:1311134 [2] NCCL INFO 16 coll channels, 16 collnet channels, 0 nvls channels, 16 p2p channels, 2 p2p channels per peer
+0: jzxh159:1308399:1311133 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512
+0: jzxh159:1308399:1311133 [0] NCCL INFO 16 coll channels, 16 collnet channels, 0 nvls channels, 16 p2p channels, 2 p2p channels per peer
+3: jzxh162:3160742:3162428 [0] NCCL INFO Connected all trees
+3: jzxh162:3160745:3162425 [3] NCCL INFO Connected all trees
+3: jzxh162:3160742:3162428 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512
+3: jzxh162:3160742:3162428 [0] NCCL INFO 16 coll channels, 16 collnet channels, 0 nvls channels, 16 p2p channels, 2 p2p channels per peer
+3: jzxh162:3160745:3162425 [3] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512
+3: jzxh162:3160745:3162425 [3] NCCL INFO 16 coll channels, 16 collnet channels, 0 nvls channels, 16 p2p channels, 2 p2p channels per peer
+3: jzxh162:3160744:3162427 [2] NCCL INFO Connected all trees
+3: jzxh162:3160743:3162426 [1] NCCL INFO Connected all trees
+3: jzxh162:3160744:3162427 [2] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512
+3: jzxh162:3160744:3162427 [2] NCCL INFO 16 coll channels, 16 collnet channels, 0 nvls channels, 16 p2p channels, 2 p2p channels per peer
+3: jzxh162:3160743:3162426 [1] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512
+3: jzxh162:3160743:3162426 [1] NCCL INFO 16 coll channels, 16 collnet channels, 0 nvls channels, 16 p2p channels, 2 p2p channels per peer
+1: jzxh160:364771:366438 [0] NCCL INFO Connected all trees
+2: jzxh161:3283536:3285122 [0] NCCL INFO Connected all trees
+2: jzxh161:3283537:3285123 [1] NCCL INFO Connected all trees
+1: jzxh160:364772:366441 [1] NCCL INFO Connected all trees
+1: jzxh160:364771:366438 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512
+1: jzxh160:364771:366438 [0] NCCL INFO 16 coll channels, 16 collnet channels, 0 nvls channels, 16 p2p channels, 2 p2p channels per peer
+1: jzxh160:364774:366440 [3] NCCL INFO Connected all trees
+1: jzxh160:364772:366441 [1] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512
+1: jzxh160:364772:366441 [1] NCCL INFO 16 coll channels, 16 collnet channels, 0 nvls channels, 16 p2p channels, 2 p2p channels per peer
+1: jzxh160:364774:366440 [3] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512
+1: jzxh160:364774:366440 [3] NCCL INFO 16 coll channels, 16 collnet channels, 0 nvls channels, 16 p2p channels, 2 p2p channels per peer
+1: jzxh160:364773:366439 [2] NCCL INFO Connected all trees
+1: jzxh160:364773:366439 [2] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512
+1: jzxh160:364773:366439 [2] NCCL INFO 16 coll channels, 16 collnet channels, 0 nvls channels, 16 p2p channels, 2 p2p channels per peer
+2: jzxh161:3283536:3285122 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512
+2: jzxh161:3283536:3285122 [0] NCCL INFO 16 coll channels, 16 collnet channels, 0 nvls channels, 16 p2p channels, 2 p2p channels per peer
+2: jzxh161:3283537:3285123 [1] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512
+2: jzxh161:3283537:3285123 [1] NCCL INFO 16 coll channels, 16 collnet channels, 0 nvls channels, 16 p2p channels, 2 p2p channels per peer
+2: jzxh161:3283539:3285120 [3] NCCL INFO Connected all trees
+2: jzxh161:3283539:3285120 [3] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512
+2: jzxh161:3283539:3285120 [3] NCCL INFO 16 coll channels, 16 collnet channels, 0 nvls channels, 16 p2p channels, 2 p2p channels per peer
+2: jzxh161:3283538:3285121 [2] NCCL INFO Connected all trees
+2: jzxh161:3283538:3285121 [2] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512
+2: jzxh161:3283538:3285121 [2] NCCL INFO 16 coll channels, 16 collnet channels, 0 nvls channels, 16 p2p channels, 2 p2p channels per peer
+2: jzxh161:3283537:3285123 [1] NCCL INFO ncclCommInitRank comm 0x147064122c40 rank 9 nranks 16 cudaDev 1 nvmlDev 1 busId 2c000 commId 0x2b939d813fa1b763 - Init COMPLETE
+2: jzxh161:3283539:3285120 [3] NCCL INFO ncclCommInitRank comm 0x14b15c115410 rank 11 nranks 16 cudaDev 3 nvmlDev 3 busId ad000 commId 0x2b939d813fa1b763 - Init COMPLETE
+2: jzxh161:3283536:3285122 [0] NCCL INFO ncclCommInitRank comm 0x14c064143000 rank 8 nranks 16 cudaDev 0 nvmlDev 0 busId 1b000 commId 0x2b939d813fa1b763 - Init COMPLETE
+2: jzxh161:3283538:3285121 [2] NCCL INFO ncclCommInitRank comm 0x14a1d01231c0 rank 10 nranks 16 cudaDev 2 nvmlDev 2 busId 9d000 commId 0x2b939d813fa1b763 - Init COMPLETE
+3: jzxh162:3160744:3162427 [2] NCCL INFO ncclCommInitRank comm 0x151768115ac0 rank 14 nranks 16 cudaDev 2 nvmlDev 2 busId 9d000 commId 0x2b939d813fa1b763 - Init COMPLETE
+3: jzxh162:3160742:3162428 [0] NCCL INFO ncclCommInitRank comm 0x14c174126240 rank 12 nranks 16 cudaDev 0 nvmlDev 0 busId 1b000 commId 0x2b939d813fa1b763 - Init COMPLETE
+3: jzxh162:3160745:3162425 [3] NCCL INFO ncclCommInitRank comm 0x152d14114450 rank 15 nranks 16 cudaDev 3 nvmlDev 3 busId ad000 commId 0x2b939d813fa1b763 - Init COMPLETE
+3: jzxh162:3160743:3162426 [1] NCCL INFO ncclCommInitRank comm 0x1481bc125340 rank 13 nranks 16 cudaDev 1 nvmlDev 1 busId 2c000 commId 0x2b939d813fa1b763 - Init COMPLETE
+1: jzxh160:364774:366440 [3] NCCL INFO ncclCommInitRank comm 0x1457001250c0 rank 7 nranks 16 cudaDev 3 nvmlDev 3 busId ad000 commId 0x2b939d813fa1b763 - Init COMPLETE
+1: jzxh160:364772:366441 [1] NCCL INFO ncclCommInitRank comm 0x144548117000 rank 5 nranks 16 cudaDev 1 nvmlDev 1 busId 2c000 commId 0x2b939d813fa1b763 - Init COMPLETE
+1: jzxh160:364771:366438 [0] NCCL INFO ncclCommInitRank comm 0x15301c1476c0 rank 4 nranks 16 cudaDev 0 nvmlDev 0 busId 1b000 commId 0x2b939d813fa1b763 - Init COMPLETE
+1: jzxh160:364773:366439 [2] NCCL INFO ncclCommInitRank comm 0x149098133e00 rank 6 nranks 16 cudaDev 2 nvmlDev 2 busId 9d000 commId 0x2b939d813fa1b763 - Init COMPLETE
+0: jzxh159:1308402:1311135 [3] NCCL INFO ncclCommInitRank comm 0x154364116f80 rank 3 nranks 16 cudaDev 3 nvmlDev 3 busId ad000 commId 0x2b939d813fa1b763 - Init COMPLETE
+0: jzxh159:1308400:1311136 [1] NCCL INFO ncclCommInitRank comm 0x14c618124680 rank 1 nranks 16 cudaDev 1 nvmlDev 1 busId 2c000 commId 0x2b939d813fa1b763 - Init COMPLETE
+0: jzxh159:1308399:1311133 [0] NCCL INFO ncclCommInitRank comm 0x151f9411a7c0 rank 0 nranks 16 cudaDev 0 nvmlDev 0 busId 1b000 commId 0x2b939d813fa1b763 - Init COMPLETE
+0: jzxh159:1308401:1311134 [2] NCCL INFO ncclCommInitRank comm 0x1463c8123fc0 rank 2 nranks 16 cudaDev 2 nvmlDev 2 busId 9d000 commId 0x2b939d813fa1b763 - Init COMPLETE
+0: {'loss': 0.7405, 'grad_norm': 2.4418386537020766, 'learning_rate': 9.05e-07, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.0}
+0: 
  0%|          | 0/1485 [00:00<?, ?it/s]
  0%|          | 1/1485 [00:54<22:24:25, 54.36s/it]
  0%|          | 2/1485 [01:07<12:21:06, 29.98s/it]
  0%|          | 3/1485 [01:17<8:41:46, 21.12s/it] 
  0%|          | 4/1485 [01:28<6:58:26, 16.95s/it]
  0%|          | 5/1485 [01:38<6:01:02, 14.64s/it]
  0%|          | 6/1485 [01:49<5:26:21, 13.24s/it]
  0%|          | 7/1485 [02:00<5:04:43, 12.37s/it]
  1%|          | 8/1485 [02:10<4:51:08, 11.83s/it]
  1%|          | 9/1485 [02:21<4:41:04, 11.43s/it]
  1%|          | 10/1485 [02:31<4:33:53, 11.14s/it]
                                                   

  1%|          | 10/1485 [02:31<4:33:53, 11.14s/it]
  1%|          | 11/1485 [02:42<4:29:02, 10.95s/it]
  1%|          | 12/1485 [02:53<4:27:06, 10.88s/it]
  1%|          | 13/1485 [03:03<4:24:13, 10.77s/it]
  1%|          | 14/1485 [03:14<4:22:12, 10.70s/it]
  1%|          | 15/1485 [03:24<4:20:20, 10.63s/it]
  1%|          | 16/1485 [03:34<4:19:07, 10.58s/it]
  1%|          | 17/1485 [03:45<4:18:46, 10.58s/it]
+0: {'loss': 0.668, 'grad_norm': 1.3553146159934355, 'learning_rate': 1.3550000000000002e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.01}
+0: {'loss': 0.6251, 'grad_norm': 0.9563773054911652, 'learning_rate': 1.805e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.01}
+0: 
  1%|          | 18/1485 [03:56<4:17:47, 10.54s/it]
  1%|▏         | 19/1485 [04:06<4:16:42, 10.51s/it]
  1%|▏         | 20/1485 [04:16<4:15:45, 10.47s/it]
                                                   

  1%|▏         | 20/1485 [04:16<4:15:45, 10.47s/it]
  1%|▏         | 21/1485 [04:27<4:15:35, 10.48s/it]
  1%|▏         | 22/1485 [04:37<4:15:57, 10.50s/it]
  2%|▏         | 23/1485 [04:48<4:16:00, 10.51s/it]
  2%|▏         | 24/1485 [04:58<4:15:47, 10.50s/it]
  2%|▏         | 25/1485 [05:09<4:17:13, 10.57s/it]
  2%|▏         | 26/1485 [05:20<4:16:32, 10.55s/it]
  2%|▏         | 27/1485 [05:30<4:16:10, 10.54s/it]
  2%|▏         | 28/1485 [05:41<4:16:00, 10.54s/it]
  2%|▏         | 29/1485 [05:52<4:17:49, 10.62s/it]
  2%|▏         | 30/1485 [06:02<4:17:04, 10.60s/it]
                                                   

  2%|▏         | 30/1485 [06:02<4:17:04, 10.60s/it]
  2%|▏         | 31/1485 [06:13<4:16:41, 10.59s/it]
  2%|▏         | 32/1485 [06:23<4:16:06, 10.58s/it]
 
+0: {'loss': 0.6002, 'grad_norm': 0.8403090442561437, 'learning_rate': 2.2550000000000004e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.02}
+0:  2%|▏         | 33/1485 [06:34<4:15:29, 10.56s/it]
  2%|▏         | 34/1485 [06:44<4:14:42, 10.53s/it]
  2%|▏         | 35/1485 [06:55<4:14:40, 10.54s/it]
  2%|▏         | 36/1485 [07:05<4:13:53, 10.51s/it]
  2%|▏         | 37/1485 [07:16<4:12:34, 10.47s/it]
  3%|▎         | 38/1485 [07:26<4:11:43, 10.44s/it]
  3%|▎         | 39/1485 [07:36<4:10:33, 10.40s/it]
  3%|▎         | 40/1485 [07:47<4:09:50, 10.37s/it]
                                                   

  3%|▎         | 40/1485 [07:47<4:09:50, 10.37s/it]
  3%|▎         | 41/1485 [07:57<4:09:18, 10.36s/it]
  3%|▎         | 42/1485 [08:07<4:08:57, 10.35s/it]
  3%|▎         | 43/1485 [08:17<4:08:37, 10.35s/it]
  3%|▎         | 44/1485 [08:28<4:08:10, 10.33s/it]
  3%|▎         | 45/1485 [08:38<4:06:44, 10.28s/it]
  3%|▎         | 46/1485 [08:48<4:05:54, 10.25s/it]
  3%|▎         | 47/1485 [08:58<4:05:12, 10.23s/it]
  3%|▎         | 48/1485 [09:09<4:05:13, 10.24s/it]
  3%|▎         | 49/1485 [09:19<4:05:13, 10.25s/it]
+0: {'loss': 0.582, 'grad_norm': 0.8253938480250915, 'learning_rate': 2.7050000000000004e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.02}
+0: {'loss': 0.5767, 'grad_norm': 0.8008208317777609, 'learning_rate': 3.1550000000000003e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.02}
+0:   3%|▎         | 50/1485 [09:29<4:05:03, 10.25s/it]
                                                   

  3%|▎         | 50/1485 [09:29<4:05:03, 10.25s/it]
  3%|▎         | 51/1485 [09:39<4:04:57, 10.25s/it]
  4%|▎         | 52/1485 [09:50<4:05:38, 10.28s/it]
  4%|▎         | 53/1485 [10:00<4:06:26, 10.33s/it]
  4%|▎         | 54/1485 [10:11<4:06:55, 10.35s/it]
  4%|▎         | 55/1485 [10:21<4:07:07, 10.37s/it]
  4%|▍         | 56/1485 [10:31<4:07:19, 10.38s/it]
  4%|▍         | 57/1485 [10:42<4:06:34, 10.36s/it]
  4%|▍         | 58/1485 [10:52<4:05:57, 10.34s/it]
  4%|▍         | 59/1485 [11:02<4:04:54, 10.30s/it]
  4%|▍         | 60/1485 [11:12<4:04:27, 10.29s/it]
                                                   

  4%|▍         | 60/1485 [11:12<4:04:27, 10.29s/it]
  4%|▍         | 61/1485 [11:23<4:05:03, 10.33s/it]
  4%|▍         | 62/1485 [11:33<4:05:16, 10.34s/it]
  4%|▍         | 63/1485 [11:44<4:05:13, 10.35s/it]
  4%|▍         | 64/1485 [11:54<4:04:22, 10.32s/it]
+0: {'loss': 0.5655, 'grad_norm': 0.7888808944923223, 'learning_rate': 3.6050000000000002e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.03}
+0: {'loss': 0.5466, 'grad_norm': 0.9196647707187424, 'learning_rate': 4.055000000000001e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.03}
+0:   4%|▍         | 65/1485 [12:04<4:04:09, 10.32s/it]
  4%|▍         | 66/1485 [12:15<4:04:03, 10.32s/it]
  5%|▍         | 67/1485 [12:25<4:03:24, 10.30s/it]
  5%|▍         | 68/1485 [12:35<4:03:32, 10.31s/it]
  5%|▍         | 69/1485 [12:45<4:03:11, 10.30s/it]
  5%|▍         | 70/1485 [12:56<4:02:48, 10.30s/it]
                                                   

  5%|▍         | 70/1485 [12:56<4:02:48, 10.30s/it]
  5%|▍         | 71/1485 [13:06<4:03:06, 10.32s/it]
  5%|▍         | 72/1485 [13:16<4:03:25, 10.34s/it]
  5%|▍         | 73/1485 [13:27<4:03:36, 10.35s/it]
  5%|▍         | 74/1485 [13:37<4:03:09, 10.34s/it]
  5%|▌         | 75/1485 [13:47<4:02:31, 10.32s/it]
  5%|▌         | 76/1485 [13:58<4:01:58, 10.30s/it]
  5%|▌         | 77/1485 [14:08<4:01:48, 10.30s/it]
  5%|▌         | 78/1485 [14:18<4:01:05, 10.28s/it]
  5%|▌         | 79/1485 [14:29<4:01:26, 10.30s/it]
  5%|▌         | 80/1485 [14:39<4:00:39, 10.28s/it]
                                                   

+0: {'loss': 0.5532, 'grad_norm': 0.7840832810764089, 'learning_rate': 4.505e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.04}
+0:   5%|▌         | 80/1485 [14:39<4:00:39, 10.28s/it]
  5%|▌         | 81/1485 [14:49<4:01:24, 10.32s/it]
  6%|▌         | 82/1485 [15:00<4:01:58, 10.35s/it]
  6%|▌         | 83/1485 [15:10<4:02:02, 10.36s/it]
  6%|▌         | 84/1485 [15:20<4:01:36, 10.35s/it]
  6%|▌         | 85/1485 [15:31<4:01:17, 10.34s/it]
  6%|▌         | 86/1485 [15:41<4:01:00, 10.34s/it]
  6%|▌         | 87/1485 [15:51<4:00:59, 10.34s/it]
  6%|▌         | 88/1485 [16:02<4:01:11, 10.36s/it]
  6%|▌         | 89/1485 [16:12<4:00:33, 10.34s/it]
  6%|▌         | 90/1485 [16:22<4:00:20, 10.34s/it]
                                                   

  6%|▌         | 90/1485 [16:22<4:00:20, 10.34s/it]
  6%|▌         | 91/1485 [16:33<3:59:55, 10.33s/it]
  6%|▌         | 92/1485 [16:43<3:59:47, 10.33s/it]
  6%|▋         | 93/1485 [16:53<3:58:10, 10.27s/it]
  6%|▋         | 94/1485 [17:03<3:57:16, 10.23s/it]
  6%|▋         | 95/1485 [17:13<3:56:37, 10.21s/it]
  6%|▋         | 96/1485 [17:24<3:55:58, 10.19s/it]
+0: {'loss': 0.533, 'grad_norm': 0.7859317698738013, 'learning_rate': 4.955e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.04}
+0: {'loss': 0.5341, 'grad_norm': 0.8927570676629697, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.04}
+0: 
  7%|▋         | 97/1485 [17:34<3:55:31, 10.18s/it]
  7%|▋         | 98/1485 [17:44<3:55:57, 10.21s/it]
  7%|▋         | 99/1485 [17:54<3:56:33, 10.24s/it]
  7%|▋         | 100/1485 [18:05<3:56:42, 10.25s/it]
                                                    

  7%|▋         | 100/1485 [18:05<3:56:42, 10.25s/it]
  7%|▋         | 101/1485 [18:15<3:57:31, 10.30s/it]
  7%|▋         | 102/1485 [18:25<3:58:24, 10.34s/it]
  7%|▋         | 103/1485 [18:36<3:58:39, 10.36s/it]
  7%|▋         | 104/1485 [18:46<4:00:02, 10.43s/it]
  7%|▋         | 105/1485 [18:57<4:00:13, 10.44s/it]
  7%|▋         | 106/1485 [19:07<4:00:09, 10.45s/it]
  7%|▋         | 107/1485 [19:18<4:00:05, 10.45s/it]
  7%|▋         | 108/1485 [19:28<4:00:20, 10.47s/it]
  7%|▋         | 109/1485 [19:39<4:00:45, 10.50s/it]
  7%|▋         | 110/1485 [19:49<3:59:05, 10.43s/it]
                                                    

  7%|▋         | 110/1485 [19:49<3:59:05, 10.43s/it]
  7%|▋         | 111/1485 [19:59<3:5
+0: {'loss': 0.5221, 'grad_norm': 0.8173820712115876, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.05}
+0: 7:51, 10.39s/it]
  8%|▊         | 112/1485 [20:10<3:57:10, 10.36s/it]
  8%|▊         | 113/1485 [20:20<3:56:54, 10.36s/it]
  8%|▊         | 114/1485 [20:30<3:56:38, 10.36s/it]
  8%|▊         | 115/1485 [20:41<3:55:29, 10.31s/it]
  8%|▊         | 116/1485 [20:51<3:55:02, 10.30s/it]
  8%|▊         | 117/1485 [21:01<3:54:40, 10.29s/it]
  8%|▊         | 118/1485 [21:12<3:55:07, 10.32s/it]
  8%|▊         | 119/1485 [21:22<3:55:48, 10.36s/it]
  8%|▊         | 120/1485 [21:32<3:55:50, 10.37s/it]
                                                    

  8%|▊         | 120/1485 [21:32<3:55:50, 10.37s/it]
  8%|▊         | 121/1485 [21:43<3:54:58, 10.34s/it]
  8%|▊         | 122/1485 [21:53<3:53:56, 10.30s/it]
  8%|▊         | 123/1485 [22:03<3:53:03, 10.27s/it]
  8%|▊         | 124/1485 [22:13<3:52:12, 10.24s/it]
  8%|▊         | 125/1485 [22:23<3:51:35, 10.22s/it]
  8%|▊         | 126/1485 [22:34<3:50:54, 10.19s/it]
  9%|▊         | 127/1485 [22:44<3:50:29, 10.18s/it]
  9%|▊         |
+0: {'loss': 0.519, 'grad_norm': 0.8201569507246312, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.05}
+0: {'loss': 0.5233, 'grad_norm': 0.73588156897563, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.06}
+0:  128/1485 [22:54<3:50:16, 10.18s/it]
  9%|▊         | 129/1485 [23:04<3:49:32, 10.16s/it]
  9%|▉         | 130/1485 [23:14<3:49:24, 10.16s/it]
                                                    

  9%|▉         | 130/1485 [23:14<3:49:24, 10.16s/it]
  9%|▉         | 131/1485 [23:24<3:48:57, 10.15s/it]
  9%|▉         | 132/1485 [23:34<3:47:50, 10.10s/it]
  9%|▉         | 133/1485 [23:44<3:46:52, 10.07s/it]
  9%|▉         | 134/1485 [23:54<3:46:57, 10.08s/it]
  9%|▉         | 135/1485 [24:04<3:45:55, 10.04s/it]
  9%|▉         | 136/1485 [24:14<3:44:46, 10.00s/it]
  9%|▉         | 137/1485 [24:24<3:45:03, 10.02s/it]
  9%|▉         | 138/1485 [24:34<3:43:57,  9.98s/it]
  9%|▉         | 139/1485 [24:44<3:43:27,  9.96s/it]
  9%|▉         | 140/1485 [24:54<3:44:35, 10.02s/it]
                                                    

  9%|▉         | 140/1485 [24:54<3:44:35, 10.02s/it]
  9%|▉         | 141/1485 [25:04<3:44:02, 10.00s/it]
 10%|▉         | 142/1485 [25:14<3:44:26, 10.03s/it]
+0: {'loss': 0.5226, 'grad_norm': 0.7483130061352808, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.06}
+0: 
 10%|▉         | 143/1485 [25:25<3:45:40, 10.09s/it]
 10%|▉         | 144/1485 [25:35<3:47:22, 10.17s/it]
 10%|▉         | 145/1485 [25:45<3:48:22, 10.23s/it]
 10%|▉         | 146/1485 [25:56<3:49:13, 10.27s/it]
 10%|▉         | 147/1485 [26:06<3:48:43, 10.26s/it]
 10%|▉         | 148/1485 [26:16<3:49:27, 10.30s/it]
 10%|█         | 149/1485 [26:27<3:49:20, 10.30s/it]
 10%|█         | 150/1485 [26:37<3:49:04, 10.30s/it]
                                                    

 10%|█         | 150/1485 [26:37<3:49:04, 10.30s/it]
 10%|█         | 151/1485 [26:47<3:47:52, 10.25s/it]
 10%|█         | 152/1485 [26:57<3:45:02, 10.13s/it]
 10%|█         | 153/1485 [27:07<3:43:08, 10.05s/it]
 10%|█         | 154/1485 [27:17<3:41:54, 10.00s/it]
 10%|█         | 155/1485 [27:26<3:40:34,  9.95s/it]
 11%|█         | 156/1485 [27:36<3:40:08,  9.94s/it]
 11%|█         | 157/1485 [27:46<3:40:05,  9.94s/it]
 11%|█         | 158/1485 [27:56<3:40:06,  9.95s/it]
 11%|█         | 159/1485 [28:06
+0: {'loss': 0.5171, 'grad_norm': 0.7447032757755208, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.06}
+0: {'loss': 0.5071, 'grad_norm': 0.7896660296808908, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.07}
+0: <3:40:27,  9.98s/it]
 11%|█         | 160/1485 [28:16<3:41:03, 10.01s/it]
                                                    

 11%|█         | 160/1485 [28:16<3:41:03, 10.01s/it]
 11%|█         | 161/1485 [28:26<3:41:15, 10.03s/it]
 11%|█         | 162/1485 [28:36<3:41:06, 10.03s/it]
 11%|█         | 163/1485 [28:46<3:39:46,  9.97s/it]
 11%|█         | 164/1485 [28:56<3:38:32,  9.93s/it]
 11%|█         | 165/1485 [29:06<3:38:09,  9.92s/it]
 11%|█         | 166/1485 [29:16<3:38:12,  9.93s/it]
 11%|█         | 167/1485 [29:26<3:38:03,  9.93s/it]
 11%|█▏        | 168/1485 [29:36<3:38:04,  9.93s/it]
 11%|█▏        | 169/1485 [29:46<3:39:17, 10.00s/it]
 11%|█▏        | 170/1485 [29:56<3:39:45, 10.03s/it]
                                                    

 11%|█▏        | 170/1485 [29:56<3:39:45, 10.03s/it]
 12%|█▏        | 171/1485 [30:06<3:39:42, 10.03s/it]
 12%|█▏        | 172/1485 [30:16<3:39:35, 10.03s/it]
 12%|█▏        | 173/1485 [30:26<3:39:31, 10.04s/it]
 
+0: {'loss': 0.5103, 'grad_norm': 0.8570943537649154, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.07}
+0: 12%|█▏        | 174/1485 [30:36<3:40:15, 10.08s/it]
 12%|█▏        | 175/1485 [30:47<3:40:21, 10.09s/it]
 12%|█▏        | 176/1485 [30:57<3:40:10, 10.09s/it]
 12%|█▏        | 177/1485 [31:07<3:38:43, 10.03s/it]
 12%|█▏        | 178/1485 [31:16<3:37:48, 10.00s/it]
 12%|█▏        | 179/1485 [31:26<3:36:37,  9.95s/it]
 12%|█▏        | 180/1485 [31:36<3:35:37,  9.91s/it]
                                                    

 12%|█▏        | 180/1485 [31:36<3:35:37,  9.91s/it]
 12%|█▏        | 181/1485 [31:46<3:35:00,  9.89s/it]
 12%|█▏        | 182/1485 [31:56<3:34:20,  9.87s/it]
 12%|█▏        | 183/1485 [32:06<3:33:34,  9.84s/it]
 12%|█▏        | 184/1485 [32:15<3:33:48,  9.86s/it]
 12%|█▏        | 185/1485 [32:25<3:33:45,  9.87s/it]
 13%|█▎        | 186/1485 [32:35<3:33:23,  9.86s/it]
 13%|█▎        | 187/1485 [32:45<3:33:14,  9.86s/it]
 13%|█▎        | 188/1485 [32:55<3:32:42,  9.84s/it]
 13%|█▎        | 189/1485 [33:05<3:32:19,  9.83s/it]
 1
+0: {'loss': 0.4978, 'grad_norm': 0.8180155032453749, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.08}
+0: {'loss': 0.501, 'grad_norm': 0.7340116918790619, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.08}
+0: 3%|█▎        | 190/1485 [33:15<3:33:53,  9.91s/it]
                                                    

 13%|█▎        | 190/1485 [33:15<3:33:53,  9.91s/it]
 13%|█▎        | 191/1485 [33:25<3:33:00,  9.88s/it]
 13%|█▎        | 192/1485 [33:34<3:32:12,  9.85s/it]
 13%|█▎        | 193/1485 [33:44<3:31:44,  9.83s/it]
 13%|█▎        | 194/1485 [33:54<3:31:31,  9.83s/it]
 13%|█▎        | 195/1485 [34:04<3:31:28,  9.84s/it]
 13%|█▎        | 196/1485 [34:14<3:31:22,  9.84s/it]
 13%|█▎        | 197/1485 [34:23<3:31:01,  9.83s/it]
 13%|█▎        | 198/1485 [34:33<3:31:20,  9.85s/it]
 13%|█▎        | 199/1485 [34:43<3:31:06,  9.85s/it]
 13%|█▎        | 200/1485 [34:53<3:31:26,  9.87s/it]
                                                    

 13%|█▎        | 200/1485 [34:53<3:31:26,  9.87s/it]
 14%|█▎        | 201/1485 [35:03<3:31:31,  9.88s/it]
 14%|█▎        | 202/1485 [35:13<3:31:10,  9.88s/it]
 14%|█▎        | 203/1485 [35:23<3:30:40,  9.86s/it]
 14%|�
+0: {'loss': 0.5103, 'grad_norm': 0.827445733545192, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.08}
+0: ��▎        | 204/1485 [35:33<3:30:39,  9.87s/it]
 14%|█▍        | 205/1485 [35:43<3:31:08,  9.90s/it]
 14%|█▍        | 206/1485 [35:52<3:31:14,  9.91s/it]
 14%|█▍        | 207/1485 [36:02<3:30:47,  9.90s/it]
 14%|█▍        | 208/1485 [36:12<3:30:34,  9.89s/it]
 14%|█▍        | 209/1485 [36:22<3:30:32,  9.90s/it]
 14%|█▍        | 210/1485 [36:32<3:30:22,  9.90s/it]
                                                    

 14%|█▍        | 210/1485 [36:32<3:30:22,  9.90s/it]
 14%|█▍        | 211/1485 [36:42<3:30:52,  9.93s/it]
 14%|█▍        | 212/1485 [36:52<3:30:06,  9.90s/it]
 14%|█▍        | 213/1485 [37:02<3:29:18,  9.87s/it]
 14%|█▍        | 214/1485 [37:12<3:29:47,  9.90s/it]
 14%|█▍        | 215/1485 [37:22<3:30:59,  9.97s/it]
 15%|█▍        | 216/1485 [37:32<3:32:06, 10.03s/it]
 15%|█▍        | 217/1485 [37:42<3:32:20, 10.05s/it]
 15%|█▍        | 218/1485 [37:52<3:33:02, 10.09s/it]
 15%|█▍        | 219/1485 [38:02<3:33:30, 10.12s/it]
 15%|�
+0: {'loss': 0.5088, 'grad_norm': 0.78690326557629, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.09}
+0: {'loss': 0.4928, 'grad_norm': 0.9296325820059997, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.09}
+0: �▍        | 220/1485 [38:13<3:33:38, 10.13s/it]
                                                    

 15%|█▍        | 220/1485 [38:13<3:33:38, 10.13s/it]
 15%|█▍        | 221/1485 [38:23<3:33:20, 10.13s/it]
 15%|█▍        | 222/1485 [38:33<3:33:04, 10.12s/it]
 15%|█▌        | 223/1485 [38:43<3:32:40, 10.11s/it]
 15%|█▌        | 224/1485 [38:53<3:32:16, 10.10s/it]
 15%|█▌        | 225/1485 [39:03<3:31:47, 10.08s/it]
 15%|█▌        | 226/1485 [39:13<3:31:05, 10.06s/it]
 15%|█▌        | 227/1485 [39:23<3:30:51, 10.06s/it]
 15%|█▌        | 228/1485 [39:33<3:30:40, 10.06s/it]
 15%|█▌        | 229/1485 [39:43<3:30:54, 10.08s/it]
 15%|█▌        | 230/1485 [39:53<3:30:39, 10.07s/it]
                                                    

 15%|█▌        | 230/1485 [39:53<3:30:39, 10.07s/it]
 16%|█▌        | 231/1485 [40:03<3:30:12, 10.06s/it]
 16%|█▌        | 232/1485 [40:13<3:29:44, 10.04s/it]
 16%|█▌        | 233/1485 [40:24<3:30:25, 10.08s/it]
 16%|█▌
+0: {'loss': 0.5009, 'grad_norm': 0.7879814652644276, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.1}
+0:         | 234/1485 [40:34<3:30:41, 10.10s/it]
 16%|█▌        | 235/1485 [40:44<3:31:21, 10.15s/it]
 16%|█▌        | 236/1485 [40:54<3:31:47, 10.17s/it]
 16%|█▌        | 237/1485 [41:04<3:31:48, 10.18s/it]
 16%|█▌        | 238/1485 [41:14<3:30:57, 10.15s/it]
 16%|█▌        | 239/1485 [41:25<3:30:23, 10.13s/it]
 16%|█▌        | 240/1485 [41:35<3:30:04, 10.12s/it]
                                                    

 16%|█▌        | 240/1485 [41:35<3:30:04, 10.12s/it]
 16%|█▌        | 241/1485 [41:45<3:30:57, 10.17s/it]
 16%|█▋        | 242/1485 [41:55<3:30:44, 10.17s/it]
 16%|█▋        | 243/1485 [42:05<3:31:42, 10.23s/it]
 16%|█▋        | 244/1485 [42:16<3:32:24, 10.27s/it]
 16%|█▋        | 245/1485 [42:26<3:32:57, 10.30s/it]
 17%|█▋        | 246/1485 [42:37<3:33:59, 10.36s/it]
 17%|█▋        | 247/1485 [42:47<3:33:16, 10.34s/it]
 17%|█▋        | 248/1485 [42:57<3:32:32, 10.31s/it]
 17%|█▋        | 249/1485 [43:07<3:30:59, 10.24s/it]
 17%|��▋ 
+0: {'loss': 0.4919, 'grad_norm': 0.840047775545624, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.1}
+0: {'loss': 0.4911, 'grad_norm': 0.7753520422493089, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.11}
+0:        | 250/1485 [43:17<3:29:37, 10.18s/it]
                                                    

 17%|█▋        | 250/1485 [43:17<3:29:37, 10.18s/it]
 17%|█▋        | 251/1485 [43:27<3:28:24, 10.13s/it]
 17%|█▋        | 252/1485 [43:37<3:27:14, 10.08s/it]
 17%|█▋        | 253/1485 [43:47<3:26:31, 10.06s/it]
 17%|█▋        | 254/1485 [43:58<3:27:35, 10.12s/it]
 17%|█▋        | 255/1485 [44:08<3:27:36, 10.13s/it]
 17%|█▋        | 256/1485 [44:18<3:27:23, 10.12s/it]
 17%|█▋        | 257/1485 [44:28<3:26:54, 10.11s/it]
 17%|█▋        | 258/1485 [44:38<3:26:53, 10.12s/it]
 17%|█▋        | 259/1485 [44:48<3:26:49, 10.12s/it]
 18%|█▊        | 260/1485 [44:58<3:27:36, 10.17s/it]
                                                    

 18%|█▊        | 260/1485 [44:59<3:27:36, 10.17s/it]
 18%|█▊        | 261/1485 [45:09<3:27:22, 10.17s/it]
 18%|█▊        | 262/1485 [45:19<3:26:42, 10.14s/it]
 18%|█▊        | 263/1485 [45:29<3:26:30, 10.14s/it]
 18%|█▊     
+0: {'loss': 0.5037, 'grad_norm': 0.8322560294119984, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.11}
+0:    | 264/1485 [45:39<3:26:01, 10.12s/it]
 18%|█▊        | 265/1485 [45:49<3:25:47, 10.12s/it]
 18%|█▊        | 266/1485 [45:59<3:25:32, 10.12s/it]
 18%|█▊        | 267/1485 [46:09<3:25:19, 10.11s/it]
 18%|█▊        | 268/1485 [46:20<3:27:18, 10.22s/it]
 18%|█▊        | 269/1485 [46:30<3:27:09, 10.22s/it]
 18%|█▊        | 270/1485 [46:40<3:26:58, 10.22s/it]
                                                    

 18%|█▊        | 270/1485 [46:40<3:26:58, 10.22s/it]
 18%|█▊        | 271/1485 [46:50<3:25:14, 10.14s/it]
 18%|█▊        | 272/1485 [47:00<3:23:49, 10.08s/it]
 18%|█▊        | 273/1485 [47:10<3:22:42, 10.04s/it]
 18%|█▊        | 274/1485 [47:20<3:22:01, 10.01s/it]
 19%|█▊        | 275/1485 [47:30<3:21:55, 10.01s/it]
 19%|█▊        | 276/1485 [47:40<3:21:50, 10.02s/it]
 19%|█▊        | 277/1485 [47:50<3:21:31, 10.01s/it]
 19%|█▊        | 278/1485 [48:00<3:21:40, 10.03s/it]
 19%|█▉        | 279/1485 [48:10<3:21:18, 10.02s/it]
 19%|█▉      
+0: {'loss': 0.4872, 'grad_norm': 0.8239831968097506, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.11}
+0: {'loss': 0.5005, 'grad_norm': 0.8316836944319748, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.12}
+0:   | 280/1485 [48:20<3:20:50, 10.00s/it]
                                                    

 19%|█▉        | 280/1485 [48:20<3:20:50, 10.00s/it]
 19%|█▉        | 281/1485 [48:30<3:21:14, 10.03s/it]
 19%|█▉        | 282/1485 [48:40<3:21:26, 10.05s/it]
 19%|█▉        | 283/1485 [48:50<3:21:30, 10.06s/it]
 19%|█▉        | 284/1485 [49:00<3:21:06, 10.05s/it]
 19%|█▉        | 285/1485 [49:11<3:22:19, 10.12s/it]
 19%|█▉        | 286/1485 [49:21<3:23:50, 10.20s/it]
 19%|█▉        | 287/1485 [49:31<3:23:54, 10.21s/it]
 19%|█▉        | 288/1485 [49:41<3:23:58, 10.22s/it]
 19%|█▉        | 289/1485 [49:52<3:22:57, 10.18s/it]
 20%|█▉        | 290/1485 [50:02<3:22:14, 10.15s/it]
                                                    

 20%|█▉        | 290/1485 [50:02<3:22:14, 10.15s/it]
 20%|█▉        | 291/1485 [50:12<3:21:22, 10.12s/it]
 20%|█▉        | 292/1485 [50:22<3:21:17, 10.12s/it]
 20%|█▉        | 293/1485 [50:32<3:21:25, 10.14s/it]
 20%|█▉        | 
+0: {'loss': 0.4965, 'grad_norm': 0.7751220358811752, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.12}
+0: 294/1485 [50:42<3:20:35, 10.11s/it]
 20%|█▉        | 295/1485 [50:52<3:20:31, 10.11s/it]
 20%|█▉        | 296/1485 [51:03<3:26:53, 10.44s/it]
 20%|██        | 297/1485 [51:14<3:24:53, 10.35s/it]
 20%|██        | 298/1485 [51:24<3:23:34, 10.29s/it]
 20%|██        | 299/1485 [51:34<3:24:24, 10.34s/it]
 20%|██        | 300/1485 [51:44<3:23:16, 10.29s/it]
                                                    

 20%|██        | 300/1485 [51:44<3:23:16, 10.29s/it]
 20%|██        | 301/1485 [51:54<3:22:22, 10.26s/it]
 20%|██        | 302/1485 [52:05<3:21:30, 10.22s/it]
 20%|██        | 303/1485 [52:15<3:21:10, 10.21s/it]
 20%|██        | 304/1485 [52:25<3:20:49, 10.20s/it]
 21%|██        | 305/1485 [52:35<3:20:36, 10.20s/it]
 21%|██        | 306/1485 [52:45<3:20:16, 10.19s/it]
 21%|██        | 307/1485 [52:56<3:19:53, 10.18s/it]
 21%|██        | 308/1485 [53:06<3:19:23, 10.16s/it]
 21%|██        | 309/1485 [53:16<3:19:16, 10.17s/it]
 21%|██        | 3
+0: {'loss': 0.4845, 'grad_norm': 0.7996838695359425, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.13}
+0: {'loss': 0.4989, 'grad_norm': 0.7911831621444595, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.13}
+0: 10/1485 [53:26<3:19:17, 10.18s/it]
                                                    

 21%|██        | 310/1485 [53:26<3:19:17, 10.18s/it]
 21%|██        | 311/1485 [53:36<3:18:26, 10.14s/it]
 21%|██        | 312/1485 [53:46<3:18:59, 10.18s/it]
 21%|██        | 313/1485 [53:57<3:18:56, 10.18s/it]
 21%|██        | 314/1485 [54:07<3:18:41, 10.18s/it]
 21%|██        | 315/1485 [54:17<3:18:07, 10.16s/it]
 21%|██▏       | 316/1485 [54:27<3:17:11, 10.12s/it]
 21%|██▏       | 317/1485 [54:37<3:16:20, 10.09s/it]
 21%|██▏       | 318/1485 [54:47<3:17:06, 10.13s/it]
 21%|██▏       | 319/1485 [54:57<3:16:33, 10.11s/it]
 22%|██▏       | 320/1485 [55:07<3:15:45, 10.08s/it]
                                                    

 22%|██▏       | 320/1485 [55:07<3:15:45, 10.08s/it]
 22%|██▏       | 321/1485 [55:17<3:15:20, 10.07s/it]
 22%|██▏       | 322/1485 [55:27<3:15:44, 10.10s/it]
 22%|██▏       | 323/1485 [55:38<3:16:33, 10.15s/it]
 22%|█
+0: {'loss': 0.4876, 'grad_norm': 0.7828960359172129, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.13}
+0: █▏       | 324/1485 [55:48<3:17:19, 10.20s/it]
 22%|██▏       | 325/1485 [56:00<3:30:07, 10.87s/it]
 22%|██▏       | 326/1485 [56:11<3:25:42, 10.65s/it]
 22%|██▏       | 327/1485 [56:21<3:22:14, 10.48s/it]
 22%|██▏       | 328/1485 [56:31<3:19:29, 10.35s/it]
 22%|██▏       | 329/1485 [56:41<3:18:36, 10.31s/it]
 22%|██▏       | 330/1485 [56:51<3:19:05, 10.34s/it]
                                                    

 22%|██▏       | 330/1485 [56:51<3:19:05, 10.34s/it]
 22%|██▏       | 331/1485 [57:02<3:18:49, 10.34s/it]
 22%|██▏       | 332/1485 [57:12<3:18:55, 10.35s/it]
 22%|██▏       | 333/1485 [57:22<3:18:34, 10.34s/it]
 22%|██▏       | 334/1485 [57:33<3:18:28, 10.35s/it]
 23%|██▎       | 335/1485 [57:43<3:17:49, 10.32s/it]
 23%|██▎       | 336/1485 [57:53<3:17:05, 10.29s/it]
 23%|██▎       | 337/1485 [58:03<3:16:22, 10.26s/it]
 23%|██▎       | 338/1485 [58:14<3:15:57, 10.25s/it]
 23%|██▎       | 339/1485 [5
+0: {'loss': 0.4766, 'grad_norm': 0.749996056005264, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.14}
+0: {'loss': 0.4763, 'grad_norm': 0.8098400158045373, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.14}
+0: 8:24<3:16:56, 10.31s/it]
 23%|██▎       | 340/1485 [58:34<3:16:37, 10.30s/it]
                                                    

 23%|██▎       | 340/1485 [58:34<3:16:37, 10.30s/it]
 23%|██▎       | 341/1485 [58:45<3:16:22, 10.30s/it]
 23%|██▎       | 342/1485 [58:55<3:16:16, 10.30s/it]
 23%|██▎       | 343/1485 [59:05<3:15:44, 10.28s/it]
 23%|██▎       | 344/1485 [59:15<3:15:39, 10.29s/it]
 23%|██▎       | 345/1485 [59:26<3:15:19, 10.28s/it]
 23%|██▎       | 346/1485 [59:36<3:15:17, 10.29s/it]
 23%|██▎       | 347/1485 [59:46<3:15:18, 10.30s/it]
 23%|██▎       | 348/1485 [59:57<3:15:11, 10.30s/it]
 24%|██▎       | 349/1485 [1:00:07<3:14:49, 10.29s/it]
 24%|██▎       | 350/1485 [1:00:17<3:13:57, 10.25s/it]
                                                      

 24%|██▎       | 350/1485 [1:00:17<3:13:57, 10.25s/it]
 24%|██▎       | 351/1485 [1:00:27<3:14:05, 10.27s/it]
 24%|██▎       | 352/1485 [1:00:38<3:13:30, 10.25
+0: {'loss': 0.4729, 'grad_norm': 0.8035953772449974, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.15}
+0: s/it]
 24%|██▍       | 353/1485 [1:00:48<3:13:23, 10.25s/it]
 24%|██▍       | 354/1485 [1:00:58<3:13:11, 10.25s/it]
 24%|██▍       | 355/1485 [1:01:08<3:12:46, 10.24s/it]
 24%|██▍       | 356/1485 [1:01:19<3:12:45, 10.24s/it]
 24%|██▍       | 357/1485 [1:01:29<3:11:41, 10.20s/it]
 24%|██▍       | 358/1485 [1:01:39<3:10:11, 10.13s/it]
 24%|██▍       | 359/1485 [1:01:49<3:09:42, 10.11s/it]
 24%|██▍       | 360/1485 [1:01:59<3:08:53, 10.07s/it]
                                                      

 24%|██▍       | 360/1485 [1:01:59<3:08:53, 10.07s/it]
 24%|██▍       | 361/1485 [1:02:09<3:08:17, 10.05s/it]
 24%|██▍       | 362/1485 [1:02:19<3:07:53, 10.04s/it]
 24%|██▍       | 363/1485 [1:02:29<3:07:29, 10.03s/it]
 25%|██▍       | 364/1485 [1:02:39<3:07:11, 10.02s/it]
 25%|██▍       | 365/1485 [1:02:49<3:06:43, 10.00s/it]
 25%|██▍       | 366/1485 [1:02:59<3:07:26, 10.05s/it]
 25%|██▍       | 367/1485 [1:03:09<3:07:0
+0: {'loss': 0.4846, 'grad_norm': 0.7699520880755788, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.15}
+0: {'loss': 0.488, 'grad_norm': 0.7667273109812945, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.15}
+0: 9, 10.04s/it]
 25%|██▍       | 368/1485 [1:03:20<3:15:44, 10.51s/it]
 25%|██▍       | 369/1485 [1:03:30<3:12:16, 10.34s/it]
 25%|██▍       | 370/1485 [1:03:41<3:12:07, 10.34s/it]
                                                      

 25%|██▍       | 370/1485 [1:03:41<3:12:07, 10.34s/it]
 25%|██▍       | 371/1485 [1:03:51<3:11:27, 10.31s/it]
 25%|██▌       | 372/1485 [1:04:01<3:09:54, 10.24s/it]
 25%|██▌       | 373/1485 [1:04:11<3:07:35, 10.12s/it]
 25%|██▌       | 374/1485 [1:04:21<3:06:09, 10.05s/it]
 25%|██▌       | 375/1485 [1:04:31<3:04:55, 10.00s/it]
 25%|██▌       | 376/1485 [1:04:40<3:03:54,  9.95s/it]
 25%|██▌       | 377/1485 [1:04:50<3:03:25,  9.93s/it]
 25%|██▌       | 378/1485 [1:05:01<3:04:54, 10.02s/it]
 26%|██▌       | 379/1485 [1:05:11<3:05:03, 10.04s/it]
 26%|██▌       | 380/1485 [1:05:21<3:04:52, 10.04s/it]
                                                      

 26%|██▌       | 380/1485 [1:05:21<3:0
+0: {'loss': 0.4786, 'grad_norm': 0.7605453117690044, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.16}
+0: 4:52, 10.04s/it]
 26%|██▌       | 381/1485 [1:05:31<3:07:14, 10.18s/it]
 26%|██▌       | 382/1485 [1:05:41<3:05:49, 10.11s/it]
 26%|██▌       | 383/1485 [1:05:51<3:06:15, 10.14s/it]
 26%|██▌       | 384/1485 [1:06:02<3:06:42, 10.17s/it]
 26%|██▌       | 385/1485 [1:06:12<3:07:48, 10.24s/it]
 26%|██▌       | 386/1485 [1:06:22<3:08:17, 10.28s/it]
 26%|██▌       | 387/1485 [1:06:33<3:08:49, 10.32s/it]
 26%|██▌       | 388/1485 [1:06:44<3:11:08, 10.45s/it]
 26%|██▌       | 389/1485 [1:06:54<3:10:42, 10.44s/it]
 26%|██▋       | 390/1485 [1:07:05<3:11:20, 10.48s/it]
                                                      

 26%|██▋       | 390/1485 [1:07:05<3:11:20, 10.48s/it]
 26%|██▋       | 391/1485 [1:07:15<3:10:49, 10.47s/it]
 26%|██▋       | 392/1485 [1:07:25<3:09:15, 10.39s/it]
 26%|██▋       | 393/1485 [1:07:35<3:08:12, 10.34s/it]
 27%|██▋       | 394/1485 [1:07:46<3:07:52, 10.33s/it]
 27%|██▋       | 395/1485 [1:0
+0: {'loss': 0.4806, 'grad_norm': 0.7441097099214423, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.16}
+0: 7:56<3:07:00, 10.29s/it]
 27%|██▋       | 396/1485 [1:08:06<3:06:43, 10.29s/it]
 27%|██▋       | 397/1485 [1:08:17<3:06:35, 10.29s/it]
 27%|██▋       | 398/1485 [1:08:27<3:06:59, 10.32s/it]
 27%|██▋       | 399/1485 [1:08:37<3:06:08, 10.28s/it]
 27%|██▋       | 400/1485 [1:08:47<3:05:44, 10.27s/it]
                                                      

 27%|██▋       | 400/1485 [1:08:47<3:05:44, 10.27s/it]
 27%|██▋       | 401/1485 [1:08:58<3:05:12, 10.25s/it]
 27%|██▋       | 402/1485 [1:09:08<3:04:44, 10.23s/it]
 27%|██▋       | 403/1485 [1:09:18<3:04:22, 10.22s/it]
 27%|██▋       | 404/1485 [1:09:28<3:04:11, 10.22s/it]
 27%|██▋       | 405/1485 [1:09:38<3:03:52, 10.21s/it]
 27%|██▋       | 406/1485 [1:09:49<3:03:18, 10.19s/it]
 27%|██▋       | 407/1485 [1:09:59<3:02:46, 10.17s/it]
 27%|██▋       | 408/1485 [1:10:09<3:03:12, 10.21s/it]
 28%|██▊       | 409/1485 [1:10:19<3:03:03, 10.21s/it]
 28%|██▊       | 410/1
+0: {'loss': 0.486, 'grad_norm': 0.7676714884534884, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.17}
+0: {'loss': 0.4724, 'grad_norm': 0.7667734667021008, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.17}
+0: 485 [1:10:29<3:02:50, 10.21s/it]
                                                      

 28%|██▊       | 410/1485 [1:10:29<3:02:50, 10.21s/it]
 28%|██▊       | 411/1485 [1:10:40<3:03:12, 10.23s/it]
 28%|█���▊       | 412/1485 [1:10:50<3:03:29, 10.26s/it]
 28%|██▊       | 413/1485 [1:11:00<3:03:06, 10.25s/it]
 28%|██▊       | 414/1485 [1:11:11<3:03:22, 10.27s/it]
 28%|██▊       | 415/1485 [1:11:21<3:03:50, 10.31s/it]
 28%|██▊       | 416/1485 [1:11:31<3:03:52, 10.32s/it]
 28%|██▊       | 417/1485 [1:11:42<3:05:04, 10.40s/it]
 28%|██▊       | 418/1485 [1:11:52<3:05:53, 10.45s/it]
 28%|██▊       | 419/1485 [1:12:03<3:06:36, 10.50s/it]
 28%|██▊       | 420/1485 [1:12:14<3:06:30, 10.51s/it]
                                                      

 28%|██▊       | 420/1485 [1:12:14<3:06:30, 10.51s/it]
 28%|██▊       | 421/1485 [1:12:24<3:05:35, 10.47s/it]
 28%|██▊       | 422/1485 [1:12:34<3:04:25, 10.41s/it]
 28%|██▊       | 42
+0: {'loss': 0.4755, 'grad_norm': 0.7213337123262135, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.17}
+0: 3/1485 [1:12:44<3:03:10, 10.35s/it]
 29%|██▊       | 424/1485 [1:12:55<3:02:20, 10.31s/it]
 29%|██▊       | 425/1485 [1:13:05<3:02:16, 10.32s/it]
 29%|██▊       | 426/1485 [1:13:15<3:02:21, 10.33s/it]
 29%|██▉       | 427/1485 [1:13:26<3:02:10, 10.33s/it]
 29%|██▉       | 428/1485 [1:13:36<3:02:05, 10.34s/it]
 29%|██▉       | 429/1485 [1:13:46<3:01:48, 10.33s/it]
 29%|██▉       | 430/1485 [1:13:57<3:01:39, 10.33s/it]
                                                      

 29%|██▉       | 430/1485 [1:13:57<3:01:39, 10.33s/it]
 29%|██▉       | 431/1485 [1:14:07<3:01:27, 10.33s/it]
 29%|██▉       | 432/1485 [1:14:17<3:01:05, 10.32s/it]
 29%|██▉       | 433/1485 [1:14:28<3:00:34, 10.30s/it]
 29%|██▉       | 434/1485 [1:14:38<2:59:55, 10.27s/it]
 29%|██▉       | 435/1485 [1:14:48<2:59:13, 10.24s/it]
 29%|██▉       | 436/1485 [1:14:58<2:58:55, 10.23s/it]
 29%|██▉       | 437/1485 [1:15:08<2:58:35, 10.22s/it]
 29%|██▉   
+0: {'loss': 0.4736, 'grad_norm': 0.9286741122639665, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.18}
+0: {'loss': 0.4655, 'grad_norm': 0.7757060139803554, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.18}
+0:     | 438/1485 [1:15:19<2:58:36, 10.24s/it]
 30%|██▉       | 439/1485 [1:15:29<2:59:25, 10.29s/it]
 30%|██▉       | 440/1485 [1:15:39<2:58:45, 10.26s/it]
                                                      

 30%|██▉       | 440/1485 [1:15:39<2:58:45, 10.26s/it]
 30%|██▉       | 441/1485 [1:15:49<2:58:04, 10.23s/it]
 30%|██▉       | 442/1485 [1:15:59<2:57:12, 10.19s/it]
 30%|██▉       | 443/1485 [1:16:10<2:56:47, 10.18s/it]
 30%|██▉       | 444/1485 [1:16:20<2:56:52, 10.19s/it]
 30%|██▉       | 445/1485 [1:16:30<2:57:50, 10.26s/it]
 30%|███       | 446/1485 [1:16:40<2:57:17, 10.24s/it]
 30%|███       | 447/1485 [1:16:51<2:56:39, 10.21s/it]
 30%|███       | 448/1485 [1:17:01<2:55:53, 10.18s/it]
 30%|███       | 449/1485 [1:17:11<2:54:48, 10.12s/it]
 30%|███       | 450/1485 [1:17:21<2:54:19, 10.11s/it]
                                                      

 30%|███       | 450/1485 [1:17:21<2:54:19, 10.11s/it]
 30%|███
+0: {'loss': 0.4761, 'grad_norm': 0.7412534320700842, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.19}
+0:        | 451/1485 [1:17:31<2:53:39, 10.08s/it]
 30%|███       | 452/1485 [1:17:41<2:52:37, 10.03s/it]
 31%|███       | 453/1485 [1:17:51<2:52:19, 10.02s/it]
 31%|███       | 454/1485 [1:18:01<2:52:19, 10.03s/it]
 31%|███       | 455/1485 [1:18:11<2:51:52, 10.01s/it]
 31%|███       | 456/1485 [1:18:21<2:51:28, 10.00s/it]
 31%|███       | 457/1485 [1:18:31<2:51:53, 10.03s/it]
 31%|███       | 458/1485 [1:18:41<2:52:28, 10.08s/it]
 31%|███       | 459/1485 [1:18:51<2:52:31, 10.09s/it]
 31%|███       | 460/1485 [1:19:01<2:52:30, 10.10s/it]
                                                      

 31%|███       | 460/1485 [1:19:01<2:52:30, 10.10s/it]
 31%|███       | 461/1485 [1:19:11<2:52:24, 10.10s/it]
 31%|███       | 462/1485 [1:19:21<2:52:11, 10.10s/it]
 31%|███       | 463/1485 [1:19:31<2:51:48, 10.09s/it]
 31%|███       | 464/1485 [1:19:42<2:51:55, 10.10s/it]
 31%|███▏      | 465/1485 [1:19:52<2:52:02, 10.12s/it]
 31%
+0: {'loss': 0.4814, 'grad_norm': 0.728343189964406, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.19}
+0: |███▏      | 466/1485 [1:20:02<2:52:15, 10.14s/it]
 31%|███▏      | 467/1485 [1:20:12<2:52:13, 10.15s/it]
 32%|███▏      | 468/1485 [1:20:22<2:52:14, 10.16s/it]
 32%|███▏      | 469/1485 [1:20:32<2:52:05, 10.16s/it]
 32%|███▏      | 470/1485 [1:20:43<2:51:26, 10.13s/it]
                                                      

 32%|███▏      | 470/1485 [1:20:43<2:51:26, 10.13s/it]
 32%|███▏      | 471/1485 [1:20:53<2:50:29, 10.09s/it]
 32%|███▏      | 472/1485 [1:21:02<2:49:41, 10.05s/it]
 32%|███▏      | 473/1485 [1:21:12<2:49:07, 10.03s/it]
 32%|███▏      | 474/1485 [1:21:23<2:49:19, 10.05s/it]
 32%|███▏      | 475/1485 [1:21:33<2:49:30, 10.07s/it]
 32%|███▏      | 476/1485 [1:21:43<2:49:17, 10.07s/it]
 32%|███▏      | 477/1485 [1:21:53<2:49:13, 10.07s/it]
 32%|███▏      | 478/1485 [1:22:03<2:49:31, 10.10s/it]
 32%|███▏      | 479/1485 [1:22:13<2:49:52, 10.13s/it]
 32%|███▏      | 48
+0: {'loss': 0.4751, 'grad_norm': 0.8917792589940368, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.19}
+0: {'loss': 0.4747, 'grad_norm': 0.7607835510277786, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.2}
+0: 0/1485 [1:22:23<2:50:06, 10.16s/it]
                                                      

 32%|███▏      | 480/1485 [1:22:23<2:50:06, 10.16s/it]
 32%|███▏      | 481/1485 [1:22:34<2:50:28, 10.19s/it]
 32%|███▏      | 482/1485 [1:22:44<2:50:32, 10.20s/it]
 33%|███▎      | 483/1485 [1:22:54<2:50:25, 10.21s/it]
 33%|███▎      | 484/1485 [1:23:04<2:50:10, 10.20s/it]
 33%|███▎      | 485/1485 [1:23:14<2:49:26, 10.17s/it]
 33%|███▎      | 486/1485 [1:23:24<2:48:59, 10.15s/it]
 33%|███▎      | 487/1485 [1:23:34<2:47:45, 10.09s/it]
 33%|███▎      | 488/1485 [1:23:44<2:46:27, 10.02s/it]
 33%|███▎      | 489/1485 [1:23:54<2:46:37, 10.04s/it]
 33%|███▎      | 490/1485 [1:24:04<2:46:50, 10.06s/it]
                                                      

 33%|███▎      | 490/1485 [1:24:05<2:46:50, 10.06s/it]
 33%|███▎      | 491/1485 [1:24:15<2:47:24, 10.11s/it]
 33%|███▎      | 492/1485 [1:24:25<2:46:52, 10.08
+0: {'loss': 0.4622, 'grad_norm': 0.8635803265202806, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.2}
+0: s/it]
 33%|███▎      | 493/1485 [1:24:35<2:46:28, 10.07s/it]
 33%|███▎      | 494/1485 [1:24:45<2:46:07, 10.06s/it]
 33%|███▎      | 495/1485 [1:24:55<2:45:47, 10.05s/it]
 33%|███▎      | 496/1485 [1:25:05<2:45:44, 10.05s/it]
 33%|███▎      | 497/1485 [1:25:15<2:45:16, 10.04s/it]
 34%|███▎      | 498/1485 [1:25:25<2:44:39, 10.01s/it]
 34%|███▎      | 499/1485 [1:25:35<2:44:20, 10.00s/it]
 34%|███▎      | 500/1485 [1:25:45<2:44:08, 10.00s/it]
                                                      

 34%|███▎      | 500/1485 [1:25:45<2:44:08, 10.00s/it]
 34%|███▎      | 501/1485 [1:25:55<2:43:49,  9.99s/it]
 34%|███▍      | 502/1485 [1:26:05<2:43:47, 10.00s/it]
 34%|███▍      | 503/1485 [1:26:15<2:43:30,  9.99s/it]
 34%|███▍      | 504/1485 [1:26:25<2:43:07,  9.98s/it]
 34%|███▍      | 505/1485 [1:26:35<2:42:40,  9.96s/it]
 34%|███▍      | 506/1485 [1:26:45<2:43:09, 10.00s/it]
 34%|███▍
+0: {'loss': 0.4785, 'grad_norm': 0.805509092193578, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.21}
+0:       | 507/1485 [1:26:55<2:42:38,  9.98s/it]
 34%|███▍      | 508/1485 [1:27:05<2:42:07,  9.96s/it]
 34%|███▍      | 509/1485 [1:27:14<2:41:48,  9.95s/it]
 34%|███▍      | 510/1485 [1:27:24<2:41:24,  9.93s/it]
                                                      

 34%|███▍      | 510/1485 [1:27:24<2:41:24,  9.93s/it]
 34%|███▍      | 511/1485 [1:27:34<2:41:18,  9.94s/it]
 34%|███▍      | 512/1485 [1:27:44<2:41:12,  9.94s/it]
 35%|███▍      | 513/1485 [1:27:54<2:41:07,  9.95s/it]
 35%|███▍      | 514/1485 [1:28:04<2:41:06,  9.96s/it]
 35%|███▍      | 515/1485 [1:28:14<2:41:02,  9.96s/it]
 35%|███▍      | 516/1485 [1:28:24<2:40:55,  9.96s/it]
 35%|███▍      | 517/1485 [1:28:34<2:40:23,  9.94s/it]
 35%|███▍      | 518/1485 [1:28:44<2:39:57,  9.93s/it]
 35%|███▍      | 519/1485 [1:28:54<2:39:36,  9.91s/it]
 35%|███▌      | 520/1485 [1:29:04<2:39:24,  9.91s/it]
                                        
+0: {'loss': 0.474, 'grad_norm': 0.776601745752629, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.21}
+0: {'loss': 0.466, 'grad_norm': 0.946434273413845, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.21}
+0:               

 35%|███▌      | 520/1485 [1:29:04<2:39:24,  9.91s/it]
 35%|███▌      | 521/1485 [1:29:14<2:39:17,  9.91s/it]
 35%|███▌      | 522/1485 [1:29:24<2:38:58,  9.90s/it]
 35%|███▌      | 523/1485 [1:29:33<2:38:33,  9.89s/it]
 35%|███▌      | 524/1485 [1:29:43<2:38:20,  9.89s/it]
 35%|███▌      | 525/1485 [1:29:53<2:37:52,  9.87s/it]
 35%|███▌      | 526/1485 [1:30:03<2:37:34,  9.86s/it]
 35%|███▌      | 527/1485 [1:30:13<2:37:51,  9.89s/it]
 36%|███▌      | 528/1485 [1:30:23<2:38:10,  9.92s/it]
 36%|███▌      | 529/1485 [1:30:33<2:38:16,  9.93s/it]
 36%|███▌      | 530/1485 [1:30:43<2:38:15,  9.94s/it]
                                                      

 36%|███▌      | 530/1485 [1:30:43<2:38:15,  9.94s/it]
 36%|███▌      | 531/1485 [1:30:53<2:38:09,  9.95s/it]
 36%|███▌      | 532/1485 [1:31:03<2:38:15,  9.96s/it]
 36%|███▌      | 533/1485 [1:31:13<2:38:46, 10.01s/it]
 36%|�
+0: {'loss': 0.4692, 'grad_norm': 0.7569679856547152, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.22}
+0: �██▌      | 534/1485 [1:31:23<2:37:59,  9.97s/it]
 36%|███▌      | 535/1485 [1:31:33<2:37:47,  9.97s/it]
 36%|███▌      | 536/1485 [1:31:43<2:37:35,  9.96s/it]
 36%|███▌      | 537/1485 [1:31:53<2:37:05,  9.94s/it]
 36%|███▌      | 538/1485 [1:32:02<2:36:29,  9.91s/it]
 36%|███▋      | 539/1485 [1:32:12<2:36:08,  9.90s/it]
 36%|███▋      | 540/1485 [1:32:22<2:35:52,  9.90s/it]
                                                      

 36%|███▋      | 540/1485 [1:32:22<2:35:52,  9.90s/it]
 36%|███▋      | 541/1485 [1:32:32<2:35:45,  9.90s/it]
 36%|███▋      | 542/1485 [1:32:42<2:35:26,  9.89s/it]
 37%|███▋      | 543/1485 [1:32:52<2:35:15,  9.89s/it]
 37%|███▋      | 544/1485 [1:33:02<2:35:11,  9.90s/it]
 37%|███▋      | 545/1485 [1:33:12<2:35:06,  9.90s/it]
 37%|███▋      | 546/1485 [1:33:22<2:34:43,  9.89s/it]
 37%|███▋      | 547/1485 [1:33:31<2:34:45,  9.90s/it]
 37%|███▋      | 548/1
+0: {'loss': 0.4592, 'grad_norm': 0.7614645276226978, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.22}
+0: {'loss': 0.4508, 'grad_norm': 0.7701263027301986, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.23}
+0: 485 [1:33:41<2:34:54,  9.92s/it]
 37%|███▋      | 549/1485 [1:33:51<2:35:06,  9.94s/it]
 37%|███▋      | 550/1485 [1:34:01<2:35:01,  9.95s/it]
                                                      

 37%|███▋      | 550/1485 [1:34:01<2:35:01,  9.95s/it]
 37%|███▋      | 551/1485 [1:34:11<2:35:01,  9.96s/it]
 37%|███▋      | 552/1485 [1:34:21<2:34:53,  9.96s/it]
 37%|███▋      | 553/1485 [1:34:31<2:35:12,  9.99s/it]
 37%|███▋      | 554/1485 [1:34:41<2:35:25, 10.02s/it]
 37%|███▋      | 555/1485 [1:34:51<2:34:31,  9.97s/it]
 37%|███▋      | 556/1485 [1:35:01<2:33:58,  9.94s/it]
 38%|███▊      | 557/1485 [1:35:11<2:33:35,  9.93s/it]
 38%|███▊      | 558/1485 [1:35:21<2:33:23,  9.93s/it]
 38%|███▊      | 559/1485 [1:35:31<2:33:06,  9.92s/it]
 38%|███▊      | 560/1485 [1:35:41<2:33:20,  9.95s/it]
                                                      

 38%|███▊      | 560/1485 [1:35:41<2:33:20,  9.95s/i
+0: {'loss': 0.4818, 'grad_norm': 0.7474490737269968, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.23}
+0: t]
 38%|███▊      | 561/1485 [1:35:51<2:33:39,  9.98s/it]
 38%|███▊      | 562/1485 [1:36:01<2:34:22, 10.03s/it]
 38%|███▊      | 563/1485 [1:36:11<2:34:17, 10.04s/it]
 38%|███▊      | 564/1485 [1:36:21<2:34:04, 10.04s/it]
 38%|███▊      | 565/1485 [1:36:31<2:34:16, 10.06s/it]
 38%|███▊      | 566/1485 [1:36:41<2:34:11, 10.07s/it]
 38%|███▊      | 567/1485 [1:36:51<2:33:24, 10.03s/it]
 38%|███▊      | 568/1485 [1:37:01<2:32:37,  9.99s/it]
 38%|███▊      | 569/1485 [1:37:11<2:32:02,  9.96s/it]
 38%|███▊      | 570/1485 [1:37:21<2:31:34,  9.94s/it]
                                                      

 38%|███▊      | 570/1485 [1:37:21<2:31:34,  9.94s/it]
 38%|███▊      | 571/1485 [1:37:31<2:31:19,  9.93s/it]
 39%|███▊      | 572/1485 [1:37:41<2:30:59,  9.92s/it]
 39%|███▊      | 573/1485 [1:37:51<2:30:42,  9.92s/it]
 39%|███▊      | 574/1485 [1:38:01<2:30:33,  9.92s/it]
 39%|███▊   
+0: {'loss': 0.4674, 'grad_norm': 0.7137483314990904, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.23}
+0:    | 575/1485 [1:38:11<2:30:29,  9.92s/it]
 39%|███▉      | 576/1485 [1:38:21<2:30:29,  9.93s/it]
 39%|███▉      | 577/1485 [1:38:31<2:30:25,  9.94s/it]
 39%|███▉      | 578/1485 [1:38:40<2:30:11,  9.94s/it]
 39%|███▉      | 579/1485 [1:38:50<2:30:03,  9.94s/it]
 39%|███▉      | 580/1485 [1:39:00<2:29:59,  9.94s/it]
                                                      

 39%|███▉      | 580/1485 [1:39:00<2:29:59,  9.94s/it]
 39%|███▉      | 581/1485 [1:39:10<2:30:04,  9.96s/it]
 39%|███▉      | 582/1485 [1:39:20<2:30:02,  9.97s/it]
 39%|███▉      | 583/1485 [1:39:30<2:29:51,  9.97s/it]
 39%|███▉      | 584/1485 [1:39:40<2:29:41,  9.97s/it]
 39%|███▉      | 585/1485 [1:39:50<2:29:25,  9.96s/it]
 39%|███▉      | 586/1485 [1:40:00<2:29:15,  9.96s/it]
 40%|███▉      | 587/1485 [1:40:10<2:28:45,  9.94s/it]
 40%|███▉      | 588/1485 [1:40:20<2:28:11,  9.91s/it]
 40%|███▉      | 589/1485 [1:40:30<
+0: {'loss': 0.4601, 'grad_norm': 0.7258971507358586, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.24}
+0: {'loss': 0.4642, 'grad_norm': 0.723974630413723, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.24}
+0: 2:27:48,  9.90s/it]
 40%|███▉      | 590/1485 [1:40:40<2:27:26,  9.88s/it]
                                                      

 40%|███▉      | 590/1485 [1:40:40<2:27:26,  9.88s/it]
 40%|███▉      | 591/1485 [1:40:50<2:28:24,  9.96s/it]
 40%|███▉      | 592/1485 [1:41:00<2:27:56,  9.94s/it]
 40%|███▉      | 593/1485 [1:41:10<2:27:25,  9.92s/it]
 40%|████      | 594/1485 [1:41:19<2:27:04,  9.90s/it]
 40%|████      | 595/1485 [1:41:29<2:26:43,  9.89s/it]
 40%|████      | 596/1485 [1:41:39<2:26:18,  9.87s/it]
 40%|████      | 597/1485 [1:41:49<2:26:01,  9.87s/it]
 40%|████      | 598/1485 [1:41:59<2:25:36,  9.85s/it]
 40%|████      | 599/1485 [1:42:09<2:25:17,  9.84s/it]
 40%|████      | 600/1485 [1:42:18<2:25:02,  9.83s/it]
                                                      

 40%|████      | 600/1485 [1:42:18<2:25:02,  9.83s/it]
 40%|████      | 601/1485 [1:42:28<2:24:41,  9.82s/it]
 41%|█�
+0: {'loss': 0.4703, 'grad_norm': 0.7258974907272255, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.25}
+0: �██      | 602/1485 [1:42:38<2:24:17,  9.80s/it]
 41%|████      | 603/1485 [1:42:48<2:23:59,  9.80s/it]
 41%|████      | 604/1485 [1:42:58<2:24:05,  9.81s/it]
 41%|████      | 605/1485 [1:43:07<2:23:58,  9.82s/it]
 41%|████      | 606/1485 [1:43:17<2:23:43,  9.81s/it]
 41%|████      | 607/1485 [1:43:27<2:23:27,  9.80s/it]
 41%|████      | 608/1485 [1:43:37<2:23:07,  9.79s/it]
 41%|████      | 609/1485 [1:43:47<2:23:07,  9.80s/it]
 41%|████      | 610/1485 [1:43:56<2:22:55,  9.80s/it]
                                                      

 41%|████      | 610/1485 [1:43:56<2:22:55,  9.80s/it]
 41%|████      | 611/1485 [1:44:06<2:22:42,  9.80s/it]
 41%|████      | 612/1485 [1:44:16<2:23:24,  9.86s/it]
 41%|████▏     | 613/1485 [1:44:26<2:22:52,  9.83s/it]
 41%|████▏     | 614/1485 [1:44:36<2:22:27,  9.81s/it]
 41%|████▏     | 615/1485 [1:44:46<2:22:24,  9.82s/it]
 41%|████▏     | 
+0: {'loss': 0.4619, 'grad_norm': 0.7207479254073837, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.25}
+0: 616/1485 [1:44:55<2:22:05,  9.81s/it]
 42%|████▏     | 617/1485 [1:45:05<2:22:01,  9.82s/it]
 42%|████▏     | 618/1485 [1:45:15<2:21:47,  9.81s/it]
 42%|████▏     | 619/1485 [1:45:25<2:21:55,  9.83s/it]
 42%|████▏     | 620/1485 [1:45:35<2:22:06,  9.86s/it]
                                                      

 42%|████▏     | 620/1485 [1:45:35<2:22:06,  9.86s/it]
 42%|████▏     | 621/1485 [1:45:45<2:21:56,  9.86s/it]
 42%|████▏     | 622/1485 [1:45:54<2:21:36,  9.85s/it]
 42%|████▏     | 623/1485 [1:46:04<2:21:04,  9.82s/it]
 42%|████▏     | 624/1485 [1:46:14<2:20:37,  9.80s/it]
 42%|████▏     | 625/1485 [1:46:24<2:20:19,  9.79s/it]
 42%|████▏     | 626/1485 [1:46:33<2:20:01,  9.78s/it]
 42%|████▏     | 627/1485 [1:46:43<2:19:43,  9.77s/it]
 42%|████▏     | 628/1485 [1:46:53<2:19:36,  9.77s/it]
 42%|████▏     | 629/1485 [1:47:03<2:19:13,  9.76s/it]
 42%|████▏
+0: {'loss': 0.4569, 'grad_norm': 0.7643439176824777, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.25}
+0: {'loss': 0.4531, 'grad_norm': 0.7420034448023466, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.26}
+0:      | 630/1485 [1:47:13<2:19:21,  9.78s/it]
                                                      

 42%|████▏     | 630/1485 [1:47:13<2:19:21,  9.78s/it]
 42%|████▏     | 631/1485 [1:47:22<2:19:04,  9.77s/it]
 43%|████▎     | 632/1485 [1:47:32<2:18:45,  9.76s/it]
 43%|████▎     | 633/1485 [1:47:42<2:18:11,  9.73s/it]
 43%|████▎     | 634/1485 [1:47:51<2:17:48,  9.72s/it]
 43%|████▎     | 635/1485 [1:48:01<2:17:39,  9.72s/it]
 43%|████▎     | 636/1485 [1:48:11<2:17:31,  9.72s/it]
 43%|████▎     | 637/1485 [1:48:21<2:18:52,  9.83s/it]
 43%|████▎     | 638/1485 [1:48:31<2:18:24,  9.80s/it]
 43%|████▎     | 639/1485 [1:48:40<2:18:15,  9.81s/it]
 43%|████▎     | 640/1485 [1:48:50<2:18:43,  9.85s/it]
                                                      

 43%|████▎     | 640/1485 [1:48:50<2:18:43,  9.85s/it]
 43%|████▎     | 641/1485 [1:49:00<2:19:00,  9.88s/it]
 43%|████▎  
+0: {'loss': 0.4508, 'grad_norm': 0.7126463112096134, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.26}
+0:    | 642/1485 [1:49:10<2:19:25,  9.92s/it]
 43%|████▎     | 643/1485 [1:49:20<2:19:15,  9.92s/it]
 43%|████▎     | 644/1485 [1:49:30<2:18:40,  9.89s/it]
 43%|████▎     | 645/1485 [1:49:40<2:18:17,  9.88s/it]
 44%|████▎     | 646/1485 [1:49:50<2:18:21,  9.89s/it]
 44%|████▎     | 647/1485 [1:50:00<2:18:05,  9.89s/it]
 44%|████▎     | 648/1485 [1:50:10<2:18:22,  9.92s/it]
 44%|████▎     | 649/1485 [1:50:20<2:18:39,  9.95s/it]
 44%|████▍     | 650/1485 [1:50:30<2:18:34,  9.96s/it]
                                                      

 44%|████▍     | 650/1485 [1:50:30<2:18:34,  9.96s/it]
 44%|████▍     | 651/1485 [1:50:40<2:18:28,  9.96s/it]
 44%|████▍     | 652/1485 [1:50:50<2:18:24,  9.97s/it]
 44%|████▍     | 653/1485 [1:51:00<2:18:28,  9.99s/it]
 44%|████▍     | 654/1485 [1:51:10<2:18:31, 10.00s/it]
 44%|████▍     | 655/1485 [1:51:20<2:18:28, 10.01s/it]
 44%|███�
+0: {'loss': 0.4639, 'grad_norm': 1.1734098530113097, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.27}
+0: ��▍     | 656/1485 [1:51:30<2:18:34, 10.03s/it]
 44%|████▍     | 657/1485 [1:51:40<2:18:13, 10.02s/it]
 44%|████▍     | 658/1485 [1:51:50<2:17:05,  9.95s/it]
 44%|████▍     | 659/1485 [1:52:00<2:17:37, 10.00s/it]
 44%|████▍     | 660/1485 [1:52:10<2:19:04, 10.11s/it]
                                                      

 44%|████▍     | 660/1485 [1:52:10<2:19:04, 10.11s/it]
 45%|████▍     | 661/1485 [1:52:21<2:20:02, 10.20s/it]
 45%|████▍     | 662/1485 [1:52:31<2:20:33, 10.25s/it]
 45%|████▍     | 663/1485 [1:52:42<2:21:52, 10.36s/it]
 45%|████▍     | 664/1485 [1:52:52<2:22:06, 10.39s/it]
 45%|████▍     | 665/1485 [1:53:02<2:22:07, 10.40s/it]
 45%|████▍     | 666/1485 [1:53:13<2:21:20, 10.36s/it]
 45%|████▍     | 667/1485 [1:53:23<2:20:35, 10.31s/it]
 45%|████▍     | 668/1485 [1:53:33<2:20:10, 10.29s/it]
 45%|████▌     | 669/1485 [1:53:43<2:19:43, 10.27s/it]
 45%|█
+0: {'loss': 0.4575, 'grad_norm': 1.0019092975811759, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.27}
+0: {'loss': 0.4496, 'grad_norm': 0.7457060598223622, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.27}
+0: ███▌     | 670/1485 [1:53:53<2:18:48, 10.22s/it]
                                                      

 45%|████▌     | 670/1485 [1:53:53<2:18:48, 10.22s/it]
 45%|████▌     | 671/1485 [1:54:04<2:17:56, 10.17s/it]
 45%|████▌     | 672/1485 [1:54:14<2:17:10, 10.12s/it]
 45%|████▌     | 673/1485 [1:54:24<2:16:43, 10.10s/it]
 45%|████▌     | 674/1485 [1:54:34<2:16:17, 10.08s/it]
 45%|████▌     | 675/1485 [1:54:44<2:16:03, 10.08s/it]
 46%|████▌     | 676/1485 [1:54:54<2:15:34, 10.05s/it]
 46%|████▌     | 677/1485 [1:55:04<2:15:04, 10.03s/it]
 46%|████▌     | 678/1485 [1:55:14<2:14:39, 10.01s/it]
 46%|████▌     | 679/1485 [1:55:24<2:14:46, 10.03s/it]
 46%|████▌     | 680/1485 [1:55:34<2:15:45, 10.12s/it]
                                                      

 46%|████▌     | 680/1485 [1:55:34<2:15:45, 10.12s/it]
 46%|████▌     | 681/1485 [1:55:45<2:17:14, 10.24s/it]
 46%|█�
+0: {'loss': 0.4579, 'grad_norm': 0.7506550229055139, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.28}
+0: �██▌     | 682/1485 [1:55:55<2:17:32, 10.28s/it]
 46%|████▌     | 683/1485 [1:56:05<2:17:45, 10.31s/it]
 46%|████▌     | 684/1485 [1:56:16<2:18:01, 10.34s/it]
 46%|████▌     | 685/1485 [1:56:26<2:17:59, 10.35s/it]
 46%|████▌     | 686/1485 [1:56:36<2:17:39, 10.34s/it]
 46%|████▋     | 687/1485 [1:56:46<2:16:30, 10.26s/it]
 46%|████▋     | 688/1485 [1:56:57<2:15:24, 10.19s/it]
 46%|████▋     | 689/1485 [1:57:06<2:14:23, 10.13s/it]
 46%|████▋     | 690/1485 [1:57:16<2:13:42, 10.09s/it]
                                                      

 46%|████▋     | 690/1485 [1:57:17<2:13:42, 10.09s/it]
 47%|████▋     | 691/1485 [1:57:27<2:13:22, 10.08s/it]
 47%|████▋     | 692/1485 [1:57:37<2:12:53, 10.05s/it]
 47%|████▋     | 693/1485 [1:57:46<2:11:49,  9.99s/it]
 47%|████▋     | 694/1485 [1:57:56<2:11:07,  9.95s/it]
 47%|████▋     | 695/1485 [1:58:06<2:10:16,  9.89s/it]
 47
+0: {'loss': 0.4514, 'grad_norm': 0.7538639662682873, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.28}
+0: %|████▋     | 696/1485 [1:58:16<2:09:40,  9.86s/it]
 47%|████▋     | 697/1485 [1:58:26<2:09:30,  9.86s/it]
 47%|████▋     | 698/1485 [1:58:35<2:09:19,  9.86s/it]
 47%|████▋     | 699/1485 [1:58:45<2:09:16,  9.87s/it]
 47%|████▋     | 700/1485 [1:58:55<2:09:23,  9.89s/it]
                                                      

 47%|████▋     | 700/1485 [1:58:55<2:09:23,  9.89s/it]
 47%|████▋     | 701/1485 [1:59:05<2:09:53,  9.94s/it]
 47%|████▋     | 702/1485 [1:59:15<2:10:01,  9.96s/it]
 47%|████▋     | 703/1485 [1:59:25<2:09:44,  9.95s/it]
 47%|████▋     | 704/1485 [1:59:35<2:09:15,  9.93s/it]
 47%|████▋     | 705/1485 [1:59:45<2:10:22, 10.03s/it]
 48%|████▊     | 706/1485 [1:59:56<2:11:23, 10.12s/it]
 48%|████▊     | 707/1485 [2:00:06<2:10:19, 10.05s/it]
 48%|████▊     | 708/1485 [2:00:16<2:09:59, 10.04s/it]
 48%|████▊     | 709/1485 [2:00:26<2:11:13, 10.15s/
+0: {'loss': 0.459, 'grad_norm': 0.8040620567030077, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.29}
+0: {'loss': 0.4584, 'grad_norm': 0.7666364096643932, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.29}
+0: it]
 48%|████▊     | 710/1485 [2:00:39<2:21:39, 10.97s/it]
                                                      

 48%|████▊     | 710/1485 [2:00:39<2:21:39, 10.97s/it]
 48%|████▊     | 711/1485 [2:00:50<2:21:08, 10.94s/it]
 48%|████▊     | 712/1485 [2:01:00<2:17:12, 10.65s/it]
 48%|████▊     | 713/1485 [2:01:10<2:15:00, 10.49s/it]
 48%|████▊     | 714/1485 [2:01:20<2:13:32, 10.39s/it]
 48%|████▊     | 715/1485 [2:01:30<2:12:30, 10.33s/it]
 48%|████▊     | 716/1485 [2:01:40<2:11:26, 10.26s/it]
 48%|████▊     | 717/1485 [2:01:51<2:10:47, 10.22s/it]
 48%|████▊     | 718/1485 [2:02:01<2:10:18, 10.19s/it]
 48%|████▊     | 719/1485 [2:02:11<2:09:29, 10.14s/it]
 48%|████▊     | 720/1485 [2:02:21<2:09:05, 10.12s/it]
                                                      

 48%|████▊     | 720/1485 [2:02:21<2:09:05, 10.12s/it]
 49%|████▊     | 721/1485 [2:02:31<2:09:10, 10.14s/it
+0: {'loss': 0.459, 'grad_norm': 0.74373529691844, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.29}
+0: ]
 49%|████▊     | 722/1485 [2:02:41<2:09:01, 10.15s/it]
 49%|████▊     | 723/1485 [2:02:51<2:09:04, 10.16s/it]
 49%|████▉     | 724/1485 [2:03:02<2:09:07, 10.18s/it]
 49%|████▉     | 725/1485 [2:03:12<2:09:11, 10.20s/it]
 49%|████▉     | 726/1485 [2:03:22<2:09:14, 10.22s/it]
 49%|████▉     | 727/1485 [2:03:32<2:08:21, 10.16s/it]
 49%|████▉     | 728/1485 [2:03:42<2:07:26, 10.10s/it]
 49%|████▉     | 729/1485 [2:03:52<2:06:32, 10.04s/it]
 49%|████▉     | 730/1485 [2:04:02<2:05:47, 10.00s/it]
                                                      

 49%|████▉     | 730/1485 [2:04:02<2:05:47, 10.00s/it]
 49%|���███▉     | 731/1485 [2:04:12<2:05:05,  9.95s/it]
 49%|████▉     | 732/1485 [2:04:21<2:04:20,  9.91s/it]
 49%|████▉     | 733/1485 [2:04:31<2:03:44,  9.87s/it]
 49%|████▉     | 734/1485 [2:04:41<2:03:31,  9.87s/it]
 49%|████▉     | 735/1485 [2:04:51<2:03:17,  9
+0: {'loss': 0.4563, 'grad_norm': 0.7487683564260629, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.3}
+0: .86s/it]
 50%|████▉     | 736/1485 [2:05:01<2:03:21,  9.88s/it]
 50%|████▉     | 737/1485 [2:05:11<2:02:56,  9.86s/it]
 50%|████▉     | 738/1485 [2:05:21<2:02:47,  9.86s/it]
 50%|████▉     | 739/1485 [2:05:30<2:02:43,  9.87s/it]
 50%|████▉     | 740/1485 [2:05:40<2:02:22,  9.86s/it]
                                                      

 50%|████▉     | 740/1485 [2:05:40<2:02:22,  9.86s/it]
 50%|████▉     | 741/1485 [2:05:50<2:02:18,  9.86s/it]
 50%|████▉     | 742/1485 [2:06:00<2:02:14,  9.87s/it]
 50%|█████     | 743/1485 [2:06:10<2:02:00,  9.87s/it]
 50%|█████     | 744/1485 [2:06:20<2:01:39,  9.85s/it]
 50%|█████     | 745/1485 [2:06:30<2:01:30,  9.85s/it]
 50%|█████     | 746/1485 [2:06:39<2:01:28,  9.86s/it]
 50%|█████     | 747/1485 [2:06:49<2:01:32,  9.88s/it]
 50%|█████     | 748/1485 [2:06:59<2:01:42,  9.91s/it]
 50%|█████     | 749/1485 [2:07:09<2:01
+0: {'loss': 0.4587, 'grad_norm': 0.7643946785194354, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.3}
+0: {'loss': 0.4537, 'grad_norm': 0.7647764662214266, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.31}
+0: :40,  9.92s/it]
 51%|█████     | 750/1485 [2:07:19<2:01:49,  9.95s/it]
                                                      

 51%|█████     | 750/1485 [2:07:19<2:01:49,  9.95s/it]
 51%|█████     | 751/1485 [2:07:29<2:01:44,  9.95s/it]
 51%|█████     | 752/1485 [2:07:39<2:01:36,  9.95s/it]
 51%|█████     | 753/1485 [2:07:49<2:01:43,  9.98s/it]
 51%|█████     | 754/1485 [2:07:59<2:01:32,  9.98s/it]
 51%|█████     | 755/1485 [2:08:09<2:01:28,  9.98s/it]
 51%|█████     | 756/1485 [2:08:19<2:01:44, 10.02s/it]
 51%|█████     | 757/1485 [2:08:29<2:01:47, 10.04s/it]
 51%|█████     | 758/1485 [2:08:39<2:01:45, 10.05s/it]
 51%|█████     | 759/1485 [2:08:50<2:02:00, 10.08s/it]
 51%|█████     | 760/1485 [2:09:00<2:02:39, 10.15s/it]
                                                      

 51%|█████     | 760/1485 [2:09:00<2:02:39, 10.15s/it]
 51%|█████     | 761/1485 [2:09:10<2:02:2
+0: {'loss': 0.4506, 'grad_norm': 0.7860264685377676, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.31}
+0: 2, 10.14s/it]
 51%|█████▏    | 762/1485 [2:09:20<2:01:45, 10.10s/it]
 51%|█████▏    | 763/1485 [2:09:30<2:01:11, 10.07s/it]
 51%|█████▏    | 764/1485 [2:09:40<2:00:46, 10.05s/it]
 52%|█████▏    | 765/1485 [2:09:50<2:00:15, 10.02s/it]
 52%|█████▏    | 766/1485 [2:10:00<2:00:14, 10.03s/it]
 52%|█████▏    | 767/1485 [2:10:10<2:00:04, 10.03s/it]
 52%|█████▏    | 768/1485 [2:10:20<1:59:56, 10.04s/it]
 52%|█████▏    | 769/1485 [2:10:30<1:59:58, 10.05s/it]
 52%|█████▏    | 770/1485 [2:10:40<2:00:02, 10.07s/it]
                                                      

 52%|█████▏    | 770/1485 [2:10:40<2:00:02, 10.07s/it]
 52%|█████▏    | 771/1485 [2:10:51<1:59:58, 10.08s/it]
 52%|█████▏    | 772/1485 [2:11:01<1:59:38, 10.07s/it]
 52%|█████▏    | 773/1485 [2:11:11<1:59:58, 10.11s/it]
 52%|█████▏    | 774/1485 [2:11:21<2:00:01, 10.13s/it]
 52%|███�
+0: {'loss': 0.4519, 'grad_norm': 0.7240075816677758, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.32}
+0: �█▏    | 775/1485 [2:11:31<1:59:31, 10.10s/it]
 52%|█████▏    | 776/1485 [2:11:41<1:59:02, 10.07s/it]
 52%|█████▏    | 777/1485 [2:11:51<1:59:03, 10.09s/it]
 52%|█████▏    | 778/1485 [2:12:01<1:58:39, 10.07s/it]
 52%|█████▏    | 779/1485 [2:12:11<1:58:23, 10.06s/it]
 53%|█████▎    | 780/1485 [2:12:24<2:07:10, 10.82s/it]
                                                      

 53%|█████▎    | 780/1485 [2:12:24<2:07:10, 10.82s/it]
 53%|█████▎    | 781/1485 [2:12:34<2:05:13, 10.67s/it]
 53%|█████▎    | 782/1485 [2:12:44<2:02:45, 10.48s/it]
 53%|█████▎    | 783/1485 [2:12:54<2:00:50, 10.33s/it]
 53%|█████▎    | 784/1485 [2:13:04<1:59:34, 10.24s/it]
 53%|█████▎    | 785/1485 [2:13:14<1:58:20, 10.14s/it]
 53%|█████▎    | 786/1485 [2:13:24<1:57:40, 10.10s/it]
 53%|█████▎    | 787/1485 [2:13:34<1:57:16, 10.08s/it]
 53%|█████▎    | 788/1485 [2:13:44
+0: {'loss': 0.4497, 'grad_norm': 0.7200368823963059, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.32}
+0: {'loss': 0.4557, 'grad_norm': 0.7485025836286483, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.32}
+0: <1:57:02, 10.08s/it]
 53%|█████▎    | 789/1485 [2:13:54<1:57:43, 10.15s/it]
 53%|█████▎    | 790/1485 [2:14:04<1:57:08, 10.11s/it]
                                                      

 53%|█████▎    | 790/1485 [2:14:05<1:57:08, 10.11s/it]
 53%|█████▎    | 791/1485 [2:14:14<1:56:36, 10.08s/it]
 53%|█████▎    | 792/1485 [2:14:25<1:56:46, 10.11s/it]
 53%|█████▎    | 793/1485 [2:14:35<1:56:51, 10.13s/it]
 53%|█████▎    | 794/1485 [2:14:45<1:57:06, 10.17s/it]
 54%|█████▎    | 795/1485 [2:14:55<1:56:29, 10.13s/it]
 54%|█████▎    | 796/1485 [2:15:05<1:56:07, 10.11s/it]
 54%|█████▎    | 797/1485 [2:15:15<1:55:39, 10.09s/it]
 54%|█████▎    | 798/1485 [2:15:25<1:54:58, 10.04s/it]
 54%|█████▍    | 799/1485 [2:15:35<1:54:19, 10.00s/it]
 54%|█████▍    | 800/1485 [2:15:45<1:54:17, 10.01s/it]
                                                      

 54%|█████
+0: {'loss': 0.4499, 'grad_norm': 0.7232431500671842, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.33}
+0: ▍    | 800/1485 [2:15:45<1:54:17, 10.01s/it]
 54%|█████▍    | 801/1485 [2:15:55<1:54:52, 10.08s/it]
 54%|█████▍    | 802/1485 [2:16:05<1:54:05, 10.02s/it]
 54%|█████▍    | 803/1485 [2:16:15<1:53:22,  9.97s/it]
 54%|█████▍    | 804/1485 [2:16:25<1:53:08,  9.97s/it]
 54%|█████▍    | 805/1485 [2:16:35<1:52:41,  9.94s/it]
 54%|█████▍    | 806/1485 [2:16:45<1:52:30,  9.94s/it]
 54%|█████▍    | 807/1485 [2:16:55<1:52:32,  9.96s/it]
 54%|█████▍    | 808/1485 [2:17:05<1:53:14, 10.04s/it]
 54%|█████▍    | 809/1485 [2:17:15<1:53:51, 10.11s/it]
 55%|█████▍    | 810/1485 [2:17:26<1:54:18, 10.16s/it]
                                                      

 55%|█████▍    | 810/1485 [2:17:26<1:54:18, 10.16s/it]
 55%|█████▍    | 811/1485 [2:17:36<1:54:34, 10.20s/it]
 55%|█████▍    | 812/1485 [2:17:46<1:54:51, 10.24s/it]
 55%|█████▍    | 813/1485 [2:17:57<1:5
+0: {'loss': 0.4537, 'grad_norm': 0.7238765804304385, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.33}
+0: 4:53, 10.26s/it]
 55%|█████▍    | 814/1485 [2:18:07<1:54:23, 10.23s/it]
 55%|█████▍    | 815/1485 [2:18:17<1:52:54, 10.11s/it]
 55%|█████▍    | 816/1485 [2:18:26<1:52:02, 10.05s/it]
 55%|█████▌    | 817/1485 [2:18:36<1:51:24, 10.01s/it]
 55%|█████▌    | 818/1485 [2:18:46<1:50:59,  9.98s/it]
 55%|█████▌    | 819/1485 [2:18:57<1:51:27, 10.04s/it]
 55%|█████▌    | 820/1485 [2:19:06<1:51:08, 10.03s/it]
                                                      

 55%|█████▌    | 820/1485 [2:19:07<1:51:08, 10.03s/it]
 55%|█████▌    | 821/1485 [2:19:17<1:50:59, 10.03s/it]
 55%|█████▌    | 822/1485 [2:19:27<1:50:56, 10.04s/it]
 55%|█████▌    | 823/1485 [2:19:37<1:50:45, 10.04s/it]
 55%|█████▌    | 824/1485 [2:19:47<1:50:18, 10.01s/it]
 56%|█████▌    | 825/1485 [2:19:57<1:50:09, 10.01s/it]
 56%|█████▌    | 826/1485 [2:20:07<1:49:56, 10.01s/it]
 56%|██�
+0: {'loss': 0.452, 'grad_norm': 0.9275378642409674, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.34}
+0: �██▌    | 827/1485 [2:20:17<1:49:42, 10.00s/it]
 56%|█████▌    | 828/1485 [2:20:27<1:49:41, 10.02s/it]
 56%|█████▌    | 829/1485 [2:20:37<1:50:00, 10.06s/it]
 56%|█████▌    | 830/1485 [2:20:47<1:49:55, 10.07s/it]
                                                      

 56%|█████▌    | 830/1485 [2:20:47<1:49:55, 10.07s/it]
 56%|█████▌    | 831/1485 [2:20:57<1:49:50, 10.08s/it]
 56%|█████▌    | 832/1485 [2:21:07<1:49:28, 10.06s/it]
 56%|█████▌    | 833/1485 [2:21:17<1:49:28, 10.07s/it]
 56%|█████▌    | 834/1485 [2:21:27<1:49:37, 10.10s/it]
 56%|█████▌    | 835/1485 [2:21:37<1:49:33, 10.11s/it]
 56%|█████▋    | 836/1485 [2:21:48<1:49:22, 10.11s/it]
 56%|█████▋    | 837/1485 [2:21:58<1:49:09, 10.11s/it]
 56%|█████▋    | 838/1485 [2:22:08<1:48:56, 10.10s/it]
 56%|█████▋    | 839/1485 [2:22:18<1:48:49, 10.11s/it]
 57%|█████▋    | 840/1485 [2:22
+0: {'loss': 0.452, 'grad_norm': 0.740587071400382, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.34}
+0: {'loss': 0.4516, 'grad_norm': 1.1474821234085517, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.34}
+0: :28<1:48:50, 10.12s/it]
                                                      

 57%|█████▋    | 840/1485 [2:22:28<1:48:50, 10.12s/it]
 57%|█████▋    | 841/1485 [2:22:38<1:48:48, 10.14s/it]
 57%|█████▋    | 842/1485 [2:22:48<1:48:29, 10.12s/it]
 57%|█████▋    | 843/1485 [2:22:58<1:48:23, 10.13s/it]
 57%|█████▋    | 844/1485 [2:23:09<1:48:19, 10.14s/it]
 57%|█████▋    | 845/1485 [2:23:19<1:48:28, 10.17s/it]
 57%|█████▋    | 846/1485 [2:23:29<1:48:45, 10.21s/it]
 57%|█████▋    | 847/1485 [2:23:39<1:48:46, 10.23s/it]
 57%|█████▋    | 848/1485 [2:23:50<1:48:36, 10.23s/it]
 57%|█████▋    | 849/1485 [2:24:00<1:48:19, 10.22s/it]
 57%|█████▋    | 850/1485 [2:24:10<1:47:56, 10.20s/it]
                                                      

 57%|█████▋    | 850/1485 [2:24:10<1:47:56, 10.20s/it]
 57%|█████▋    | 851/1485 [2:24:20<1:47:45, 10.20s/it]
 57%|████
+0: {'loss': 0.4525, 'grad_norm': 0.961894497135354, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.35}
+0: █▋    | 852/1485 [2:24:30<1:47:37, 10.20s/it]
 57%|█████▋    | 853/1485 [2:24:41<1:47:27, 10.20s/it]
 58%|█████▊    | 854/1485 [2:24:51<1:47:19, 10.21s/it]
 58%|█████▊    | 855/1485 [2:25:01<1:47:10, 10.21s/it]
 58%|█████▊    | 856/1485 [2:25:11<1:47:05, 10.22s/it]
 58%|█████▊    | 857/1485 [2:25:21<1:46:45, 10.20s/it]
 58%|█████▊    | 858/1485 [2:25:32<1:46:47, 10.22s/it]
 58%|█████▊    | 859/1485 [2:25:42<1:46:43, 10.23s/it]
 58%|█████▊    | 860/1485 [2:25:52<1:46:39, 10.24s/it]
                                                      

 58%|█████▊    | 860/1485 [2:25:52<1:46:39, 10.24s/it]
 58%|█████▊    | 861/1485 [2:26:02<1:46:42, 10.26s/it]
 58%|█████▊    | 862/1485 [2:26:13<1:46:17, 10.24s/it]
 58%|█████▊    | 863/1485 [2:26:23<1:46:07, 10.24s/it]
 58%|█████▊    | 864/1485 [2:26:33<1:46:01, 10.24s/it]
 58%|█████▊    | 865/1485 [2:26:43<
+0: {'loss': 0.4508, 'grad_norm': 0.7212586095072385, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.35}
+0: 1:45:56, 10.25s/it]
 58%|█████▊    | 866/1485 [2:26:54<1:45:31, 10.23s/it]
 58%|█████▊    | 867/1485 [2:27:04<1:45:02, 10.20s/it]
 58%|█████▊    | 868/1485 [2:27:14<1:44:44, 10.19s/it]
 59%|█████▊    | 869/1485 [2:27:24<1:44:35, 10.19s/it]
 59%|█████▊    | 870/1485 [2:27:34<1:44:14, 10.17s/it]
                                                      

 59%|█████▊    | 870/1485 [2:27:34<1:44:14, 10.17s/it]
 59%|█████▊    | 871/1485 [2:27:44<1:43:57, 10.16s/it]
 59%|█████▊    | 872/1485 [2:27:55<1:44:21, 10.22s/it]
 59%|█████▉    | 873/1485 [2:28:05<1:43:59, 10.20s/it]
 59%|█████▉    | 874/1485 [2:28:15<1:43:36, 10.17s/it]
 59%|█████▉    | 875/1485 [2:28:25<1:43:20, 10.16s/it]
 59%|█████▉    | 876/1485 [2:28:35<1:42:57, 10.14s/it]
 59%|█████▉    | 877/1485 [2:28:45<1:42:41, 10.13s/it]
 59%|█████▉    | 878/1485 [2:28:55<1:42:29, 10.13s/it]
 59%|█�
+0: {'loss': 0.4564, 'grad_norm': 0.7630536816738686, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.36}
+0: {'loss': 0.4513, 'grad_norm': 0.7352580910658896, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.36}
+0: �███▉    | 879/1485 [2:29:06<1:42:20, 10.13s/it]
 59%|█████▉    | 880/1485 [2:29:16<1:41:56, 10.11s/it]
                                                      

 59%|█████▉    | 880/1485 [2:29:16<1:41:56, 10.11s/it]
 59%|█████▉    | 881/1485 [2:29:26<1:41:38, 10.10s/it]
 59%|█████▉    | 882/1485 [2:29:36<1:41:27, 10.10s/it]
 59%|█████▉    | 883/1485 [2:29:46<1:41:14, 10.09s/it]
 60%|█████▉    | 884/1485 [2:29:56<1:41:02, 10.09s/it]
 60%|█████▉    | 885/1485 [2:30:06<1:40:41, 10.07s/it]
 60%|█████▉    | 886/1485 [2:30:16<1:40:34, 10.07s/it]
 60%|█████▉    | 887/1485 [2:30:26<1:40:54, 10.12s/it]
 60%|█████▉    | 888/1485 [2:30:37<1:41:14, 10.18s/it]
 60%|█████▉    | 889/1485 [2:30:47<1:40:58, 10.17s/it]
 60%|█████▉    | 890/1485 [2:30:57<1:41:34, 10.24s/it]
                                                      

 60%|█████▉    | 890/1485 [2:30:57<1:41
+0: {'loss': 0.454, 'grad_norm': 1.073148360545092, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.36}
+0: :34, 10.24s/it]
 60%|██████    | 891/1485 [2:31:07<1:41:30, 10.25s/it]
 60%|██████    | 892/1485 [2:31:18<1:41:17, 10.25s/it]
 60%|██████    | 893/1485 [2:31:28<1:40:49, 10.22s/it]
 60%|██████    | 894/1485 [2:31:38<1:40:22, 10.19s/it]
 60%|██████    | 895/1485 [2:31:48<1:40:06, 10.18s/it]
 60%|██████    | 896/1485 [2:31:58<1:39:45, 10.16s/it]
 60%|██████    | 897/1485 [2:32:08<1:39:37, 10.17s/it]
 60%|██████    | 898/1485 [2:32:19<1:39:41, 10.19s/it]
 61%|██████    | 899/1485 [2:32:29<1:39:45, 10.21s/it]
 61%|██████    | 900/1485 [2:32:39<1:39:42, 10.23s/it]
                                                      

 61%|██████    | 900/1485 [2:32:39<1:39:42, 10.23s/it]
 61%|██████    | 901/1485 [2:32:49<1:39:28, 10.22s/it]
 61%|██████    | 902/1485 [2:33:00<1:39:19, 10.22s/it]
 61%|██████    | 903/1485 [2:33:10<1:39:11, 10.23s/it]
 61%|███
+0: {'loss': 0.4556, 'grad_norm': 0.7500106928819219, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.37}
+0: ███    | 904/1485 [2:33:20<1:38:58, 10.22s/it]
 61%|██████    | 905/1485 [2:33:30<1:38:53, 10.23s/it]
 61%|██████    | 906/1485 [2:33:40<1:38:30, 10.21s/it]
 61%|██████    | 907/1485 [2:33:51<1:38:19, 10.21s/it]
 61%|██████    | 908/1485 [2:34:01<1:38:09, 10.21s/it]
 61%|██████    | 909/1485 [2:34:11<1:38:15, 10.23s/it]
 61%|██████▏   | 910/1485 [2:34:21<1:38:05, 10.24s/it]
                                                      

 61%|██████▏   | 910/1485 [2:34:21<1:38:05, 10.24s/it]
 61%|██████▏   | 911/1485 [2:34:32<1:38:06, 10.26s/it]
 61%|██████▏   | 912/1485 [2:34:42<1:37:52, 10.25s/it]
 61%|██████▏   | 913/1485 [2:34:52<1:37:33, 10.23s/it]
 62%|██████▏   | 914/1485 [2:35:02<1:37:19, 10.23s/it]
 62%|██████▏   | 915/1485 [2:35:13<1:37:10, 10.23s/it]
 62%|██████▏   | 916/1485 [2:35:23<1:36:51, 10.21s/it]
 62%|██████▏  
+0: {'loss': 0.6124, 'grad_norm': 1.7152604292005151, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.37}
+0:  | 917/1485 [2:35:33<1:36:45, 10.22s/it]
 62%|██████▏   | 918/1485 [2:35:43<1:36:41, 10.23s/it]
 62%|██████▏   | 919/1485 [2:35:54<1:37:16, 10.31s/it]
 62%|██████▏   | 920/1485 [2:36:04<1:36:41, 10.27s/it]
                                                      

 62%|██████▏   | 920/1485 [2:36:04<1:36:41, 10.27s/it]
 62%|██████▏   | 921/1485 [2:36:14<1:36:12, 10.23s/it]
 62%|██████▏   | 922/1485 [2:36:24<1:36:08, 10.25s/it]
 62%|██████▏   | 923/1485 [2:36:35<1:36:38, 10.32s/it]
 62%|██████▏   | 924/1485 [2:36:45<1:36:14, 10.29s/it]
 62%|██████▏   | 925/1485 [2:36:55<1:35:39, 10.25s/it]
 62%|██████▏   | 926/1485 [2:37:05<1:35:14, 10.22s/it]
 62%|██████▏   | 927/1485 [2:37:16<1:34:46, 10.19s/it]
 62%|██████▏   | 928/1485 [2:37:26<1:34:15, 10.15s/it]
 63%|██████▎   | 929/1485 [2:37:36<1:34:08, 10.16s/it]
 63%|██████▎   |
+0: {'loss': 0.7141, 'grad_norm': 1.352356765907374, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.38}
+0: {'loss': 0.6763, 'grad_norm': 1.0767486499911019, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.38}
+0:  930/1485 [2:37:46<1:33:57, 10.16s/it]
                                                      

 63%|██████▎   | 930/1485 [2:37:46<1:33:57, 10.16s/it]
 63%|██████▎   | 931/1485 [2:37:56<1:34:03, 10.19s/it]
 63%|██████▎   | 932/1485 [2:38:06<1:34:01, 10.20s/it]
 63%|██████▎   | 933/1485 [2:38:17<1:33:49, 10.20s/it]
 63%|██████▎   | 934/1485 [2:38:27<1:34:06, 10.25s/it]
 63%|██████▎   | 935/1485 [2:38:37<1:34:27, 10.30s/it]
 63%|██████▎   | 936/1485 [2:38:48<1:34:21, 10.31s/it]
 63%|██████▎   | 937/1485 [2:38:58<1:33:51, 10.28s/it]
 63%|██████▎   | 938/1485 [2:39:08<1:33:40, 10.27s/it]
 63%|██████▎   | 939/1485 [2:39:18<1:33:37, 10.29s/it]
 63%|██████▎   | 940/1485 [2:39:29<1:33:34, 10.30s/it]
                                                      

 63%|██████▎   | 940/1485 [2:39:29<1:33:34, 10.30s/it]
 63%|██████▎   | 941/1485 [2:39
+0: {'loss': 0.6546, 'grad_norm': 0.9935189614875325, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.38}
+0: :39<1:33:12, 10.28s/it]
 63%|██████▎   | 942/1485 [2:39:49<1:32:58, 10.27s/it]
 64%|██████▎   | 943/1485 [2:40:00<1:32:41, 10.26s/it]
 64%|██████▎   | 944/1485 [2:40:10<1:32:23, 10.25s/it]
 64%|██████▎   | 945/1485 [2:40:20<1:32:12, 10.25s/it]
 64%|██████▎   | 946/1485 [2:40:30<1:31:42, 10.21s/it]
 64%|██████▍   | 947/1485 [2:40:40<1:31:30, 10.21s/it]
 64%|██████▍   | 948/1485 [2:40:51<1:31:19, 10.20s/it]
 64%|██████▍   | 949/1485 [2:41:01<1:31:26, 10.24s/it]
 64%|██████▍   | 950/1485 [2:41:11<1:31:30, 10.26s/it]
                                                      

 64%|██████▍   | 950/1485 [2:41:11<1:31:30, 10.26s/it]
 64%|██████▍   | 951/1485 [2:41:21<1:31:31, 10.28s/it]
 64%|██████▍   | 952/1485 [2:41:32<1:31:39, 10.32s/it]
 64%|██████▍   | 953/1485 [2:41:42<1:31:30, 10.32s/it]
 64%|██████▍   | 954/1485 [2:41:5
+0: {'loss': 0.6324, 'grad_norm': 1.351561259660997, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.39}
+0: 3<1:31:33, 10.35s/it]
 64%|██████▍   | 955/1485 [2:42:03<1:31:35, 10.37s/it]
 64%|██████▍   | 956/1485 [2:42:13<1:31:34, 10.39s/it]
 64%|██████▍   | 957/1485 [2:42:24<1:31:36, 10.41s/it]
 65%|██████▍   | 958/1485 [2:42:34<1:31:31, 10.42s/it]
 65%|██████▍   | 959/1485 [2:42:45<1:31:31, 10.44s/it]
 65%|██████▍   | 960/1485 [2:42:55<1:31:37, 10.47s/it]
                                                      

 65%|██████▍   | 960/1485 [2:42:55<1:31:37, 10.47s/it]
 65%|██████▍   | 961/1485 [2:43:06<1:31:38, 10.49s/it]
 65%|██████▍   | 962/1485 [2:43:17<1:31:39, 10.51s/it]
 65%|██████▍   | 963/1485 [2:43:27<1:31:33, 10.52s/it]
 65%|██████▍   | 964/1485 [2:43:37<1:31:01, 10.48s/it]
 65%|██████▍   | 965/1485 [2:43:48<1:30:30, 10.44s/it]
 65%|██████▌   | 966/1485 [2:43:58<1:30:05, 10.41s/it]
 65%|██████▌   | 967/1485 [2:44:09<
+0: {'loss': 0.6239, 'grad_norm': 1.052359756366879, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.39}
+0: 1:29:51, 10.41s/it]
 65%|██████▌   | 968/1485 [2:44:19<1:29:35, 10.40s/it]
 65%|██████▌   | 969/1485 [2:44:29<1:29:23, 10.39s/it]
 65%|██████▌   | 970/1485 [2:44:40<1:29:24, 10.42s/it]
                                                      

 65%|██████▌   | 970/1485 [2:44:40<1:29:24, 10.42s/it]
 65%|██████▌   | 971/1485 [2:44:50<1:29:09, 10.41s/it]
 65%|██████▌   | 972/1485 [2:45:01<1:29:07, 10.42s/it]
 66%|██████▌   | 973/1485 [2:45:11<1:29:11, 10.45s/it]
 66%|██████▌   | 974/1485 [2:45:22<1:29:04, 10.46s/it]
 66%|██████▌   | 975/1485 [2:45:32<1:28:53, 10.46s/it]
 66%|██████▌   | 976/1485 [2:45:42<1:28:01, 10.38s/it]
 66%|██████▌   | 977/1485 [2:45:52<1:27:09, 10.30s/it]
 66%|██████▌   | 978/1485 [2:46:03<1:26:40, 10.26s/it]
 66%|██████▌   | 979/1485 [2:46:13<1:26:11, 10.22s/it]
 66%|██████▌   | 980/1485 [2:46:23<1:
+0: {'loss': 0.6079, 'grad_norm': 1.2796206848720668, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.4}
+0: {'loss': 0.6142, 'grad_norm': 1.087736813019815, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.4}
+0: 25:46, 10.19s/it]
                                                      

 66%|██████▌   | 980/1485 [2:46:23<1:25:46, 10.19s/it]
 66%|██████▌   | 981/1485 [2:46:33<1:25:30, 10.18s/it]
 66%|██████���   | 982/1485 [2:46:43<1:25:12, 10.16s/it]
 66%|██████▌   | 983/1485 [2:46:53<1:24:41, 10.12s/it]
 66%|██████▋   | 984/1485 [2:47:03<1:24:17, 10.10s/it]
 66%|██████▋   | 985/1485 [2:47:13<1:24:07, 10.10s/it]
 66%|██████▋   | 986/1485 [2:47:23<1:24:02, 10.10s/it]
 66%|██████▋   | 987/1485 [2:47:33<1:23:46, 10.09s/it]
 67%|██████▋   | 988/1485 [2:47:43<1:23:31, 10.08s/it]
 67%|██████▋   | 989/1485 [2:47:54<1:23:18, 10.08s/it]
 67%|██████▋   | 990/1485 [2:48:04<1:23:08, 10.08s/it]
                                                      

 67%|██████▋   | 990/1485 [2:48:04<1:23:08, 10.08s/it]
 67%|██████▋   | 991/1485 [2:48:14<1:23:06, 10.09s/i
+0: {'loss': 0.6092, 'grad_norm': 1.1236750638763475, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.4}
+0: t]
 67%|██████▋   | 992/1485 [2:48:24<1:23:08, 10.12s/it]
 67%|██████▋   | 993/1485 [2:48:34<1:23:03, 10.13s/it]
 67%|██████▋   | 994/1485 [2:48:44<1:22:52, 10.13s/it]
 67%|██████▋   | 995/1485 [2:48:54<1:22:38, 10.12s/it]
 67%|██████▋   | 996/1485 [2:49:04<1:22:28, 10.12s/it]
 67%|██████▋   | 997/1485 [2:49:14<1:21:49, 10.06s/it]
 67%|██████▋   | 998/1485 [2:49:24<1:21:27, 10.04s/it]
 67%|██████▋   | 999/1485 [2:49:34<1:21:13, 10.03s/it]
 67%|██████▋   | 1000/1485 [2:49:44<1:21:19, 10.06s/it]
                                                       

 67%|██████▋   | 1000/1485 [2:49:44<1:21:19, 10.06s/it]
 67%|██████▋   | 1001/1485 [2:49:55<1:21:22, 10.09s/it]
 67%|██████▋   | 1002/1485 [2:50:05<1:21:16, 10.10s/it]
 68%|██████▊   | 1003/1485 [2:50:15<1:21:05, 10.09s/it]
 68%|██████▊   | 1004/1485 [2:50:25<1:20:50, 10.
+0: {'loss': 0.608, 'grad_norm': 1.5342585006566427, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.41}
+0: 08s/it]
 68%|██████▊   | 1005/1485 [2:50:35<1:20:29, 10.06s/it]
 68%|██████▊   | 1006/1485 [2:50:45<1:20:18, 10.06s/it]
 68%|██████▊   | 1007/1485 [2:50:55<1:20:03, 10.05s/it]
 68%|██████▊   | 1008/1485 [2:51:05<1:19:48, 10.04s/it]
 68%|██████▊   | 1009/1485 [2:51:15<1:19:42, 10.05s/it]
 68%|██████▊   | 1010/1485 [2:51:25<1:19:45, 10.07s/it]
                                                       

 68%|██████▊   | 1010/1485 [2:51:25<1:19:45, 10.07s/it]
 68%|██████▊   | 1011/1485 [2:51:35<1:19:46, 10.10s/it]
 68%|██████▊   | 1012/1485 [2:51:45<1:19:28, 10.08s/it]
 68%|██████▊   | 1013/1485 [2:51:55<1:19:05, 10.05s/it]
 68%|██████▊   | 1014/1485 [2:52:05<1:18:47, 10.04s/it]
 68%|██████▊   | 1015/1485 [2:52:15<1:18:30, 10.02s/it]
 68%|██████▊   | 1016/1485 [2:52:25<1:18:26, 10.03s/it]
 68%|██████▊   | 1017/1485 [2:52:35
+0: {'loss': 0.6042, 'grad_norm': 1.1115342305830545, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.41}
+0: <1:18:17, 10.04s/it]
 69%|██████▊   | 1018/1485 [2:52:45<1:18:03, 10.03s/it]
 69%|██████▊   | 1019/1485 [2:52:56<1:17:58, 10.04s/it]
 69%|██████▊   | 1020/1485 [2:53:06<1:17:56, 10.06s/it]
                                                       

 69%|██████▊   | 1020/1485 [2:53:06<1:17:56, 10.06s/it]
 69%|██████▉   | 1021/1485 [2:53:16<1:17:53, 10.07s/it]
 69%|██████▉   | 1022/1485 [2:53:26<1:17:51, 10.09s/it]
 69%|██████▉   | 1023/1485 [2:53:36<1:17:52, 10.11s/it]
 69%|██████▉   | 1024/1485 [2:53:46<1:17:28, 10.08s/it]
 69%|██████▉   | 1025/1485 [2:53:56<1:17:13, 10.07s/it]
 69%|██████▉   | 1026/1485 [2:54:06<1:16:53, 10.05s/it]
 69%|██████▉   | 1027/1485 [2:54:16<1:16:36, 10.04s/it]
 69%|██████▉   | 1028/1485 [2:54:26<1:16:15, 10.01s/it]
 69%|██████▉   | 1029/1485 [2:54:36<1:15:57,  9.99s/it]
 69%|██████▉   | 1030/
+0: {'loss': 0.5939, 'grad_norm': 1.0957818972453761, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.42}
+0: {'loss': 0.5966, 'grad_norm': 1.0482459876936372, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.42}
+0: 1485 [2:54:46<1:15:42,  9.98s/it]
                                                       

 69%|██████▉   | 1030/1485 [2:54:46<1:15:42,  9.98s/it]
 69%|█████���▉   | 1031/1485 [2:54:56<1:15:49, 10.02s/it]
 69%|██████▉   | 1032/1485 [2:55:06<1:15:30, 10.00s/it]
 70%|██████▉   | 1033/1485 [2:55:16<1:15:23, 10.01s/it]
 70%|██████▉   | 1034/1485 [2:55:26<1:15:03,  9.99s/it]
 70%|██████▉   | 1035/1485 [2:55:36<1:14:56,  9.99s/it]
 70%|██████▉   | 1036/1485 [2:55:46<1:14:44,  9.99s/it]
 70%|██████▉   | 1037/1485 [2:55:56<1:14:42, 10.01s/it]
 70%|██████▉   | 1038/1485 [2:56:06<1:14:23,  9.99s/it]
 70%|██████▉   | 1039/1485 [2:56:16<1:14:19, 10.00s/it]
 70%|███████   | 1040/1485 [2:56:26<1:14:12, 10.01s/it]
                                                       

 70%|███████   | 1040/1485 [2:56:26<1:14:12, 10.01s/it]
 70%|███████   | 1041/
+0: {'loss': 0.5977, 'grad_norm': 0.9925437599923913, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.42}
+0: 1485 [2:56:36<1:14:03, 10.01s/it]
 70%|███████   | 1042/1485 [2:56:46<1:14:00, 10.02s/it]
 70%|███████   | 1043/1485 [2:56:56<1:13:54, 10.03s/it]
 70%|███████   | 1044/1485 [2:57:06<1:14:06, 10.08s/it]
 70%|███████   | 1045/1485 [2:57:16<1:13:46, 10.06s/it]
 70%|███████   | 1046/1485 [2:57:26<1:13:28, 10.04s/it]
 71%|███████   | 1047/1485 [2:57:36<1:13:22, 10.05s/it]
 71%|███████   | 1048/1485 [2:57:47<1:13:13, 10.05s/it]
 71%|███████   | 1049/1485 [2:57:56<1:12:51, 10.03s/it]
 71%|███████   | 1050/1485 [2:58:06<1:12:36, 10.01s/it]
                                                       

 71%|███████   | 1050/1485 [2:58:06<1:12:36, 10.01s/it]
 71%|███████   | 1051/1485 [2:58:16<1:12:22, 10.01s/it]
 71%|███████   | 1052/1485 [2:58:27<1:12:23, 10.03s/it]
 71%|███████   | 1053/1485 [2:58:37<1:12:18, 10.04s/it]
 71%|██████
+0: {'loss': 0.5881, 'grad_norm': 0.9931374882111083, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.43}
+0: █   | 1054/1485 [2:58:47<1:12:08, 10.04s/it]
 71%|███████   | 1055/1485 [2:58:57<1:11:51, 10.03s/it]
 71%|███████   | 1056/1485 [2:59:07<1:11:52, 10.05s/it]
 71%|███████   | 1057/1485 [2:59:17<1:12:32, 10.17s/it]
 71%|███████   | 1058/1485 [2:59:27<1:12:23, 10.17s/it]
 71%|███████▏  | 1059/1485 [2:59:38<1:12:15, 10.18s/it]
 71%|███████▏  | 1060/1485 [2:59:48<1:12:30, 10.24s/it]
                                                       

 71%|███████▏  | 1060/1485 [2:59:48<1:12:30, 10.24s/it]
 71%|███████▏  | 1061/1485 [2:59:58<1:12:15, 10.23s/it]
 72%|███████▏  | 1062/1485 [3:00:08<1:11:41, 10.17s/it]
 72%|███████▏  | 1063/1485 [3:00:18<1:11:39, 10.19s/it]
 72%|███████▏  | 1064/1485 [3:00:28<1:11:06, 10.14s/it]
 72%|███████▏  | 1065/1485 [3:00:38<1:10:45, 10.11s/it]
 72%|███████▏  | 1066/1485 [3:00:48<1:10:14, 10.
+0: {'loss': 0.5912, 'grad_norm': 1.1321451372479736, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.43}
+0: 06s/it]
 72%|███████▏  | 1067/1485 [3:00:58<1:09:47, 10.02s/it]
 72%|███████▏  | 1068/1485 [3:01:08<1:09:43, 10.03s/it]
 72%|███████▏  | 1069/1485 [3:01:18<1:09:39, 10.05s/it]
 72%|███████▏  | 1070/1485 [3:01:29<1:09:35, 10.06s/it]
                                                       

 72%|███████▏  | 1070/1485 [3:01:29<1:09:35, 10.06s/it]
 72%|███████▏  | 1071/1485 [3:01:39<1:09:30, 10.07s/it]
 72%|███████▏  | 1072/1485 [3:01:49<1:09:02, 10.03s/it]
 72%|███████▏  | 1073/1485 [3:01:59<1:08:56, 10.04s/it]
 72%|███████▏  | 1074/1485 [3:02:09<1:08:35, 10.01s/it]
 72%|███████▏  | 1075/1485 [3:02:19<1:08:26, 10.01s/it]
 72%|███████▏  | 1076/1485 [3:02:29<1:08:28, 10.04s/it]
 73%|███████▎  | 1077/1485 [3:02:39<1:08:07, 10.02s/it]
 73%|███████▎  | 1078/1485 [3:02:49<1:07:52, 10.01s/it]
 73%|██████
+0: {'loss': 0.5843, 'grad_norm': 0.9956499190776079, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.44}
+0: █▎  | 1079/1485 [3:02:59<1:07:42, 10.01s/it]
 73%|███████▎  | 1080/1485 [3:03:09<1:07:33, 10.01s/it]
                                                       

 73%|███████▎  | 1080/1485 [3:03:09<1:07:33, 10.01s/it]
 73%|███████▎  | 1081/1485 [3:03:19<1:07:35, 10.04s/it]
 73%|███��███▎  | 1082/1485 [3:03:29<1:07:32, 10.06s/it]
 73%|███████▎  | 1083/1485 [3:03:39<1:07:23, 10.06s/it]
 73%|███████▎  | 1084/1485 [3:03:49<1:07:13, 10.06s/it]
 73%|███████▎  | 1085/1485 [3:03:59<1:06:56, 10.04s/it]
 73%|███████▎  | 1086/1485 [3:04:09<1:06:36, 10.02s/it]
 73%|███████▎  | 1087/1485 [3:04:19<1:06:30, 10.03s/it]
 73%|███████▎  | 1088/1485 [3:04:30<1:07:13, 10.16s/it]
 73%|███████▎  | 1089/1485 [3:04:40<1:06:50, 10.13s/it]
 73%|███████▎  | 1090/1485 [3:04:50<1:06:25, 10.09s/it]
                                                      
+0: {'loss': 0.5855, 'grad_norm': 0.8784036987422035, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.44}
+0: {'loss': 0.5915, 'grad_norm': 1.021696024420777, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.44}
+0:  

 73%|███████▎  | 1090/1485 [3:04:50<1:06:25, 10.09s/it]
 73%|███████▎  | 1091/1485 [3:05:00<1:06:16, 10.09s/it]
 74%|███████▎  | 1092/1485 [3:05:10<1:05:52, 10.06s/it]
 74%|███████▎  | 1093/1485 [3:05:20<1:05:32, 10.03s/it]
 74%|███████▎  | 1094/1485 [3:05:31<1:07:38, 10.38s/it]
 74%|███████▎  | 1095/1485 [3:05:42<1:10:02, 10.78s/it]
 74%|███████▍  | 1096/1485 [3:05:53<1:09:15, 10.68s/it]
 74%|███████▍  | 1097/1485 [3:06:03<1:07:56, 10.51s/it]
 74%|███████▍  | 1098/1485 [3:06:13<1:07:10, 10.42s/it]
 74%|███████▍  | 1099/1485 [3:06:23<1:06:19, 10.31s/it]
 74%|███████▍  | 1100/1485 [3:06:34<1:05:57, 10.28s/it]
                                                       

 74%|███████▍  | 1100/1485 [3:06:34<1:05:57, 10.28s/it]
 74%|███████▍  | 1101/1485 [3:06:44<1:05:22, 10.21s/it]
 74%|███████�
+0: {'loss': 0.5855, 'grad_norm': 1.0263095545954193, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.45}
+0: �  | 1102/1485 [3:06:54<1:04:49, 10.16s/it]
 74%|███████▍  | 1103/1485 [3:07:04<1:04:08, 10.07s/it]
 74%|███████▍  | 1104/1485 [3:07:13<1:03:30, 10.00s/it]
 74%|███████▍  | 1105/1485 [3:07:23<1:03:12,  9.98s/it]
 74%|███████▍  | 1106/1485 [3:07:33<1:02:50,  9.95s/it]
 75%|███████▍  | 1107/1485 [3:07:43<1:02:33,  9.93s/it]
 75%|███████▍  | 1108/1485 [3:07:53<1:02:19,  9.92s/it]
 75%|███████▍  | 1109/1485 [3:08:03<1:02:42, 10.01s/it]
 75%|███████▍  | 1110/1485 [3:08:13<1:02:38, 10.02s/it]
                                                       

 75%|███████▍  | 1110/1485 [3:08:13<1:02:38, 10.02s/it]
 75%|███████▍  | 1111/1485 [3:08:23<1:02:27, 10.02s/it]
 75%|███████▍  | 1112/1485 [3:08:33<1:02:15, 10.02s/it]
 75%|███████▍  | 1113/1485 [3:08:43<1:02:08, 10.02s/it]
 75%|███████▌  | 1114/1485 [3:08:53<1:02:00
+0: {'loss': 0.5719, 'grad_norm': 1.122425155378109, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.45}
+0: , 10.03s/it]
 75%|███████▌  | 1115/1485 [3:09:03<1:01:50, 10.03s/it]
 75%|███████▌  | 1116/1485 [3:09:13<1:01:26,  9.99s/it]
 75%|███████▌  | 1117/1485 [3:09:23<1:01:17,  9.99s/it]
 75%|███████▌  | 1118/1485 [3:09:33<1:01:04,  9.99s/it]
 75%|███████▌  | 1119/1485 [3:09:43<1:00:55,  9.99s/it]
 75%|███████▌  | 1120/1485 [3:09:53<1:00:41,  9.98s/it]
                                                       

 75%|███████▌  | 1120/1485 [3:09:53<1:00:41,  9.98s/it]
 75%|███████▌  | 1121/1485 [3:10:03<1:00:26,  9.96s/it]
 76%|███████▌  | 1122/1485 [3:10:13<1:00:13,  9.96s/it]
 76%|███████▌  | 1123/1485 [3:10:23<1:00:19, 10.00s/it]
 76%|███████▌  | 1124/1485 [3:10:33<1:00:16, 10.02s/it]
 76%|███████▌  | 1125/1485 [3:10:43<1:00:21, 10.06s/it]
 76%|███████▌  | 1126/1485 [3:10:53<1:00:13, 10.07s/it]
 76%|████�
+0: {'loss': 0.5754, 'grad_norm': 1.013980908488649, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.46}
+0: ��██▌  | 1127/1485 [3:11:03<1:00:00, 10.06s/it]
 76%|███████▌  | 1128/1485 [3:11:13<59:38, 10.02s/it]  
 76%|███████▌  | 1129/1485 [3:11:23<59:22, 10.01s/it]
 76%|███████▌  | 1130/1485 [3:11:33<59:07,  9.99s/it]
                                                     

 76%|███████▌  | 1130/1485 [3:11:33<59:07,  9.99s/it]
 76%|███████▌  | 1131/1485 [3:11:43<58:54,  9.99s/it]
 76%|███████▌  | 1132/1485 [3:11:53<58:56, 10.02s/it]
 76%|███████▋  | 1133/1485 [3:12:03<58:52, 10.03s/it]
 76%|███████▋  | 1134/1485 [3:12:13<58:43, 10.04s/it]
 76%|███████▋  | 1135/1485 [3:12:24<58:41, 10.06s/it]
 76%|███████▋  | 1136/1485 [3:12:34<58:32, 10.07s/it]
 77%|███████▋  | 1137/1485 [3:12:44<58:26, 10.08s/it]
 77%|███████▋  | 1138/1485 [3:12:54<58:24, 10.10s/it]
 77%|███████▋  | 1139/1485 [3:13:04<58:10, 10.09s/it]
 77
+0: {'loss': 0.5764, 'grad_norm': 1.2649220614583778, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.46}
+0: {'loss': 0.5753, 'grad_norm': 0.9692656057783067, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.46}
+0: %|███████▋  | 1140/1485 [3:13:14<57:55, 10.08s/it]
                                                     

 77%|███████▋  | 1140/1485 [3:13:14<57:55, 10.08s/it]
 77%|███████▋  | 1141/1485 [3:13:24<57:52, 10.09s/it]
 77%|███████▋  | 1142/1485 [3:13:34<57:49, 10.11s/it]
 77%|███████▋  | 1143/1485 [3:13:44<57:28, 10.08s/it]
 77%|███████▋  | 1144/1485 [3:13:54<57:16, 10.08s/it]
 77%|███████▋  | 1145/1485 [3:14:05<57:09, 10.09s/it]
 77%|███████▋  | 1146/1485 [3:14:15<56:54, 10.07s/it]
 77%|███████▋  | 1147/1485 [3:14:25<56:45, 10.08s/it]
 77%|███████▋  | 1148/1485 [3:14:35<56:35, 10.07s/it]
 77%|███████▋  | 1149/1485 [3:14:45<56:25, 10.07s/it]
 77%|███████▋  | 1150/1485 [3:14:55<56:16, 10.08s/it]
                                                     

 77%|███████▋  | 1150/1485 [3:14:55<56:16, 10.08s/it]
 78%|�
+0: {'loss': 0.5692, 'grad_norm': 1.0706492342061156, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.47}
+0: �██████▊  | 1151/1485 [3:15:05<56:07, 10.08s/it]
 78%|███████▊  | 1152/1485 [3:15:15<55:43, 10.04s/it]
 78%|███████▊  | 1153/1485 [3:15:25<55:32, 10.04s/it]
 78%|███████▊  | 1154/1485 [3:15:35<55:25, 10.05s/it]
 78%|███████▊  | 1155/1485 [3:15:45<55:38, 10.12s/it]
 78%|███████▊  | 1156/1485 [3:15:55<55:14, 10.08s/it]
 78%|███████▊  | 1157/1485 [3:16:05<55:10, 10.09s/it]
 78%|███████▊  | 1158/1485 [3:16:16<55:00, 10.09s/it]
 78%|███████▊  | 1159/1485 [3:16:26<55:04, 10.14s/it]
 78%|███████▊  | 1160/1485 [3:16:36<54:48, 10.12s/it]
                                                     

 78%|███████▊  | 1160/1485 [3:16:36<54:48, 10.12s/it]
 78%|███████▊  | 1161/1485 [3:16:46<54:26, 10.08s/it]
 78%|███████▊  | 1162/1485 [3:16:56<54:10, 10.06s/it]
 78%|███████▊  | 1163/1485 [3:17:06<54:03, 10.07s/
+0: {'loss': 0.5608, 'grad_norm': 0.95228789237032, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.47}
+0: it]
 78%|███████▊  | 1164/1485 [3:17:16<54:12, 10.13s/it]
 78%|███████▊  | 1165/1485 [3:17:26<54:03, 10.14s/it]
 79%|███████▊  | 1166/1485 [3:17:36<53:52, 10.13s/it]
 79%|███████▊  | 1167/1485 [3:17:47<53:59, 10.19s/it]
 79%|███████▊  | 1168/1485 [3:17:57<53:31, 10.13s/it]
 79%|███████▊  | 1169/1485 [3:18:07<53:22, 10.13s/it]
 79%|███████▉  | 1170/1485 [3:18:17<53:14, 10.14s/it]
                                                     

 79%|███████▉  | 1170/1485 [3:18:17<53:14, 10.14s/it]
 79%|███████▉  | 1171/1485 [3:18:27<53:09, 10.16s/it]
 79%|███████▉  | 1172/1485 [3:18:37<53:01, 10.16s/it]
 79%|███████▉  | 1173/1485 [3:18:48<52:46, 10.15s/it]
 79%|███████▉  | 1174/1485 [3:18:58<52:36, 10.15s/it]
 79%|███████▉  | 1175/1485 [3:19:08<52:18, 10.12s/it]
 79%|███████▉  | 1176/1485 [3:19:18<52:
+0: {'loss': 0.585, 'grad_norm': 1.0084552655315817, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.48}
+0: 05, 10.12s/it]
 79%|███████▉  | 1177/1485 [3:19:28<51:57, 10.12s/it]
 79%|███████▉  | 1178/1485 [3:19:38<51:44, 10.11s/it]
 79%|███████▉  | 1179/1485 [3:19:48<51:33, 10.11s/it]
 79%|��██████▉  | 1180/1485 [3:19:58<51:22, 10.11s/it]
                                                     

 79%|███████▉  | 1180/1485 [3:19:58<51:22, 10.11s/it]
 80%|███████▉  | 1181/1485 [3:20:08<51:15, 10.12s/it]
 80%|███████▉  | 1182/1485 [3:20:19<51:06, 10.12s/it]
 80%|███████▉  | 1183/1485 [3:20:29<50:57, 10.13s/it]
 80%|███████▉  | 1184/1485 [3:20:39<50:35, 10.09s/it]
 80%|███████▉  | 1185/1485 [3:20:49<50:25, 10.09s/it]
 80%|███████▉  | 1186/1485 [3:20:59<50:15, 10.08s/it]
 80%|███████▉  | 1187/1485 [3:21:09<50:02, 10.08s/it]
 80%|████████  | 1188/1485 [3:21:19<49:51, 10.07s/it]
 80%|████████  | 1189/1485 [
+0: {'loss': 0.5595, 'grad_norm': 0.9565808511881246, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.48}
+0: {'loss': 0.5646, 'grad_norm': 0.9892358947091678, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.48}
+0: 3:21:29<49:33, 10.05s/it]
 80%|████████  | 1190/1485 [3:21:40<50:45, 10.32s/it]
                                                     

 80%|████████  | 1190/1485 [3:21:40<50:45, 10.32s/it]
 80%|████████  | 1191/1485 [3:21:51<51:25, 10.49s/it]
 80%|████████  | 1192/1485 [3:22:01<51:07, 10.47s/it]
 80%|████████  | 1193/1485 [3:22:12<51:34, 10.60s/it]
 80%|████████  | 1194/1485 [3:22:22<50:29, 10.41s/it]
 80%|████████  | 1195/1485 [3:22:32<49:43, 10.29s/it]
 81%|████████  | 1196/1485 [3:22:42<49:08, 10.20s/it]
 81%|████████  | 1197/1485 [3:22:52<48:41, 10.14s/it]
 81%|████████  | 1198/1485 [3:23:02<48:19, 10.10s/it]
 81%|████████  | 1199/1485 [3:23:12<48:01, 10.08s/it]
 81%|████████  | 1200/1485 [3:23:22<47:43, 10.05s/it]
                                                     

 81%|████████  | 1200/1485 [3:23
+0: {'loss': 0.5756, 'grad_norm': 0.9665923341804936, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.49}
+0: :22<47:43, 10.05s/it]
 81%|████████  | 1201/1485 [3:23:32<47:40, 10.07s/it]
 81%|████████  | 1202/1485 [3:23:42<47:26, 10.06s/it]
 81%|████████  | 1203/1485 [3:23:52<47:13, 10.05s/it]
 81%|████████  | 1204/1485 [3:24:02<46:58, 10.03s/it]
 81%|████████  | 1205/1485 [3:24:12<46:44, 10.02s/it]
 81%|████████  | 1206/1485 [3:24:22<46:40, 10.04s/it]
 81%|████████▏ | 1207/1485 [3:24:33<46:38, 10.07s/it]
 81%|████████▏ | 1208/1485 [3:24:43<46:28, 10.07s/it]
 81%|████████▏ | 1209/1485 [3:24:53<46:17, 10.06s/it]
 81%|████████▏ | 1210/1485 [3:25:03<46:03, 10.05s/it]
                                                     

 81%|████████▏ | 1210/1485 [3:25:03<46:03, 10.05s/it]
 82%|████████▏ | 1211/1485 [3:25:13<46:22, 10.16s/it]
 82%|████████▏ | 1212/1485 [3:25:23<46:00, 10.11s/it]
 82%|██████
+0: {'loss': 0.5664, 'grad_norm': 1.286844817633749, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.49}
+0: ██▏ | 1213/1485 [3:25:33<45:41, 10.08s/it]
 82%|████████▏ | 1214/1485 [3:25:43<45:27, 10.07s/it]
 82%|████████▏ | 1215/1485 [3:25:53<45:17, 10.07s/it]
 82%|████████▏ | 1216/1485 [3:26:03<44:58, 10.03s/it]
 82%|████████▏ | 1217/1485 [3:26:13<44:41, 10.01s/it]
 82%|████████▏ | 1218/1485 [3:26:23<44:31, 10.01s/it]
 82%|████████▏ | 1219/1485 [3:26:33<44:18,  9.99s/it]
 82%|████████▏ | 1220/1485 [3:26:43<44:03,  9.98s/it]
                                                     

 82%|████████▏ | 1220/1485 [3:26:43<44:03,  9.98s/it]
 82%|████████▏ | 1221/1485 [3:26:53<43:44,  9.94s/it]
 82%|████████▏ | 1222/1485 [3:27:03<43:29,  9.92s/it]
 82%|████████▏ | 1223/1485 [3:27:13<43:19,  9.92s/it]
 82%|████████▏ | 1224/1485 [3:27:23<43:09,  9.92s/it]
 82%|████████▏ | 1225/1485 [3:27:32<42
+0: {'loss': 0.5622, 'grad_norm': 1.452982388945615, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.5}
+0: :56,  9.91s/it]
 83%|████████▎ | 1226/1485 [3:27:42<42:44,  9.90s/it]
 83%|████████▎ | 1227/1485 [3:27:52<42:35,  9.90s/it]
 83%|████��███▎ | 1228/1485 [3:28:02<42:26,  9.91s/it]
 83%|████████▎ | 1229/1485 [3:28:12<42:18,  9.92s/it]
 83%|████████▎ | 1230/1485 [3:28:22<42:12,  9.93s/it]
                                                     

 83%|████████▎ | 1230/1485 [3:28:22<42:12,  9.93s/it]
 83%|████████▎ | 1231/1485 [3:28:32<42:02,  9.93s/it]
 83%|████████▎ | 1232/1485 [3:28:42<41:50,  9.92s/it]
 83%|████████▎ | 1233/1485 [3:28:52<41:33,  9.89s/it]
 83%|████████▎ | 1234/1485 [3:29:02<41:19,  9.88s/it]
 83%|████████▎ | 1235/1485 [3:29:11<41:12,  9.89s/it]
 83%|████████▎ | 1236/1485 [3:29:21<41:06,  9.91s/it]
 83%|████████▎ | 1237/1485 [3:29:31<41:01,  9.93s/it]
 83%|████
+0: {'loss': 0.5622, 'grad_norm': 1.0160603795164613, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.5}
+0: ████▎ | 1238/1485 [3:29:41<40:49,  9.92s/it]
 83%|████████▎ | 1239/1485 [3:29:51<40:42,  9.93s/it]
 84%|████████▎ | 1240/1485 [3:30:01<40:33,  9.93s/it]
                                                     

 84%|████████▎ | 1240/1485 [3:30:01<40:33,  9.93s/it]
 84%|████████▎ | 1241/1485 [3:30:11<40:28,  9.95s/it]
 84%|████████▎ | 1242/1485 [3:30:21<40:20,  9.96s/it]
 84%|████████▎ | 1243/1485 [3:30:31<40:05,  9.94s/it]
 84%|████████▍ | 1244/1485 [3:30:41<39:56,  9.94s/it]
 84%|████████▍ | 1245/1485 [3:30:51<39:45,  9.94s/it]
 84%|████████▍ | 1246/1485 [3:31:01<39:33,  9.93s/it]
 84%|████████▍ | 1247/1485 [3:31:11<39:27,  9.95s/it]
 84%|████████▍ | 1248/1485 [3:31:21<39:17,  9.95s/it]
 84%|████████▍ | 1249/1485 [3:31:31<39:06,  9.94s/it]
 84%|████████▍ | 1250/1485 [3:31
+0: {'loss': 0.5612, 'grad_norm': 1.0536412444770056, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.51}
+0: {'loss': 0.561, 'grad_norm': 0.9129946529617663, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.51}
+0: :41<38:57,  9.95s/it]
                                                     

 84%|████████▍ | 1250/1485 [3:31:41<38:57,  9.95s/it]
 84%|████████▍ | 1251/1485 [3:31:51<38:53,  9.97s/it]
 84%|████████▍ | 1252/1485 [3:32:01<38:45,  9.98s/it]
 84%|████████▍ | 1253/1485 [3:32:11<38:39, 10.00s/it]
 84%|████████▍ | 1254/1485 [3:32:21<38:30, 10.00s/it]
 85%|████████▍ | 1255/1485 [3:32:31<38:21, 10.01s/it]
 85%|████████▍ | 1256/1485 [3:32:41<38:11, 10.01s/it]
 85%|████████▍ | 1257/1485 [3:32:51<37:59, 10.00s/it]
 85%|████████▍ | 1258/1485 [3:33:01<37:47,  9.99s/it]
 85%|████████▍ | 1259/1485 [3:33:11<37:36,  9.98s/it]
 85%|████████▍ | 1260/1485 [3:33:21<37:28,  9.99s/it]
                                                     

 85%|████████▍ | 1260/1485 [3:33:21<37:28,  9.99s/it]
 85%|███████�
+0: {'loss': 0.5601, 'grad_norm': 1.1080715652049435, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.51}
+0: �▍ | 1261/1485 [3:33:31<37:11,  9.96s/it]
 85%|████████▍ | 1262/1485 [3:33:40<36:53,  9.92s/it]
 85%|████████▌ | 1263/1485 [3:33:50<36:36,  9.89s/it]
 85%|████████▌ | 1264/1485 [3:34:00<36:17,  9.85s/it]
 85%|████████▌ | 1265/1485 [3:34:10<36:05,  9.84s/it]
 85%|████████▌ | 1266/1485 [3:34:20<35:51,  9.82s/it]
 85%|████████▌ | 1267/1485 [3:34:29<35:40,  9.82s/it]
 85%|████████▌ | 1268/1485 [3:34:39<35:31,  9.82s/it]
 85%|████████▌ | 1269/1485 [3:34:49<35:19,  9.81s/it]
 86%|████████▌ | 1270/1485 [3:34:59<35:10,  9.82s/it]
                                                     

 86%|████████▌ | 1270/1485 [3:34:59<35:10,  9.82s/it]
 86%|████████▌ | 1271/1485 [3:35:09<35:01,  9.82s/it]
 86%|████████▌ | 1272/1485 [3:35:19<35:09,  9.90s/it]
 86%|████████▌ | 1273/1485 [3:35:29<35:13, 
+0: {'loss': 0.5504, 'grad_norm': 0.9871547040517965, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.52}
+0:  9.97s/it]
 86%|████████▌ | 1274/1485 [3:35:39<35:09, 10.00s/it]
 86%|████████▌ | 1275/1485 [3:35:49<34:57,  9.99s/it]
 86%|████████▌ | 1276/1485 [3:35:59<34:45,  9.98s/it]
 86%|████████▌ | 1277/1485 [3:36:09<34:33,  9.97s/it]
 86%|████████▌ | 1278/1485 [3:36:19<34:23,  9.97s/it]
 86%|████████▌ | 1279/1485 [3:36:29<34:13,  9.97s/it]
 86%|████████▌ | 1280/1485 [3:36:39<34:01,  9.96s/it]
                                                     

 86%|████████▌ | 1280/1485 [3:36:39<34:01,  9.96s/it]
 86%|████████▋ | 1281/1485 [3:36:48<33:39,  9.90s/it]
 86%|████████▋ | 1282/1485 [3:36:58<33:23,  9.87s/it]
 86%|████████▋ | 1283/1485 [3:37:08<33:12,  9.86s/it]
 86%|████████▋ | 1284/1485 [3:37:18<33:00,  9.85s/it]
 87%|████████▋ | 1285/1485 [3:37:28<32:47,  9.84s/it]
 87%|█████�
+0: {'loss': 0.5494, 'grad_norm': 1.1345222078197241, 'learning_rate': 4.995560138963612e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.52}
+0: �██▋ | 1286/1485 [3:37:38<32:34,  9.82s/it]
 87%|████████▋ | 1287/1485 [3:37:47<32:22,  9.81s/it]
 87%|████████▋ | 1288/1485 [3:37:57<32:07,  9.78s/it]
 87%|████████▋ | 1289/1485 [3:38:07<31:56,  9.78s/it]
 87%|████████▋ | 1290/1485 [3:38:17<31:44,  9.76s/it]
                                                     

 87%|████████▋ | 1290/1485 [3:38:17<31:44,  9.76s/it]
 87%|████████▋ | 1291/1485 [3:38:26<31:35,  9.77s/it]
 87%|████████▋ | 1292/1485 [3:38:36<31:22,  9.75s/it]
 87%|████████▋ | 1293/1485 [3:38:46<31:11,  9.75s/it]
 87%|████████▋ | 1294/1485 [3:38:56<31:08,  9.78s/it]
 87%|████████▋ | 1295/1485 [3:39:06<31:07,  9.83s/it]
 87%|████████▋ | 1296/1485 [3:39:15<30:58,  9.83s/it]
 87%|████████▋ | 1297/1485 [3:39:25<30:48,  9.83s/it]
 87%|████████▋ | 1298/1485 [3:39:35<3
+0: {'loss': 0.556, 'grad_norm': 0.9374886518374393, 'learning_rate': 4.945812714362182e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.53}
+0: 0:41,  9.85s/it]
 87%|████████▋ | 1299/1485 [3:39:45<30:29,  9.84s/it]
 88%|████████▊ | 1300/1485 [3:39:55<30:17,  9.83s/it]
                                                     

 88%|████████▊ | 1300/1485 [3:39:55<30:17,  9.83s/it]
 88%|████████▊ | 1301/1485 [3:40:05<30:10,  9.84s/it]
 88%|████████▊ | 1302/1485 [3:40:14<30:00,  9.84s/it]
 88%|████████▊ | 1303/1485 [3:40:24<29:52,  9.85s/it]
 88%|████████▊ | 1304/1485 [3:40:35<30:00,  9.95s/it]
 88%|████████▊ | 1305/1485 [3:40:44<29:42,  9.90s/it]
 88%|████████▊ | 1306/1485 [3:40:54<29:26,  9.87s/it]
 88%|████████▊ | 1307/1485 [3:41:04<29:11,  9.84s/it]
 88%|████████▊ | 1308/1485 [3:41:14<29:04,  9.85s/it]
 88%|████████▊ | 1309/1485 [3:41:24<28:57,  9.87s/it]
 88%|████████▊ | 1310/1485 [3:41:34<28:50,  9.89s/it]
                
+0: {'loss': 0.5552, 'grad_norm': 1.2100924239054385, 'learning_rate': 4.841997093248566e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.53}
+0: {'loss': 0.5489, 'grad_norm': 0.931812325919886, 'learning_rate': 4.686669560758874e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.53}
+0:                                      

 88%|████████▊ | 1310/1485 [3:41:34<28:50,  9.89s/it]
 88%|████████▊ | 1311/1485 [3:41:44<28:43,  9.91s/it]
 88%|████████▊ | 1312/1485 [3:41:54<28:36,  9.92s/it]
 88%|████████▊ | 1313/1485 [3:42:03<28:27,  9.93s/it]
 88%|████████▊ | 1314/1485 [3:42:13<28:18,  9.93s/it]
 89%|████████▊ | 1315/1485 [3:42:23<28:09,  9.94s/it]
 89%|████████▊ | 1316/1485 [3:42:33<27:58,  9.93s/it]
 89%|████████▊ | 1317/1485 [3:42:43<27:48,  9.93s/it]
 89%|████████▉ | 1318/1485 [3:42:53<27:33,  9.90s/it]
 89%|████████▉ | 1319/1485 [3:43:03<27:21,  9.89s/it]
 89%|████████▉ | 1320/1485 [3:43:13<27:10,  9.88s/it]
                                                     

 89%|████████▉ | 1320/1485 [3:43:13<27:10,  9.88s/it]
 89%|████████▉ | 1321/1485 [3:43:23<26:56,  9.85
+0: {'loss': 0.5559, 'grad_norm': 0.9283714915897459, 'learning_rate': 4.483654796245526e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.54}
+0: s/it]
 89%|████████▉ | 1322/1485 [3:43:32<26:44,  9.84s/it]
 89%|████████▉ | 1323/1485 [3:43:42<26:35,  9.85s/it]
 89%|████████▉ | 1324/1485 [3:43:52<26:24,  9.84s/it]
 89%|████████▉ | 1325/1485 [3:44:02<26:21,  9.89s/it]
 89%|████████▉ | 1326/1485 [3:44:12<26:11,  9.88s/it]
 89%|████████▉ | 1327/1485 [3:44:22<25:57,  9.86s/it]
 89%|████████▉ | 1328/1485 [3:44:32<25:47,  9.86s/it]
 89%|████████▉ | 1329/1485 [3:44:41<25:37,  9.85s/it]
 90%|████████▉ | 1330/1485 [3:44:51<25:27,  9.86s/it]
                                                     

 90%|████████▉ | 1330/1485 [3:44:51<25:27,  9.86s/it]
 90%|████████▉ | 1331/1485 [3:45:01<25:21,  9.88s/it]
 90%|████████▉ | 1332/1485 [3:45:11<25:14,  9.90s/it]
 90%|████████▉ | 1333/1485 [3:45:21<25:03,  9.89s/it]
 90%|███████�
+0: {'loss': 0.556, 'grad_norm': 0.8923782043400419, 'learning_rate': 4.237951696978217e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.54}
+0: ��▉ | 1334/1485 [3:45:31<24:52,  9.89s/it]
 90%|████████▉ | 1335/1485 [3:45:41<24:43,  9.89s/it]
 90%|████████▉ | 1336/1485 [3:45:51<24:38,  9.92s/it]
 90%|█████████ | 1337/1485 [3:46:01<24:34,  9.96s/it]
 90%|█████████ | 1338/1485 [3:46:11<24:32, 10.01s/it]
 90%|█████████ | 1339/1485 [3:46:21<24:22, 10.02s/it]
 90%|█████████ | 1340/1485 [3:46:31<24:11, 10.01s/it]
                                                     

 90%|█████████ | 1340/1485 [3:46:31<24:11, 10.01s/it]
 90%|█████████ | 1341/1485 [3:46:41<24:05, 10.04s/it]
 90%|█████████ | 1342/1485 [3:46:51<23:49,  9.99s/it]
 90%|█████████ | 1343/1485 [3:47:01<23:33,  9.96s/it]
 91%|█████████ | 1344/1485 [3:47:11<23:17,  9.91s/it]
 91%|█████████ | 1345/1485 [3:47:21<23:04,  9.89s/it]
 91%|█████████ | 1346/1485 [3:47:30<22:49,
+0: {'loss': 0.5523, 'grad_norm': 0.9575210150516472, 'learning_rate': 3.9556102887027425e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.55}
+0:   9.85s/it]
 91%|█████████ | 1347/1485 [3:47:40<22:33,  9.81s/it]
 91%|█████████ | 1348/1485 [3:47:50<22:18,  9.77s/it]
 91%|█████████ | 1349/1485 [3:47:59<22:06,  9.75s/it]
 91%|█████████ | 1350/1485 [3:48:09<21:56,  9.75s/it]
                                                     

 91%|█████████ | 1350/1485 [3:48:09<21:56,  9.75s/it]
 91%|█████████ | 1351/1485 [3:48:19<21:49,  9.77s/it]
 91%|█████████ | 1352/1485 [3:48:29<21:41,  9.78s/it]
 91%|█████████ | 1353/1485 [3:48:39<21:32,  9.79s/it]
 91%|█████████ | 1354/1485 [3:48:48<21:21,  9.78s/it]
 91%|█████████ | 1355/1485 [3:48:58<21:12,  9.79s/it]
 91%|█████████▏| 1356/1485 [3:49:08<21:02,  9.79s/it]
 91%|█████████▏| 1357/1485 [3:49:18<20:52,  9.78s/it]
 91%|█████████▏| 1358/1485 [3:49:28<20:43,  9.79s/it]
 92%|███�
+0: {'loss': 0.5486, 'grad_norm': 0.8597067846920559, 'learning_rate': 3.6435827539282564e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.55}
+0: ��█████▏| 1359/1485 [3:49:37<20:35,  9.81s/it]
 92%|█████████▏| 1360/1485 [3:49:47<20:27,  9.82s/it]
                                                     

 92%|█████████▏| 1360/1485 [3:49:47<20:27,  9.82s/it]
 92%|█████████▏| 1361/1485 [3:49:57<20:19,  9.83s/it]
 92%|█████████▏| 1362/1485 [3:50:07<20:11,  9.85s/it]
 92%|█████████▏| 1363/1485 [3:50:17<20:02,  9.86s/it]
 92%|█████████▏| 1364/1485 [3:50:27<19:55,  9.88s/it]
 92%|█████████▏| 1365/1485 [3:50:37<19:45,  9.88s/it]
 92%|█████████▏| 1366/1485 [3:50:46<19:34,  9.87s/it]
 92%|█████████▏| 1367/1485 [3:50:56<19:20,  9.83s/it]
 92%|█████████▏| 1368/1485 [3:51:06<19:09,  9.83s/it]
 92%|█████████▏| 1369/1485 [3:51:16<19:00,  9.83s/it]
 92%|█████████▏| 1370/1485 [3:51:26<18:49,  9.82s/it]
                      
+0: {'loss': 0.5466, 'grad_norm': 0.848775244700898, 'learning_rate': 3.309552246120924e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.55}
+0: {'loss': 0.5552, 'grad_norm': 0.9190679986437699, 'learning_rate': 2.9617437049666582e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.56}
+0:                                

 92%|█████████▏| 1370/1485 [3:51:26<18:49,  9.82s/it]
 92%|█████████▏| 1371/1485 [3:51:36<18:43,  9.85s/it]
 92%|█████████▏| 1372/1485 [3:51:45<18:28,  9.81s/it]
 92%|█████████▏| 1373/1485 [3:51:55<18:26,  9.88s/it]
 93%|█████████▎| 1374/1485 [3:52:05<18:21,  9.93s/it]
 93%|█████████▎| 1375/1485 [3:52:15<18:14,  9.95s/it]
 93%|█████████▎| 1376/1485 [3:52:25<17:57,  9.89s/it]
 93%|█████████▎| 1377/1485 [3:52:35<17:44,  9.85s/it]
 93%|█████████▎| 1378/1485 [3:52:45<17:32,  9.83s/it]
 93%|█████████▎| 1379/1485 [3:52:55<17:21,  9.83s/it]
 93%|█████████▎| 1380/1485 [3:53:04<17:11,  9.82s/it]
                                                     

 93%|█████████▎| 1380/1485 [3:53:04<17:11,  9.82s/it]
 93%|█████████▎| 1381/1485 [
+0: {'loss': 0.5661, 'grad_norm': 0.957116469830608, 'learning_rate': 2.6087213310590455e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.56}
+0: 3:53:14<17:01,  9.82s/it]
 93%|█████████▎| 1382/1485 [3:53:24<16:50,  9.81s/it]
 93%|█████████▎| 1383/1485 [3:53:34<16:39,  9.80s/it]
 93%|█████████▎| 1384/1485 [3:53:44<16:31,  9.82s/it]
 93%|█████████▎| 1385/1485 [3:53:53<16:21,  9.82s/it]
 93%|█████████▎| 1386/1485 [3:54:03<16:11,  9.81s/it]
 93%|█████████▎| 1387/1485 [3:54:13<16:00,  9.80s/it]
 93%|█████████▎| 1388/1485 [3:54:23<15:49,  9.79s/it]
 94%|█████████▎| 1389/1485 [3:54:33<15:39,  9.79s/it]
 94%|█████████▎| 1390/1485 [3:54:42<15:29,  9.78s/it]
                                                     

 94%|█████████▎| 1390/1485 [3:54:42<15:29,  9.78s/it]
 94%|█████████▎| 1391/1485 [3:54:52<15:20,  9.79s/it]
 94%|█████████▎| 1392/1485 [3:55:02<15:08,  9.76s/it]
 94%|█████████▍| 1393/1485 [3:55:12
+0: {'loss': 0.5476, 'grad_norm': 0.8947702045869597, 'learning_rate': 2.2591777068577802e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.57}
+0: <14:57,  9.76s/it]
 94%|█████████▍| 1394/1485 [3:55:21<14:49,  9.77s/it]
 94%|█████████▍| 1395/1485 [3:55:31<14:39,  9.77s/it]
 94%|█████████▍| 1396/1485 [3:55:41<14:33,  9.81s/it]
 94%|█████████▍| 1397/1485 [3:55:51<14:27,  9.86s/it]
 94%|█████████▍| 1398/1485 [3:56:01<14:22,  9.92s/it]
 94%|█████████▍| 1399/1485 [3:56:11<14:11,  9.90s/it]
 94%|█████████▍| 1400/1485 [3:56:21<14:00,  9.89s/it]
                                                     

 94%|█████████▍| 1400/1485 [3:56:21<14:00,  9.89s/it]
 94%|█████████▍| 1401/1485 [3:56:31<13:49,  9.87s/it]
 94%|█████████▍| 1402/1485 [3:56:40<13:38,  9.87s/it]
 94%|█████████▍| 1403/1485 [3:56:50<13:29,  9.87s/it]
 95%|█████████▍| 1404/1485 [3:57:00<13:18,  9.85s/it]
 95%|█████████▍| 1405/1485 [3:57:10<13:09,
+0: {'loss': 0.5461, 'grad_norm': 0.9965990741811698, 'learning_rate': 1.9217197564594746e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.57}
+0:   9.87s/it]
 95%|█████████▍| 1406/1485 [3:57:20<13:06,  9.95s/it]
 95%|█████████▍| 1407/1485 [3:57:30<13:02, 10.04s/it]
 95%|█████████▍| 1408/1485 [3:57:41<12:56, 10.08s/it]
 95%|█████████▍| 1409/1485 [3:57:51<12:48, 10.11s/it]
 95%|█████████▍| 1410/1485 [3:58:01<12:39, 10.13s/it]
                                                     

 95%|█████████▍| 1410/1485 [3:58:01<12:39, 10.13s/it]
 95%|█████████▌| 1411/1485 [3:58:11<12:29, 10.13s/it]
 95%|█████████▌| 1412/1485 [3:58:21<12:16, 10.09s/it]
 95%|█████████▌| 1413/1485 [3:58:31<12:03, 10.05s/it]
 95%|█████████▌| 1414/1485 [3:58:41<11:50, 10.01s/it]
 95%|█████████▌| 1415/1485 [3:58:51<11:41, 10.02s/it]
 95%|█████████▌| 1416/1485 [3:59:01<11:31, 10.02s/it]
 95%|█████████▌| 1417/1485 [3:59:11<11:19,  9.99s
+0: {'loss': 0.541, 'grad_norm': 0.8509585086694226, 'learning_rate': 1.6046568145616647e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.57}
+0: /it]
 95%|█████████▌| 1418/1485 [3:59:21<11:09,  9.99s/it]
 96%|█████████▌| 1419/1485 [3:59:31<11:00, 10.01s/it]
 96%|█████████▌| 1420/1485 [3:59:41<10:52, 10.04s/it]
                                                     

 96%|█████████▌| 1420/1485 [3:59:41<10:52, 10.04s/it]
 96%|█████████▌| 1421/1485 [3:59:51<10:43, 10.06s/it]
 96%|█████████▌| 1422/1485 [4:00:01<10:33, 10.05s/it]
 96%|█████████▌| 1423/1485 [4:00:11<10:23, 10.06s/it]
 96%|█████████▌| 1424/1485 [4:00:21<10:13, 10.06s/it]
 96%|█████████▌| 1425/1485 [4:00:31<10:03, 10.05s/it]
 96%|█████████▌| 1426/1485 [4:00:41<09:51, 10.03s/it]
 96%|█████████▌| 1427/1485 [4:00:51<09:41, 10.02s/it]
 96%|█████████▌| 1428/1485 [4:01:02<09:34, 10.07s/it]
 96%|█████████▌| 1429/1485 [4:01:12<09:25, 10.10s/it]
 9
+0: {'loss': 0.5434, 'grad_norm': 0.8297780054539933, 'learning_rate': 1.3157960230654482e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.58}
+0: {'loss': 0.5408, 'grad_norm': 0.914450827070863, 'learning_rate': 1.0622500933314661e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.58}
+0: 6%|█████████▋| 1430/1485 [4:01:22<09:16, 10.11s/it]
                                                     

 96%|█████████▋| 1430/1485 [4:01:22<09:16, 10.11s/it]
 96%|█████████▋| 1431/1485 [4:01:32<09:07, 10.14s/it]
 96%|█████████▋| 1432/1485 [4:01:42<08:57, 10.15s/it]
 96%|█████████▋| 1433/1485 [4:01:53<08:49, 10.18s/it]
 97%|█████████▋| 1434/1485 [4:02:03<08:39, 10.19s/it]
 97%|█████████▋| 1435/1485 [4:02:13<08:25, 10.11s/it]
 97%|█████████▋| 1436/1485 [4:02:23<08:13, 10.08s/it]
 97%|█████████▋| 1437/1485 [4:02:33<08:03, 10.08s/it]
 97%|█████████▋| 1438/1485 [4:02:43<07:54, 10.10s/it]
 97%|█████████▋| 1439/1485 [4:02:53<07:45, 10.11s/it]
 97%|█████████▋| 1440/1485 [4:03:03<07:36, 10.14s/it]
                                                     

 97%|███████�
+0: {'loss': 0.5402, 'grad_norm': 0.8220799986658005, 'learning_rate': 8.502621676204667e-07, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.59}
+0: �█▋| 1440/1485 [4:03:03<07:36, 10.14s/it]
 97%|█████████▋| 1441/1485 [4:03:13<07:27, 10.17s/it]
 97%|█████████▋| 1442/1485 [4:03:24<07:16, 10.15s/it]
 97%|█████████▋| 1443/1485 [4:03:34<07:04, 10.11s/it]
 97%|█████████▋| 1444/1485 [4:03:44<06:52, 10.07s/it]
 97%|█████████▋| 1445/1485 [4:03:54<06:42, 10.07s/it]
 97%|█████████▋| 1446/1485 [4:04:04<06:31, 10.03s/it]
 97%|█████████▋| 1447/1485 [4:04:14<06:21, 10.04s/it]
 98%|█████████▊| 1448/1485 [4:04:24<06:12, 10.06s/it]
 98%|█████████▊| 1449/1485 [4:04:34<06:03, 10.08s/it]
 98%|█████████▊| 1450/1485 [4:04:44<05:53, 10.09s/it]
                                                     

 98%|█████████▊| 1450/1485 [4:04:44<05:53, 10.09s/it]
 98%|█████████▊| 1451/1485 [4:04:54<05:45, 10.17s/it]
 98%|█████████▊
+0: {'loss': 0.5342, 'grad_norm': 0.9009899621677557, 'learning_rate': 6.850520922110425e-07, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.59}
+0: | 1452/1485 [4:05:04<05:34, 10.15s/it]
 98%|█████████▊| 1453/1485 [4:05:15<05:24, 10.14s/it]
 98%|█████████▊| 1454/1485 [4:05:25<05:14, 10.15s/it]
 98%|█████████▊| 1455/1485 [4:05:35<05:04, 10.14s/it]
 98%|█████████▊| 1456/1485 [4:05:45<04:54, 10.15s/it]
 98%|█████████▊| 1457/1485 [4:05:55<04:44, 10.16s/it]
 98%|█████████▊| 1458/1485 [4:06:05<04:34, 10.18s/it]
 98%|█████████▊| 1459/1485 [4:06:16<04:25, 10.20s/it]
 98%|█████████▊| 1460/1485 [4:06:26<04:16, 10.26s/it]
                                                     

 98%|█████████▊| 1460/1485 [4:06:26<04:16, 10.26s/it]
 98%|█████████▊| 1461/1485 [4:06:36<04:07, 10.29s/it]
 98%|█████████▊| 1462/1485 [4:06:47<03:56, 10.28s/it]
 99%|█████████▊| 1463/1485 [4:06:57<03:47, 10.33s/it]
 99%|█████████▊| 1464/
+0: {'loss': 0.5403, 'grad_norm': 0.893829940300402, 'learning_rate': 5.706878874605801e-07, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.59}
+0: 1485 [4:07:08<03:37, 10.37s/it]
 99%|█████████▊| 1465/1485 [4:07:18<03:27, 10.38s/it]
 99%|█████████▊| 1466/1485 [4:07:28<03:17, 10.40s/it]
 99%|█████████▉| 1467/1485 [4:07:39<03:07, 10.41s/it]
 99%|█████████▉| 1468/1485 [4:07:49<02:55, 10.35s/it]
 99%|█████████▉| 1469/1485 [4:07:59<02:44, 10.28s/it]
 99%|█████████▉| 1470/1485 [4:08:09<02:34, 10.27s/it]
                                                     

 99%|█████████▉| 1470/1485 [4:08:09<02:34, 10.27s/it]
 99%|█████████▉| 1471/1485 [4:08:20<02:23, 10.28s/it]
 99%|█████████▉| 1472/1485 [4:08:30<02:13, 10.26s/it]
 99%|█████████▉| 1473/1485 [4:08:40<02:02, 10.24s/it]
 99%|█████████▉| 1474/1485 [4:08:50<01:52, 10.23s/it]
 99%|█████████▉| 1475/1485 [4:09:01<01:42, 10.24s/it]
 99%|█████████▉| 1476/1485 [4
+0: {'loss': 0.5491, 'grad_norm': 0.830269493391465, 'learning_rate': 5.0998557964307e-07, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.6}
+0: [2025-11-23 19:00:45,538] [INFO] [axolotl.core.trainers.base._save:613] [PID:1308399] [RANK:0] Saving model checkpoint to /lustre/fswork/projects/rech/dgo/udv55np/ift/Nemotron-Super-49B-v1_5/gemma-3-4b/0.25/checkpoint-1485
+0: [2025-11-23 19:00:52,448] [INFO] [axolotl.core.trainers.base._save:662] [PID:1308399] [RANK:0] Saving Trainer.data_collator.tokenizer by default as Trainer.processing_class is `None`
+0: {'train_runtime': 15067.5819, 'train_samples_per_second': 1.577, 'train_steps_per_second': 0.099, 'train_loss': 0.5228189742926396, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.6}
+0: :09:11<01:32, 10.24s/it]
 99%|█████████▉| 1477/1485 [4:09:21<01:21, 10.23s/it]
100%|█████████▉| 1478/1485 [4:09:31<01:11, 10.26s/it]
100%|█████████▉| 1479/1485 [4:09:42<01:01, 10.24s/it]
100%|█████████▉| 1480/1485 [4:09:52<00:50, 10.19s/it]
                                                     

100%|█████████▉| 1480/1485 [4:09:52<00:50, 10.19s/it]
100%|█████████▉| 1481/1485 [4:10:02<00:40, 10.15s/it]
100%|█████████▉| 1482/1485 [4:10:12<00:30, 10.12s/it]
100%|█████████▉| 1483/1485 [4:10:22<00:20, 10.10s/it]
100%|█████████▉| 1484/1485 [4:10:34<00:10, 10.60s/it]
100%|██████████| 1485/1485 [4:10:46<00:00, 11.09s/it]
                                                     

100%|██████████| 1485/1485 [4:11:07<00:00, 11.09s/it]
100%|██████████| 1485/1485 [4:11:07<00:00, 10.15s/it]
+0: [2025-11-23 19:01:03,779] [INFO] [axolotl.train.save_trained_model:228] [PID:1308399] [RANK:0] Training completed! Saving trained model to /lustre/fswork/projects/rech/dgo/udv55np/ift/Nemotron-Super-49B-v1_5/gemma-3-4b/0.25.
+0: [2025-11-23 19:01:16,647] [INFO] [axolotl.core.trainers.base._save:613] [PID:1308399] [RANK:0] Saving model checkpoint to /lustre/fswork/projects/rech/dgo/udv55np/ift/Nemotron-Super-49B-v1_5/gemma-3-4b/0.25
+0: [2025-11-23 19:01:23,323] [INFO] [axolotl.core.trainers.base._save:662] [PID:1308399] [RANK:0] Saving Trainer.data_collator.tokenizer by default as Trainer.processing_class is `None`
+0: [2025-11-23 19:01:23,828] [INFO] [axolotl.train.save_trained_model:350] [PID:1308399] [RANK:0] Model successfully saved to /lustre/fswork/projects/rech/dgo/udv55np/ift/Nemotron-Super-49B-v1_5/gemma-3-4b/0.25
+0: jzxh159:1308402:1311139 [5445] NCCL INFO [Service thread] Connection closed by localRank 1
+0: jzxh159:1308402:1310277 [5445] NCCL INFO [Service thread] Connection closed by localRank 1
+0: jzxh159:1308402:1311139 [5445] NCCL INFO [Service thread] Connection closed by localRank 2
+0: jzxh159:1308402:1310277 [5445] NCCL INFO [Service thread] Connection closed by localRank 2