Upload folder using huggingface_hub

.ipynb_checkpoints/config-checkpoint.json

@@ -0,0 +1,29 @@
+{
+  "_name_or_path": "meta-llama/Llama-2-7b-hf",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "bos_token_id": 1,
+  "end_token_id": 2,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "max_position_embeddings": 4096,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 32,
+  "pad_token_id": 2,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float16",
+  "transformers_version": "4.34.1",
+  "use_cache": true,
+  "vocab_size": 32008
+}

.ipynb_checkpoints/training-checkpoint.log

@@ -0,0 +1,34 @@
+[2023-12-06 04:07:59,731] [INFO] [real_accelerator.py:161:get_accelerator] Setting ds_accelerator to cuda (auto detect)
+[2023-12-06 04:08:01,957] [WARNING] [runner.py:203:fetch_hostfile] Unable to find hostfile, will proceed with training with local resources only.
+[2023-12-06 04:08:01,957] [INFO] [runner.py:570:main] cmd = /home/t-sokumar/miniconda3/envs/ft/bin/python -u -m deepspeed.launcher.launch --world_info=eyJsb2NhbGhvc3QiOiBbMCwgMSwgMiwgM119 --master_addr=127.0.0.1 --master_port=29500 --enable_each_rank_log=None main.py --data_path local/jsonfile --data_split 2,4,4 --model_name_or_path meta-llama/Llama-2-7b-hf --per_device_train_batch_size 4 --per_device_eval_batch_size 4 --max_seq_len 512 --learning_rate 9.65e-6 --weight_decay 0. --num_train_epochs 4 --gradient_accumulation_steps 1 --lr_scheduler_type cosine --num_warmup_steps 0 --seed 1234 --gradient_checkpointing --zero_stage 3 --deepspeed --lora_dim 128 --lora_module_name layers. --output_dir ./output_step1_llama2_7b_lora
+[2023-12-06 04:08:04,758] [INFO] [real_accelerator.py:161:get_accelerator] Setting ds_accelerator to cuda (auto detect)
+[2023-12-06 04:08:06,901] [INFO] [launch.py:145:main] WORLD INFO DICT: {'localhost': [0, 1, 2, 3]}
+[2023-12-06 04:08:06,901] [INFO] [launch.py:151:main] nnodes=1, num_local_procs=4, node_rank=0
+[2023-12-06 04:08:06,901] [INFO] [launch.py:162:main] global_rank_mapping=defaultdict(<class 'list'>, {'localhost': [0, 1, 2, 3]})
+[2023-12-06 04:08:06,901] [INFO] [launch.py:163:main] dist_world_size=4
+[2023-12-06 04:08:06,901] [INFO] [launch.py:165:main] Setting CUDA_VISIBLE_DEVICES=0,1,2,3
+[2023-12-06 04:08:10,349] [INFO] [real_accelerator.py:161:get_accelerator] Setting ds_accelerator to cuda (auto detect)
+[2023-12-06 04:08:10,349] [INFO] [real_accelerator.py:161:get_accelerator] Setting ds_accelerator to cuda (auto detect)
+[2023-12-06 04:08:10,360] [INFO] [real_accelerator.py:161:get_accelerator] Setting ds_accelerator to cuda (auto detect)
+[2023-12-06 04:08:10,559] [INFO] [real_accelerator.py:161:get_accelerator] Setting ds_accelerator to cuda (auto detect)
+Traceback (most recent call last):
+  File "/home/t-sokumar/llama2_finetune/DeepSpeedExamples/applications/DeepSpeed-Chat/training/step1_supervised_finetuning/main.py", line 24, in <module>
+    from dschat.utils.data.data_utils import create_prompt_dataset
+ModuleNotFoundError: No module named 'dschat'
+Traceback (most recent call last):
+  File "/home/t-sokumar/llama2_finetune/DeepSpeedExamples/applications/DeepSpeed-Chat/training/step1_supervised_finetuning/main.py", line 24, in <module>
+    from dschat.utils.data.data_utils import create_prompt_dataset
+ModuleNotFoundError: No module named 'dschat'
+Traceback (most recent call last):
+  File "/home/t-sokumar/llama2_finetune/DeepSpeedExamples/applications/DeepSpeed-Chat/training/step1_supervised_finetuning/main.py", line 24, in <module>
+    from dschat.utils.data.data_utils import create_prompt_dataset
+ModuleNotFoundError: No module named 'dschat'
+Traceback (most recent call last):
+  File "/home/t-sokumar/llama2_finetune/DeepSpeedExamples/applications/DeepSpeed-Chat/training/step1_supervised_finetuning/main.py", line 24, in <module>
+    from dschat.utils.data.data_utils import create_prompt_dataset
+ModuleNotFoundError: No module named 'dschat'
+[2023-12-06 04:08:12,905] [INFO] [launch.py:315:sigkill_handler] Killing subprocess 2459201
+[2023-12-06 04:08:12,905] [INFO] [launch.py:315:sigkill_handler] Killing subprocess 2459202
+[2023-12-06 04:08:12,931] [INFO] [launch.py:315:sigkill_handler] Killing subprocess 2459203
+[2023-12-06 04:08:12,949] [INFO] [launch.py:315:sigkill_handler] Killing subprocess 2459204
+[2023-12-06 04:08:12,967] [ERROR] [launch.py:321:sigkill_handler] ['/home/t-sokumar/miniconda3/envs/ft/bin/python', '-u', 'main.py', '--local_rank=3', '--data_path', 'local/jsonfile', '--data_split', '2,4,4', '--model_name_or_path', 'meta-llama/Llama-2-7b-hf', '--per_device_train_batch_size', '4', '--per_device_eval_batch_size', '4', '--max_seq_len', '512', '--learning_rate', '9.65e-6', '--weight_decay', '0.', '--num_train_epochs', '4', '--gradient_accumulation_steps', '1', '--lr_scheduler_type', 'cosine', '--num_warmup_steps', '0', '--seed', '1234', '--gradient_checkpointing', '--zero_stage', '3', '--deepspeed', '--lora_dim', '128', '--lora_module_name', 'layers.', '--output_dir', './output_step1_llama2_7b_lora'] exits with return code = 1

config.json

@@ -0,0 +1,29 @@
+{
+  "_name_or_path": "meta-llama/Llama-2-7b-hf",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "bos_token_id": 1,
+  "end_token_id": 2,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "max_position_embeddings": 4096,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 32,
+  "pad_token_id": 2,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float16",
+  "transformers_version": "4.34.1",
+  "use_cache": true,
+  "vocab_size": 32008
+}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b071c072847031331fe840a75f3ad7db7b19c12dea3fb1085a410628f4fd2c32
+size 13477059118

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
+size 499723

training.log

@@ -0,0 +1,389 @@
+[2023-12-06 04:40:04,481] [INFO] [real_accelerator.py:161:get_accelerator] Setting ds_accelerator to cuda (auto detect)
+[2023-12-06 04:40:06,410] [WARNING] [runner.py:203:fetch_hostfile] Unable to find hostfile, will proceed with training with local resources only.
+[2023-12-06 04:40:06,410] [INFO] [runner.py:570:main] cmd = /home/t-sokumar/miniconda3/envs/ft/bin/python -u -m deepspeed.launcher.launch --world_info=eyJsb2NhbGhvc3QiOiBbMCwgMSwgMiwgM119 --master_addr=127.0.0.1 --master_port=29500 --enable_each_rank_log=None main.py --data_path local/jsonfile --data_split 2,4,4 --model_name_or_path meta-llama/Llama-2-7b-hf --per_device_train_batch_size 4 --per_device_eval_batch_size 4 --max_seq_len 512 --learning_rate 9.65e-6 --weight_decay 0. --num_train_epochs 10 --gradient_accumulation_steps 1 --lr_scheduler_type cosine --num_warmup_steps 0 --seed 1234 --gradient_checkpointing --zero_stage 3 --deepspeed --lora_dim 128 --lora_module_name layers. --output_dir ./output_step1_llama2_7b_lora
+[2023-12-06 04:40:09,034] [INFO] [real_accelerator.py:161:get_accelerator] Setting ds_accelerator to cuda (auto detect)
+[2023-12-06 04:40:11,238] [INFO] [launch.py:145:main] WORLD INFO DICT: {'localhost': [0, 1, 2, 3]}
+[2023-12-06 04:40:11,239] [INFO] [launch.py:151:main] nnodes=1, num_local_procs=4, node_rank=0
+[2023-12-06 04:40:11,239] [INFO] [launch.py:162:main] global_rank_mapping=defaultdict(<class 'list'>, {'localhost': [0, 1, 2, 3]})
+[2023-12-06 04:40:11,239] [INFO] [launch.py:163:main] dist_world_size=4
+[2023-12-06 04:40:11,239] [INFO] [launch.py:165:main] Setting CUDA_VISIBLE_DEVICES=0,1,2,3
+[2023-12-06 04:40:14,677] [INFO] [real_accelerator.py:161:get_accelerator] Setting ds_accelerator to cuda (auto detect)
+[2023-12-06 04:40:14,691] [INFO] [real_accelerator.py:161:get_accelerator] Setting ds_accelerator to cuda (auto detect)
+[2023-12-06 04:40:14,691] [INFO] [real_accelerator.py:161:get_accelerator] Setting ds_accelerator to cuda (auto detect)
+[2023-12-06 04:40:14,713] [INFO] [real_accelerator.py:161:get_accelerator] Setting ds_accelerator to cuda (auto detect)
+/home/t-sokumar/miniconda3/envs/ft/lib/python3.11/site-packages/transformers/deepspeed.py:23: FutureWarning: transformers.deepspeed module is deprecated and will be removed in a future version. Please import deepspeed modules directly from transformers.integrations
+  warnings.warn(
+/home/t-sokumar/miniconda3/envs/ft/lib/python3.11/site-packages/transformers/deepspeed.py:23: FutureWarning: transformers.deepspeed module is deprecated and will be removed in a future version. Please import deepspeed modules directly from transformers.integrations
+  warnings.warn(
+/home/t-sokumar/miniconda3/envs/ft/lib/python3.11/site-packages/transformers/deepspeed.py:23: FutureWarning: transformers.deepspeed module is deprecated and will be removed in a future version. Please import deepspeed modules directly from transformers.integrations
+  warnings.warn(
+/home/t-sokumar/miniconda3/envs/ft/lib/python3.11/site-packages/transformers/deepspeed.py:23: FutureWarning: transformers.deepspeed module is deprecated and will be removed in a future version. Please import deepspeed modules directly from transformers.integrations
+  warnings.warn(
+[2023-12-06 04:40:16,288] [INFO] [comm.py:637:init_distributed] cdb=None
+[2023-12-06 04:40:16,288] [INFO] [comm.py:668:init_distributed] Initializing TorchBackend in DeepSpeed with backend nccl
+[2023-12-06 04:40:16,558] [INFO] [comm.py:637:init_distributed] cdb=None
+[2023-12-06 04:40:16,642] [INFO] [comm.py:637:init_distributed] cdb=None
+[2023-12-06 04:40:16,658] [INFO] [comm.py:637:init_distributed] cdb=None
+[2023-12-06 04:40:20,136] [INFO] [partition_parameters.py:348:__exit__] finished initializing model - num_params = 291, num_elems = 6.74B
+
+
+
+
+Using /home/t-sokumar/.cache/torch_extensions/py311_cu121 as PyTorch extensions root...
+Using /home/t-sokumar/.cache/torch_extensions/py311_cu121 as PyTorch extensions root...
+Using /home/t-sokumar/.cache/torch_extensions/py311_cu121 as PyTorch extensions root...
+Using /home/t-sokumar/.cache/torch_extensions/py311_cu121 as PyTorch extensions root...
+Detected CUDA files, patching ldflags
+Emitting ninja build file /home/t-sokumar/.cache/torch_extensions/py311_cu121/fused_adam/build.ninja...
+Building extension module fused_adam...
+Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N)
+ninja: no work to do.
+Loading extension module fused_adam...
+Time to load fused_adam op: 0.1045382022857666 seconds
+/home/t-sokumar/miniconda3/envs/ft/lib/python3.11/site-packages/deepspeed/ops/adam/fused_adam.py:96: UserWarning: The torch.cuda.*DtypeTensor constructors are no longer recommended. It's best to use methods such as torch.tensor(data, dtype=*, device='cuda') to create tensors. (Triggered internally at ../torch/csrc/tensor/python_tensor.cpp:83.)
+  self._dummy_overflow_buf = get_accelerator().IntTensor([0])
+Loading extension module fused_adam...
+Time to load fused_adam op: 0.10168743133544922 seconds
+/home/t-sokumar/miniconda3/envs/ft/lib/python3.11/site-packages/deepspeed/ops/adam/fused_adam.py:96: UserWarning: The torch.cuda.*DtypeTensor constructors are no longer recommended. It's best to use methods such as torch.tensor(data, dtype=*, device='cuda') to create tensors. (Triggered internally at ../torch/csrc/tensor/python_tensor.cpp:83.)
+  self._dummy_overflow_buf = get_accelerator().IntTensor([0])
+Loading extension module fused_adam...
+Time to load fused_adam op: 0.10143876075744629 seconds
+/home/t-sokumar/miniconda3/envs/ft/lib/python3.11/site-packages/deepspeed/ops/adam/fused_adam.py:96: UserWarning: The torch.cuda.*DtypeTensor constructors are no longer recommended. It's best to use methods such as torch.tensor(data, dtype=*, device='cuda') to create tensors. (Triggered internally at ../torch/csrc/tensor/python_tensor.cpp:83.)
+  self._dummy_overflow_buf = get_accelerator().IntTensor([0])
+[2023-12-06 04:40:29,694] [INFO] [logging.py:96:log_dist] [Rank 0] DeepSpeed info: version=0.12.4, git-hash=unknown, git-branch=unknown
+Loading extension module fused_adam...
+[2023-12-06 04:40:29,696] [INFO] [comm.py:662:init_distributed] Distributed backend already initialized
+Time to load fused_adam op: 0.10245537757873535 seconds
+/home/t-sokumar/miniconda3/envs/ft/lib/python3.11/site-packages/deepspeed/ops/adam/fused_adam.py:96: UserWarning: The torch.cuda.*DtypeTensor constructors are no longer recommended. It's best to use methods such as torch.tensor(data, dtype=*, device='cuda') to create tensors. (Triggered internally at ../torch/csrc/tensor/python_tensor.cpp:83.)
+  self._dummy_overflow_buf = get_accelerator().IntTensor([0])
+[2023-12-06 04:40:29,737] [INFO] [logging.py:96:log_dist] [Rank 0] DeepSpeed Flops Profiler Enabled: False
+[2023-12-06 04:40:29,739] [INFO] [logging.py:96:log_dist] [Rank 0] Using client Optimizer as basic optimizer
+[2023-12-06 04:40:29,739] [INFO] [logging.py:96:log_dist] [Rank 0] Removing param_group that has no 'params' in the basic Optimizer
+[2023-12-06 04:40:29,778] [INFO] [logging.py:96:log_dist] [Rank 0] DeepSpeed Basic Optimizer = FusedAdam
+[2023-12-06 04:40:29,778] [INFO] [utils.py:56:is_zero_supported_optimizer] Checking ZeRO support for optimizer=FusedAdam type=<class 'deepspeed.ops.adam.fused_adam.FusedAdam'>
+[2023-12-06 04:40:29,778] [INFO] [logging.py:96:log_dist] [Rank 0] Creating fp16 ZeRO stage 3 optimizer, MiCS is enabled False, Hierarchical params gather False
+[2023-12-06 04:40:29,778] [INFO] [logging.py:96:log_dist] [Rank 0] Creating torch.float16 ZeRO stage 3 optimizer
+[2023-12-06 04:40:29,894] [INFO] [utils.py:795:see_memory_usage] Stage 3 initialize beginning
+[2023-12-06 04:40:29,895] [INFO] [utils.py:796:see_memory_usage] MA 4.17 GB         Max_MA 4.39 GB         CA 7.75 GB         Max_CA 8 GB 
+[2023-12-06 04:40:29,895] [INFO] [utils.py:803:see_memory_usage] CPU Virtual Memory:  used = 13.59 GB, percent = 5.4%
+[2023-12-06 04:40:29,897] [INFO] [stage3.py:127:__init__] Reduce bucket size 500,000,000
+[2023-12-06 04:40:29,897] [INFO] [stage3.py:128:__init__] Prefetch bucket size 30000000
+[2023-12-06 04:40:29,998] [INFO] [utils.py:795:see_memory_usage] DeepSpeedZeRoOffload initialize [begin]
+[2023-12-06 04:40:29,999] [INFO] [utils.py:796:see_memory_usage] MA 4.17 GB         Max_MA 4.17 GB         CA 7.75 GB         Max_CA 8 GB 
+[2023-12-06 04:40:29,999] [INFO] [utils.py:803:see_memory_usage] CPU Virtual Memory:  used = 13.59 GB, percent = 5.4%
+Parameter Offload: Total persistent parameters: 266240 in 65 params
+[2023-12-06 04:40:30,335] [INFO] [utils.py:795:see_memory_usage] DeepSpeedZeRoOffload initialize [end]
+[2023-12-06 04:40:30,335] [INFO] [utils.py:796:see_memory_usage] MA 3.35 GB         Max_MA 4.23 GB         CA 7.75 GB         Max_CA 8 GB 
+[2023-12-06 04:40:30,336] [INFO] [utils.py:803:see_memory_usage] CPU Virtual Memory:  used = 13.6 GB, percent = 5.4%
+[2023-12-06 04:40:30,446] [INFO] [utils.py:795:see_memory_usage] Before creating fp16 partitions
+[2023-12-06 04:40:30,447] [INFO] [utils.py:796:see_memory_usage] MA 3.35 GB         Max_MA 3.35 GB         CA 7.75 GB         Max_CA 8 GB 
+[2023-12-06 04:40:30,447] [INFO] [utils.py:803:see_memory_usage] CPU Virtual Memory:  used = 13.57 GB, percent = 5.4%
+[2023-12-06 04:40:31,148] [INFO] [utils.py:795:see_memory_usage] After creating fp16 partitions: 3
+[2023-12-06 04:40:31,149] [INFO] [utils.py:796:see_memory_usage] MA 3.35 GB         Max_MA 3.35 GB         CA 4.38 GB         Max_CA 8 GB 
+[2023-12-06 04:40:31,149] [INFO] [utils.py:803:see_memory_usage] CPU Virtual Memory:  used = 13.64 GB, percent = 5.4%
+[2023-12-06 04:40:31,292] [INFO] [utils.py:795:see_memory_usage] Before creating fp32 partitions
+[2023-12-06 04:40:31,293] [INFO] [utils.py:796:see_memory_usage] MA 3.35 GB         Max_MA 3.35 GB         CA 4.38 GB         Max_CA 4 GB 
+[2023-12-06 04:40:31,293] [INFO] [utils.py:803:see_memory_usage] CPU Virtual Memory:  used = 11.18 GB, percent = 4.4%
+[2023-12-06 04:40:31,429] [INFO] [utils.py:795:see_memory_usage] After creating fp32 partitions
+[2023-12-06 04:40:31,430] [INFO] [utils.py:796:see_memory_usage] MA 3.9 GB         Max_MA 4.05 GB         CA 5.07 GB         Max_CA 5 GB 
+[2023-12-06 04:40:31,430] [INFO] [utils.py:803:see_memory_usage] CPU Virtual Memory:  used = 11.19 GB, percent = 4.4%
+[2023-12-06 04:40:31,541] [INFO] [utils.py:795:see_memory_usage] Before initializing optimizer states
+[2023-12-06 04:40:31,541] [INFO] [utils.py:796:see_memory_usage] MA 3.9 GB         Max_MA 3.9 GB         CA 5.07 GB         Max_CA 5 GB 
+[2023-12-06 04:40:31,542] [INFO] [utils.py:803:see_memory_usage] CPU Virtual Memory:  used = 11.19 GB, percent = 4.4%
+[2023-12-06 04:40:31,686] [INFO] [utils.py:795:see_memory_usage] After initializing optimizer states
+[2023-12-06 04:40:31,687] [INFO] [utils.py:796:see_memory_usage] MA 4.98 GB         Max_MA 5.28 GB         CA 6.46 GB         Max_CA 6 GB 
+[2023-12-06 04:40:31,687] [INFO] [utils.py:803:see_memory_usage] CPU Virtual Memory:  used = 11.15 GB, percent = 4.4%
+[2023-12-06 04:40:31,688] [INFO] [stage3.py:479:_setup_for_real_optimizer] optimizer state initialized
+[2023-12-06 04:40:32,196] [INFO] [utils.py:795:see_memory_usage] After initializing ZeRO optimizer
+[2023-12-06 04:40:32,196] [INFO] [utils.py:796:see_memory_usage] MA 6.19 GB         Max_MA 6.67 GB         CA 8.14 GB         Max_CA 8 GB 
+[2023-12-06 04:40:32,196] [INFO] [utils.py:803:see_memory_usage] CPU Virtual Memory:  used = 11.18 GB, percent = 4.4%
+[2023-12-06 04:40:32,197] [INFO] [logging.py:96:log_dist] [Rank 0] DeepSpeed Final Optimizer = FusedAdam
+[2023-12-06 04:40:32,197] [INFO] [logging.py:96:log_dist] [Rank 0] DeepSpeed using client LR scheduler
+[2023-12-06 04:40:32,197] [INFO] [logging.py:96:log_dist] [Rank 0] DeepSpeed LR Scheduler = <torch.optim.lr_scheduler.LambdaLR object at 0x7fed2182b890>
+[2023-12-06 04:40:32,197] [INFO] [logging.py:96:log_dist] [Rank 0] step=0, skipped=0, lr=[9.65e-06, 0.0005, 9.65e-06], mom=[(0.9, 0.95), (0.9, 0.95), (0.9, 0.95)]
+[2023-12-06 04:40:32,198] [INFO] [config.py:979:print] DeepSpeedEngine configuration:
+[2023-12-06 04:40:32,198] [INFO] [config.py:983:print]   activation_checkpointing_config  {
+    "partition_activations": false, 
+    "contiguous_memory_optimization": false, 
+    "cpu_checkpointing": false, 
+    "number_checkpoints": null, 
+    "synchronize_checkpoint_boundary": false, 
+    "profile": false
+}
+[2023-12-06 04:40:32,198] [INFO] [config.py:983:print]   aio_config ................... {'block_size': 1048576, 'queue_depth': 8, 'thread_count': 1, 'single_submit': False, 'overlap_events': True}
+[2023-12-06 04:40:32,199] [INFO] [config.py:983:print]   amp_enabled .................. False
+[2023-12-06 04:40:32,199] [INFO] [config.py:983:print]   amp_params ................... False
+[2023-12-06 04:40:32,199] [INFO] [config.py:983:print]   autotuning_config ............ {
+    "enabled": false, 
+    "start_step": null, 
+    "end_step": null, 
+    "metric_path": null, 
+    "arg_mappings": null, 
+    "metric": "throughput", 
+    "model_info": null, 
+    "results_dir": "autotuning_results", 
+    "exps_dir": "autotuning_exps", 
+    "overwrite": true, 
+    "fast": true, 
+    "start_profile_step": 3, 
+    "end_profile_step": 5, 
+    "tuner_type": "gridsearch", 
+    "tuner_early_stopping": 5, 
+    "tuner_num_trials": 50, 
+    "model_info_path": null, 
+    "mp_size": 1, 
+    "max_train_batch_size": null, 
+    "min_train_batch_size": 1, 
+    "max_train_micro_batch_size_per_gpu": 1.024000e+03, 
+    "min_train_micro_batch_size_per_gpu": 1, 
+    "num_tuning_micro_batch_sizes": 3
+}
+[2023-12-06 04:40:32,199] [INFO] [config.py:983:print]   bfloat16_enabled ............. False
+[2023-12-06 04:40:32,199] [INFO] [config.py:983:print]   checkpoint_parallel_write_pipeline  False
+[2023-12-06 04:40:32,199] [INFO] [config.py:983:print]   checkpoint_tag_validation_enabled  True
+[2023-12-06 04:40:32,199] [INFO] [config.py:983:print]   checkpoint_tag_validation_fail  False
+[2023-12-06 04:40:32,199] [INFO] [config.py:983:print]   comms_config ................. <deepspeed.comm.config.DeepSpeedCommsConfig object at 0x7fed21536210>
+[2023-12-06 04:40:32,199] [INFO] [config.py:983:print]   communication_data_type ...... None
+[2023-12-06 04:40:32,199] [INFO] [config.py:983:print]   compression_config ........... {'weight_quantization': {'shared_parameters': {'enabled': False, 'quantizer_kernel': False, 'schedule_offset': 0, 'quantize_groups': 1, 'quantize_verbose': False, 'quantization_type': 'symmetric', 'quantize_weight_in_forward': False, 'rounding': 'nearest', 'fp16_mixed_quantize': False, 'quantize_change_ratio': 0.001}, 'different_groups': {}}, 'activation_quantization': {'shared_parameters': {'enabled': False, 'quantization_type': 'symmetric', 'range_calibration': 'dynamic', 'schedule_offset': 1000}, 'different_groups': {}}, 'sparse_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'row_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'head_pruning': {'shared_parameters': {'enabled': False, 'method': 'topk', 'schedule_offset': 1000}, 'different_groups': {}}, 'channel_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'layer_reduction': {'enabled': False}}
+[2023-12-06 04:40:32,199] [INFO] [config.py:983:print]   curriculum_enabled_legacy .... False
+[2023-12-06 04:40:32,199] [INFO] [config.py:983:print]   curriculum_params_legacy ..... False
+[2023-12-06 04:40:32,199] [INFO] [config.py:983:print]   data_efficiency_config ....... {'enabled': False, 'seed': 1234, 'data_sampling': {'enabled': False, 'num_epochs': 1000, 'num_workers': 0, 'curriculum_learning': {'enabled': False}}, 'data_routing': {'enabled': False, 'random_ltd': {'enabled': False, 'layer_token_lr_schedule': {'enabled': False}}}}
+[2023-12-06 04:40:32,199] [INFO] [config.py:983:print]   data_efficiency_enabled ...... False
+[2023-12-06 04:40:32,199] [INFO] [config.py:983:print]   dataloader_drop_last ......... False
+[2023-12-06 04:40:32,199] [INFO] [config.py:983:print]   disable_allgather ............ False
+[2023-12-06 04:40:32,199] [INFO] [config.py:983:print]   dump_state ................... False
+[2023-12-06 04:40:32,199] [INFO] [config.py:983:print]   dynamic_loss_scale_args ...... {'init_scale': 65536, 'scale_window': 100, 'delayed_shift': 2, 'consecutive_hysteresis': False, 'min_scale': 1}
+[2023-12-06 04:40:32,199] [INFO] [config.py:983:print]   eigenvalue_enabled ........... False
+[2023-12-06 04:40:32,199] [INFO] [config.py:983:print]   eigenvalue_gas_boundary_resolution  1
+[2023-12-06 04:40:32,199] [INFO] [config.py:983:print]   eigenvalue_layer_name ........ bert.encoder.layer
+[2023-12-06 04:40:32,199] [INFO] [config.py:983:print]   eigenvalue_layer_num ......... 0
+[2023-12-06 04:40:32,199] [INFO] [config.py:983:print]   eigenvalue_max_iter .......... 100
+[2023-12-06 04:40:32,200] [INFO] [config.py:983:print]   eigenvalue_stability ......... 1e-06
+[2023-12-06 04:40:32,200] [INFO] [config.py:983:print]   eigenvalue_tol ............... 0.01
+[2023-12-06 04:40:32,200] [INFO] [config.py:983:print]   eigenvalue_verbose ........... False
+[2023-12-06 04:40:32,200] [INFO] [config.py:983:print]   elasticity_enabled ........... False
+[2023-12-06 04:40:32,200] [INFO] [config.py:983:print]   flops_profiler_config ........ {
+    "enabled": false, 
+    "recompute_fwd_factor": 0.0, 
+    "profile_step": 1, 
+    "module_depth": -1, 
+    "top_modules": 1, 
+    "detailed": true, 
+    "output_file": null
+}
+[2023-12-06 04:40:32,200] [INFO] [config.py:983:print]   fp16_auto_cast ............... False
+[2023-12-06 04:40:32,200] [INFO] [config.py:983:print]   fp16_enabled ................. True
+[2023-12-06 04:40:32,200] [INFO] [config.py:983:print]   fp16_master_weights_and_gradients  False
+[2023-12-06 04:40:32,200] [INFO] [config.py:983:print]   global_rank .................. 0
+[2023-12-06 04:40:32,200] [INFO] [config.py:983:print]   grad_accum_dtype ............. None
+[2023-12-06 04:40:32,200] [INFO] [config.py:983:print]   gradient_accumulation_steps .. 1
+[2023-12-06 04:40:32,200] [INFO] [config.py:983:print]   gradient_clipping ............ 1.0
+[2023-12-06 04:40:32,200] [INFO] [config.py:983:print]   gradient_predivide_factor .... 1.0
+[2023-12-06 04:40:32,200] [INFO] [config.py:983:print]   hybrid_engine ................ enabled=False max_out_tokens=512 inference_tp_size=1 release_inference_cache=False pin_parameters=True tp_gather_partition_size=8
+[2023-12-06 04:40:32,200] [INFO] [config.py:983:print]   initial_dynamic_scale ........ 65536
+[2023-12-06 04:40:32,200] [INFO] [config.py:983:print]   load_universal_checkpoint .... False
+[2023-12-06 04:40:32,200] [INFO] [config.py:983:print]   loss_scale ................... 0
+[2023-12-06 04:40:32,200] [INFO] [config.py:983:print]   memory_breakdown ............. False
+[2023-12-06 04:40:32,200] [INFO] [config.py:983:print]   mics_hierarchial_params_gather  False
+[2023-12-06 04:40:32,200] [INFO] [config.py:983:print]   mics_shard_size .............. -1
+[2023-12-06 04:40:32,200] [INFO] [config.py:983:print]   monitor_config ............... tensorboard=TensorBoardConfig(enabled=False, output_path='step1_tensorboard/ds_tensorboard_logs/', job_name='step1_model_tensorboard') wandb=WandbConfig(enabled=False, group=None, team=None, project='deepspeed') csv_monitor=CSVConfig(enabled=False, output_path='', job_name='DeepSpeedJobName') enabled=False
+[2023-12-06 04:40:32,200] [INFO] [config.py:983:print]   nebula_config ................ {
+    "enabled": false, 
+    "persistent_storage_path": null, 
+    "persistent_time_interval": 100, 
+    "num_of_version_in_retention": 2, 
+    "enable_nebula_load": true, 
+    "load_path": null
+}
+[2023-12-06 04:40:32,200] [INFO] [config.py:983:print]   optimizer_legacy_fusion ...... False
+[2023-12-06 04:40:32,200] [INFO] [config.py:983:print]   optimizer_name ............... None
+[2023-12-06 04:40:32,200] [INFO] [config.py:983:print]   optimizer_params ............. None
+[2023-12-06 04:40:32,200] [INFO] [config.py:983:print]   pipeline ..................... {'stages': 'auto', 'partition': 'best', 'seed_layers': False, 'activation_checkpoint_interval': 0, 'pipe_partitioned': True, 'grad_partitioned': True}
+[2023-12-06 04:40:32,200] [INFO] [config.py:983:print]   pld_enabled .................. False
+[2023-12-06 04:40:32,200] [INFO] [config.py:983:print]   pld_params ................... False
+[2023-12-06 04:40:32,200] [INFO] [config.py:983:print]   prescale_gradients ........... False
+[2023-12-06 04:40:32,201] [INFO] [config.py:983:print]   scheduler_name ............... None
+[2023-12-06 04:40:32,201] [INFO] [config.py:983:print]   scheduler_params ............. None
+[2023-12-06 04:40:32,201] [INFO] [config.py:983:print]   seq_parallel_communication_data_type  torch.float32
+[2023-12-06 04:40:32,201] [INFO] [config.py:983:print]   sparse_attention ............. None
+[2023-12-06 04:40:32,201] [INFO] [config.py:983:print]   sparse_gradients_enabled ..... False
+[2023-12-06 04:40:32,201] [INFO] [config.py:983:print]   steps_per_print .............. 10
+[2023-12-06 04:40:32,201] [INFO] [config.py:983:print]   train_batch_size ............. 16
+[2023-12-06 04:40:32,201] [INFO] [config.py:983:print]   train_micro_batch_size_per_gpu  4
+[2023-12-06 04:40:32,201] [INFO] [config.py:983:print]   use_data_before_expert_parallel_  False
+[2023-12-06 04:40:32,201] [INFO] [config.py:983:print]   use_node_local_storage ....... False
+[2023-12-06 04:40:32,201] [INFO] [config.py:983:print]   wall_clock_breakdown ......... False
+[2023-12-06 04:40:32,201] [INFO] [config.py:983:print]   weight_quantization_config ... None
+[2023-12-06 04:40:32,201] [INFO] [config.py:983:print]   world_size ................... 4
+[2023-12-06 04:40:32,201] [INFO] [config.py:983:print]   zero_allow_untested_optimizer  False
+[2023-12-06 04:40:32,201] [INFO] [config.py:983:print]   zero_config .................. stage=3 contiguous_gradients=True reduce_scatter=True reduce_bucket_size=500,000,000 use_multi_rank_bucket_allreduce=True allgather_partitions=True allgather_bucket_size=500,000,000 overlap_comm=True load_from_fp32_weights=True elastic_checkpoint=False offload_param=DeepSpeedZeroOffloadParamConfig(device='none', nvme_path=None, buffer_count=5, buffer_size=100,000,000, max_in_cpu=1,000,000,000, pin_memory=False) offload_optimizer=DeepSpeedZeroOffloadOptimizerConfig(device='none', nvme_path=None, buffer_count=4, pin_memory=False, pipeline=False, pipeline_read=False, pipeline_write=False, fast_init=False, ratio=1.0) sub_group_size=1,000,000,000 cpu_offload_param=None cpu_offload_use_pin_memory=None cpu_offload=None prefetch_bucket_size=30000000 param_persistence_threshold=10000 model_persistence_threshold=sys.maxsize max_live_parameters=30000000 max_reuse_distance=1,000,000,000 gather_16bit_weights_on_model_save=False stage3_gather_fp16_weights_on_model_save=False ignore_unused_parameters=True legacy_stage1=False round_robin_gradients=False zero_hpz_partition_size=1 zero_quantized_weights=False zero_quantized_nontrainable_weights=False zero_quantized_gradients=False mics_shard_size=-1 mics_hierarchical_params_gather=False memory_efficient_linear=False pipeline_loading_checkpoint=False override_module_apply=True
+[2023-12-06 04:40:32,201] [INFO] [config.py:983:print]   zero_enabled ................. True
+[2023-12-06 04:40:32,201] [INFO] [config.py:983:print]   zero_force_ds_cpu_optimizer .. True
+[2023-12-06 04:40:32,201] [INFO] [config.py:983:print]   zero_optimization_stage ...... 3
+[2023-12-06 04:40:32,201] [INFO] [config.py:969:print_user_config]   json = {
+    "train_batch_size": 16, 
+    "train_micro_batch_size_per_gpu": 4, 
+    "steps_per_print": 10, 
+    "zero_optimization": {
+        "stage": 3, 
+        "offload_param": {
+            "device": "none"
+        }, 
+        "offload_optimizer": {
+            "device": "none"
+        }, 
+        "stage3_param_persistence_threshold": 1.000000e+04, 
+        "stage3_max_live_parameters": 3.000000e+07, 
+        "stage3_prefetch_bucket_size": 3.000000e+07, 
+        "memory_efficient_linear": false
+    }, 
+    "fp16": {
+        "enabled": true, 
+        "loss_scale_window": 100
+    }, 
+    "gradient_clipping": 1.0, 
+    "prescale_gradients": false, 
+    "wall_clock_breakdown": false, 
+    "hybrid_engine": {
+        "enabled": false, 
+        "max_out_tokens": 512, 
+        "inference_tp_size": 1, 
+        "release_inference_cache": false, 
+        "pin_parameters": true, 
+        "tp_gather_partition_size": 8
+    }, 
+    "tensorboard": {
+        "enabled": false, 
+        "output_path": "step1_tensorboard/ds_tensorboard_logs/", 
+        "job_name": "step1_model_tensorboard"
+    }
+}
+***** Running training *****
+***** Evaluating perplexity, Epoch 0/10 *****
+ppl: 6.489692211151123, loss: 1.8702150583267212
+Beginning of Epoch 1/10, Total Micro Batches 6
+/home/t-sokumar/miniconda3/envs/ft/lib/python3.11/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.
+  warnings.warn(
+/home/t-sokumar/miniconda3/envs/ft/lib/python3.11/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.
+  warnings.warn(
+/home/t-sokumar/miniconda3/envs/ft/lib/python3.11/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.
+  warnings.warn(
+/home/t-sokumar/miniconda3/envs/ft/lib/python3.11/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.
+  warnings.warn(
+Model Parameters: 6.927 B, Latency: 2.79s, TFLOPs: 7.49, Samples/sec: 1.43, Time/seq 0.70s, Batch Size: 4, Sequence Length: 512
+Invalidate trace cache @ step 0: expected module 6, but got module 0
+Model Parameters: 6.927 B, Latency: 2.32s, TFLOPs: 9.00, Samples/sec: 1.72, Time/seq 0.58s, Batch Size: 4, Sequence Length: 512
+Model Parameters: 6.927 B, Latency: 2.33s, TFLOPs: 8.98, Samples/sec: 1.72, Time/seq 0.58s, Batch Size: 4, Sequence Length: 512
+Model Parameters: 6.927 B, Latency: 2.63s, TFLOPs: 7.95, Samples/sec: 1.52, Time/seq 0.66s, Batch Size: 4, Sequence Length: 512
+Model Parameters: 6.927 B, Latency: 2.38s, TFLOPs: 8.80, Samples/sec: 1.68, Time/seq 0.59s, Batch Size: 4, Sequence Length: 512
+Model Parameters: 6.927 B, Latency: 1.58s, TFLOPs: 13.24, Samples/sec: 2.53, Time/seq 0.40s, Batch Size: 4, Sequence Length: 512
+***** Evaluating perplexity, Epoch 1/10 *****
+Invalidate trace cache @ step 0: expected module 0, but got module 6
+ppl: 3.473783493041992, loss: 1.2452443838119507
+Beginning of Epoch 2/10, Total Micro Batches 6
+Model Parameters: 6.927 B, Latency: 2.31s, TFLOPs: 9.07, Samples/sec: 1.73, Time/seq 0.58s, Batch Size: 4, Sequence Length: 512
+Model Parameters: 6.927 B, Latency: 2.32s, TFLOPs: 9.01, Samples/sec: 1.72, Time/seq 0.58s, Batch Size: 4, Sequence Length: 512
+Model Parameters: 6.927 B, Latency: 2.07s, TFLOPs: 10.10, Samples/sec: 1.93, Time/seq 0.52s, Batch Size: 4, Sequence Length: 512
+[2023-12-06 04:40:57,343] [INFO] [logging.py:96:log_dist] [Rank 0] step=10, skipped=0, lr=[9.003572573259918e-06, 0.00046650635094610973, 9.003572573259918e-06], mom=[(0.9, 0.95), (0.9, 0.95), (0.9, 0.95)]
+[2023-12-06 04:40:57,343] [INFO] [timer.py:260:stop] epoch=1/micro_step=4/global_step=10, RunningAvgSamplesPerSec=7.236305009057328, CurrSamplesPerSec=7.733712400366698, MemAllocated=6.45GB, MaxMemAllocated=8.41GB
+Model Parameters: 6.927 B, Latency: 2.07s, TFLOPs: 10.11, Samples/sec: 1.93, Time/seq 0.52s, Batch Size: 4, Sequence Length: 512
+Model Parameters: 6.927 B, Latency: 2.09s, TFLOPs: 10.04, Samples/sec: 1.92, Time/seq 0.52s, Batch Size: 4, Sequence Length: 512
+Model Parameters: 6.927 B, Latency: 1.57s, TFLOPs: 13.36, Samples/sec: 2.55, Time/seq 0.39s, Batch Size: 4, Sequence Length: 512
+***** Evaluating perplexity, Epoch 2/10 *****
+Invalidate trace cache @ step 0: expected module 0, but got module 6
+ppl: 1.778276801109314, loss: 0.5756447911262512
+Beginning of Epoch 3/10, Total Micro Batches 6
+Model Parameters: 6.927 B, Latency: 2.31s, TFLOPs: 9.05, Samples/sec: 1.73, Time/seq 0.58s, Batch Size: 4, Sequence Length: 512
+Model Parameters: 6.927 B, Latency: 2.32s, TFLOPs: 9.01, Samples/sec: 1.72, Time/seq 0.58s, Batch Size: 4, Sequence Length: 512
+Model Parameters: 6.927 B, Latency: 2.11s, TFLOPs: 9.91, Samples/sec: 1.89, Time/seq 0.53s, Batch Size: 4, Sequence Length: 512
+Model Parameters: 6.927 B, Latency: 2.07s, TFLOPs: 10.12, Samples/sec: 1.93, Time/seq 0.52s, Batch Size: 4, Sequence Length: 512
+Model Parameters: 6.927 B, Latency: 2.06s, TFLOPs: 10.18, Samples/sec: 1.95, Time/seq 0.51s, Batch Size: 4, Sequence Length: 512
+Model Parameters: 6.927 B, Latency: 1.57s, TFLOPs: 13.32, Samples/sec: 2.54, Time/seq 0.39s, Batch Size: 4, Sequence Length: 512
+***** Evaluating perplexity, Epoch 3/10 *****
+Invalidate trace cache @ step 0: expected module 0, but got module 6
+ppl: 1.1743606328964233, loss: 0.16072379052639008
+Beginning of Epoch 4/10, Total Micro Batches 6
+Model Parameters: 6.927 B, Latency: 2.33s, TFLOPs: 8.98, Samples/sec: 1.72, Time/seq 0.58s, Batch Size: 4, Sequence Length: 512
+[2023-12-06 04:41:19,315] [INFO] [logging.py:96:log_dist] [Rank 0] step=20, skipped=0, lr=[7.237500000000001e-06, 0.000375, 7.237500000000001e-06], mom=[(0.9, 0.95), (0.9, 0.95), (0.9, 0.95)]
+[2023-12-06 04:41:19,316] [INFO] [timer.py:260:stop] epoch=3/micro_step=2/global_step=20, RunningAvgSamplesPerSec=7.491063054773498, CurrSamplesPerSec=6.840971579160688, MemAllocated=6.45GB, MaxMemAllocated=8.42GB
+Model Parameters: 6.927 B, Latency: 2.34s, TFLOPs: 8.94, Samples/sec: 1.71, Time/seq 0.59s, Batch Size: 4, Sequence Length: 512
+Model Parameters: 6.927 B, Latency: 2.06s, TFLOPs: 10.18, Samples/sec: 1.94, Time/seq 0.51s, Batch Size: 4, Sequence Length: 512
+Model Parameters: 6.927 B, Latency: 2.06s, TFLOPs: 10.17, Samples/sec: 1.94, Time/seq 0.51s, Batch Size: 4, Sequence Length: 512
+Model Parameters: 6.927 B, Latency: 2.05s, TFLOPs: 10.19, Samples/sec: 1.95, Time/seq 0.51s, Batch Size: 4, Sequence Length: 512
+Model Parameters: 6.927 B, Latency: 1.56s, TFLOPs: 13.39, Samples/sec: 2.56, Time/seq 0.39s, Batch Size: 4, Sequence Length: 512
+***** Evaluating perplexity, Epoch 4/10 *****
+Invalidate trace cache @ step 0: expected module 0, but got module 6
+ppl: 1.0426948070526123, loss: 0.04180852696299553
+Beginning of Epoch 5/10, Total Micro Batches 6
+Model Parameters: 6.927 B, Latency: 2.33s, TFLOPs: 9.00, Samples/sec: 1.72, Time/seq 0.58s, Batch Size: 4, Sequence Length: 512
+Model Parameters: 6.927 B, Latency: 2.33s, TFLOPs: 8.99, Samples/sec: 1.72, Time/seq 0.58s, Batch Size: 4, Sequence Length: 512
+Model Parameters: 6.927 B, Latency: 2.06s, TFLOPs: 10.15, Samples/sec: 1.94, Time/seq 0.52s, Batch Size: 4, Sequence Length: 512
+Model Parameters: 6.927 B, Latency: 2.07s, TFLOPs: 10.13, Samples/sec: 1.93, Time/seq 0.52s, Batch Size: 4, Sequence Length: 512
+Model Parameters: 6.927 B, Latency: 2.06s, TFLOPs: 10.14, Samples/sec: 1.94, Time/seq 0.52s, Batch Size: 4, Sequence Length: 512
+[2023-12-06 04:41:40,115] [INFO] [logging.py:96:log_dist] [Rank 0] step=30, skipped=0, lr=[4.825e-06, 0.00025, 4.825e-06], mom=[(0.9, 0.95), (0.9, 0.95), (0.9, 0.95)]
+[2023-12-06 04:41:40,116] [INFO] [timer.py:260:stop] epoch=4/micro_step=6/global_step=30, RunningAvgSamplesPerSec=7.642425087608659, CurrSamplesPerSec=9.97939015094399, MemAllocated=6.26GB, MaxMemAllocated=8.42GB
+Model Parameters: 6.927 B, Latency: 1.60s, TFLOPs: 13.04, Samples/sec: 2.49, Time/seq 0.40s, Batch Size: 4, Sequence Length: 512
+***** Evaluating perplexity, Epoch 5/10 *****
+Invalidate trace cache @ step 0: expected module 0, but got module 6
+ppl: 1.0130127668380737, loss: 0.012928837910294533
+Beginning of Epoch 6/10, Total Micro Batches 6
+Model Parameters: 6.927 B, Latency: 2.31s, TFLOPs: 9.08, Samples/sec: 1.73, Time/seq 0.58s, Batch Size: 4, Sequence Length: 512
+Model Parameters: 6.927 B, Latency: 2.34s, TFLOPs: 8.93, Samples/sec: 1.71, Time/seq 0.59s, Batch Size: 4, Sequence Length: 512
+Model Parameters: 6.927 B, Latency: 2.06s, TFLOPs: 10.16, Samples/sec: 1.94, Time/seq 0.52s, Batch Size: 4, Sequence Length: 512
+Model Parameters: 6.927 B, Latency: 2.06s, TFLOPs: 10.15, Samples/sec: 1.94, Time/seq 0.52s, Batch Size: 4, Sequence Length: 512
+Model Parameters: 6.927 B, Latency: 2.06s, TFLOPs: 10.15, Samples/sec: 1.94, Time/seq 0.52s, Batch Size: 4, Sequence Length: 512
+Model Parameters: 6.927 B, Latency: 1.57s, TFLOPs: 13.31, Samples/sec: 2.54, Time/seq 0.39s, Batch Size: 4, Sequence Length: 512
+***** Evaluating perplexity, Epoch 6/10 *****
+Invalidate trace cache @ step 0: expected module 0, but got module 6
+ppl: 1.006258487701416, loss: 0.0062390221282839775
+Beginning of Epoch 7/10, Total Micro Batches 6
+Model Parameters: 6.927 B, Latency: 2.31s, TFLOPs: 9.06, Samples/sec: 1.73, Time/seq 0.58s, Batch Size: 4, Sequence Length: 512
+Model Parameters: 6.927 B, Latency: 2.33s, TFLOPs: 9.00, Samples/sec: 1.72, Time/seq 0.58s, Batch Size: 4, Sequence Length: 512
+Model Parameters: 6.927 B, Latency: 2.06s, TFLOPs: 10.15, Samples/sec: 1.94, Time/seq 0.52s, Batch Size: 4, Sequence Length: 512
+[2023-12-06 04:42:02,526] [INFO] [logging.py:96:log_dist] [Rank 0] step=40, skipped=0, lr=[2.4125000000000015e-06, 0.00012500000000000006, 2.4125000000000015e-06], mom=[(0.9, 0.95), (0.9, 0.95), (0.9, 0.95)]
+[2023-12-06 04:42:02,526] [INFO] [timer.py:260:stop] epoch=6/micro_step=4/global_step=40, RunningAvgSamplesPerSec=7.620713816433709, CurrSamplesPerSec=7.736858876659631, MemAllocated=6.45GB, MaxMemAllocated=8.42GB
+Model Parameters: 6.927 B, Latency: 2.07s, TFLOPs: 10.12, Samples/sec: 1.93, Time/seq 0.52s, Batch Size: 4, Sequence Length: 512
+Model Parameters: 6.927 B, Latency: 2.07s, TFLOPs: 10.09, Samples/sec: 1.93, Time/seq 0.52s, Batch Size: 4, Sequence Length: 512
+Model Parameters: 6.927 B, Latency: 1.58s, TFLOPs: 13.22, Samples/sec: 2.53, Time/seq 0.40s, Batch Size: 4, Sequence Length: 512
+***** Evaluating perplexity, Epoch 7/10 *****
+Invalidate trace cache @ step 0: expected module 0, but got module 6
+ppl: 1.0054712295532227, loss: 0.005456325598061085
+Beginning of Epoch 8/10, Total Micro Batches 6
+Model Parameters: 6.927 B, Latency: 2.31s, TFLOPs: 9.06, Samples/sec: 1.73, Time/seq 0.58s, Batch Size: 4, Sequence Length: 512
+Model Parameters: 6.927 B, Latency: 2.33s, TFLOPs: 8.97, Samples/sec: 1.71, Time/seq 0.58s, Batch Size: 4, Sequence Length: 512
+Model Parameters: 6.927 B, Latency: 2.07s, TFLOPs: 10.09, Samples/sec: 1.93, Time/seq 0.52s, Batch Size: 4, Sequence Length: 512
+Model Parameters: 6.927 B, Latency: 2.07s, TFLOPs: 10.09, Samples/sec: 1.93, Time/seq 0.52s, Batch Size: 4, Sequence Length: 512
+Model Parameters: 6.927 B, Latency: 2.07s, TFLOPs: 10.10, Samples/sec: 1.93, Time/seq 0.52s, Batch Size: 4, Sequence Length: 512
+Model Parameters: 6.927 B, Latency: 1.59s, TFLOPs: 13.14, Samples/sec: 2.51, Time/seq 0.40s, Batch Size: 4, Sequence Length: 512
+***** Evaluating perplexity, Epoch 8/10 *****
+Invalidate trace cache @ step 0: expected module 0, but got module 6
+ppl: 1.005515694618225, loss: 0.0055005596950650215
+Beginning of Epoch 9/10, Total Micro Batches 6
+Model Parameters: 6.927 B, Latency: 2.30s, TFLOPs: 9.09, Samples/sec: 1.74, Time/seq 0.58s, Batch Size: 4, Sequence Length: 512
+[2023-12-06 04:42:24,503] [INFO] [logging.py:96:log_dist] [Rank 0] step=50, skipped=0, lr=[6.464274267400833e-07, 3.3493649053890325e-05, 6.464274267400833e-07], mom=[(0.9, 0.95), (0.9, 0.95), (0.9, 0.95)]
+[2023-12-06 04:42:24,503] [INFO] [timer.py:260:stop] epoch=8/micro_step=2/global_step=50, RunningAvgSamplesPerSec=7.638560733715662, CurrSamplesPerSec=6.81366785411214, MemAllocated=6.45GB, MaxMemAllocated=8.42GB
+Model Parameters: 6.927 B, Latency: 2.35s, TFLOPs: 8.91, Samples/sec: 1.70, Time/seq 0.59s, Batch Size: 4, Sequence Length: 512
+Model Parameters: 6.927 B, Latency: 2.06s, TFLOPs: 10.15, Samples/sec: 1.94, Time/seq 0.52s, Batch Size: 4, Sequence Length: 512
+Model Parameters: 6.927 B, Latency: 2.06s, TFLOPs: 10.14, Samples/sec: 1.94, Time/seq 0.52s, Batch Size: 4, Sequence Length: 512
+Model Parameters: 6.927 B, Latency: 2.07s, TFLOPs: 10.13, Samples/sec: 1.94, Time/seq 0.52s, Batch Size: 4, Sequence Length: 512
+Model Parameters: 6.927 B, Latency: 1.59s, TFLOPs: 13.19, Samples/sec: 2.52, Time/seq 0.40s, Batch Size: 4, Sequence Length: 512
+***** Evaluating perplexity, Epoch 9/10 *****
+Invalidate trace cache @ step 0: expected module 0, but got module 6
+ppl: 1.0054373741149902, loss: 0.005422617308795452
+Beginning of Epoch 10/10, Total Micro Batches 6
+Model Parameters: 6.927 B, Latency: 2.31s, TFLOPs: 9.08, Samples/sec: 1.74, Time/seq 0.58s, Batch Size: 4, Sequence Length: 512
+Model Parameters: 6.927 B, Latency: 2.32s, TFLOPs: 9.02, Samples/sec: 1.72, Time/seq 0.58s, Batch Size: 4, Sequence Length: 512
+Model Parameters: 6.927 B, Latency: 2.06s, TFLOPs: 10.15, Samples/sec: 1.94, Time/seq 0.52s, Batch Size: 4, Sequence Length: 512
+Model Parameters: 6.927 B, Latency: 2.06s, TFLOPs: 10.14, Samples/sec: 1.94, Time/seq 0.52s, Batch Size: 4, Sequence Length: 512
+Model Parameters: 6.927 B, Latency: 2.06s, TFLOPs: 10.15, Samples/sec: 1.94, Time/seq 0.52s, Batch Size: 4, Sequence Length: 512
+[2023-12-06 04:42:45,284] [INFO] [logging.py:96:log_dist] [Rank 0] step=60, skipped=0, lr=[0.0, 0.0, 0.0], mom=[(0.9, 0.95), (0.9, 0.95), (0.9, 0.95)]
+[2023-12-06 04:42:45,284] [INFO] [timer.py:260:stop] epoch=9/micro_step=6/global_step=60, RunningAvgSamplesPerSec=7.688756169911607, CurrSamplesPerSec=10.190957881216764, MemAllocated=6.26GB, MaxMemAllocated=8.42GB
+Model Parameters: 6.927 B, Latency: 1.57s, TFLOPs: 13.32, Samples/sec: 2.55, Time/seq 0.39s, Batch Size: 4, Sequence Length: 512
+***** Evaluating perplexity, Epoch 10/10 *****
+Invalidate trace cache @ step 0: expected module 0, but got module 6
+ppl: 1.005433440208435, loss: 0.005418714135885239
+saving the final model ...
+[2023-12-06 04:42:59,262] [INFO] [launch.py:347:main] Process 2519341 exits successfully.
+[2023-12-06 04:42:59,263] [INFO] [launch.py:347:main] Process 2519342 exits successfully.
+[2023-12-06 04:42:59,263] [INFO] [launch.py:347:main] Process 2519343 exits successfully.
+[2023-12-06 04:44:22,272] [INFO] [launch.py:347:main] Process 2519340 exits successfully.