init dump op

dropout1+norm1_dp_0_step_0_block_0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a415736a3ca38d0834819842a7d78d1d1b6cf8481eaac3ee01e2b83342892684
+size 67109756

dropout2+norm2_dp_0_step_0_block_0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:238d667ec49817b51869d1dcc364eceed663ff22f01f2d3fd87024564bf6b7b3
+size 67109756

embedding_dp_0_step_0_block_0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:afd5a5a542b1d67ff7949a0b25e3af07fa46e6582355706d25762917f2c402da
+size 67109741

head_dp_0_step_0_block_0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2a11d67187396ffe6c6cf81eb290bbf9613d96e56227070ef6e7039f805195d6
+size 1690305374

inner_attn_dp_0_step_0_block_0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:981deed2f78989963611d2f2a531bd4f17fb296e8d3399ea642a318836ca5703
+size 67109744

mlp_dp_0_step_0_block_0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9251ba22eeae2bea6794349c85a36294e1dfb25c4537d85c2515f9c096536c66
+size 67109723

norm_dp_0_step_0_block_0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:93505a58566f013908b96acad48dda9e856629b29653e77702dce2c5c6454b6a
+size 67109726

out_proj_dp_0_step_0_block_0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e1e2e2fa21a78a96837429c8df8ef0b923f330f414d3d3b3416c3165da342420
+size 67109738

print_op.log

@@ -0,0 +1,510 @@
+srun: job 2085180 queued and waiting for resources
+srun: job 2085180 has been allocated resources
+srun: Job 2085180 scheduled successfully!
+Current QUOTA_TYPE is [spot], which means the job has occupied quota in SPOT_TOTAL under your partition.
+[NOTE]: This job MAY BE PREEMPTED by other jobs of reserved QUOTA_TYPE.
+[NOTE]: phx_priority has NO effect on job of spot quota type.
+
+2023-12-04 21:29:09,765	INFO launch.py:557 in try_bind_numa -- Try bind numa failed! Package import error, if numa is not installed, please implement: pip install --upgrade py-libnuma, Ref: https://pypi.org/project/py-libnuma/
+2023-12-04 21:29:10,043	INFO parallel_context.py:555 in set_device -- process rank 6 is bound to host:HOST-10-140-60-82 device: 6
+2023-12-04 21:29:10,047	INFO parallel_context.py:555 in set_device -- process rank 0 is bound to host:HOST-10-140-60-82 device: 0
+2023-12-04 21:29:10,050	INFO parallel_context.py:555 in set_device -- process rank 4 is bound to host:HOST-10-140-60-82 device: 4
+2023-12-04 21:29:10,080	INFO parallel_context.py:555 in set_device -- process rank 1 is bound to host:HOST-10-140-60-82 device: 1
+2023-12-04 21:29:10,081	INFO parallel_context.py:555 in set_device -- process rank 7 is bound to host:HOST-10-140-60-82 device: 7
+2023-12-04 21:29:10,111	INFO parallel_context.py:555 in set_device -- process rank 2 is bound to host:HOST-10-140-60-82 device: 2
+2023-12-04 21:29:10,111	INFO parallel_context.py:555 in set_device -- process rank 5 is bound to host:HOST-10-140-60-82 device: 5
+2023-12-04 21:29:10,141	INFO parallel_context.py:555 in set_device -- process rank 3 is bound to host:HOST-10-140-60-82 device: 3
+2023-12-04 21:29:13,419	INFO parallel_context.py:593 in set_seed -- initialized seed on rank 0, numpy: 1024, python random: 1024, ParallelMode.DATA: 1024, ParallelMode.DUMMY: 1024, ParallelMode.TENSOR: 1024,the default parallel seed is ParallelMode.DATA.
+2023-12-04 21:29:13,420	INFO launch.py:411 in launch -- Distributed environment is initialized, data parallel size: 8, pipeline parallel size: 1, tensor parallel size: 1
+2023-12-04 21:29:13,419	INFO parallel_context.py:593 in set_seed -- initialized seed on rank 6, numpy: 1024, python random: 1024, ParallelMode.DATA: 1024, ParallelMode.DUMMY: 1024, ParallelMode.TENSOR: 1024,the default parallel seed is ParallelMode.DATA.
+2023-12-04 21:29:13,420	INFO launch.py:114 in args_sanity_check -- gradient_accumulation size will be setted to 1.
+2023-12-04 21:29:13,420	INFO parallel_context.py:593 in set_seed -- initialized seed on rank 3, numpy: 1024, python random: 1024, ParallelMode.DATA: 1024, ParallelMode.DUMMY: 1024, ParallelMode.TENSOR: 1024,the default parallel seed is ParallelMode.DATA.
+2023-12-04 21:29:13,420	INFO launch.py:142 in args_sanity_check -- +++++++++++++++ Data Info +++++++++++++++
+2023-12-04 21:29:13,420	INFO launch.py:143 in args_sanity_check -- seq_len: 2048
+2023-12-04 21:29:13,420	INFO launch.py:144 in args_sanity_check -- micro_num: 1
+2023-12-04 21:29:13,420	INFO launch.py:145 in args_sanity_check -- micro_bsz: 4
+2023-12-04 21:29:13,420	INFO launch.py:146 in args_sanity_check -- packed_length: 8192
+2023-12-04 21:29:13,420	INFO launch.py:147 in args_sanity_check -- pack_sample_into_one: False
+2023-12-04 21:29:13,420	INFO launch.py:148 in args_sanity_check -- min_length: 50
+2023-12-04 21:29:13,420	INFO launch.py:149 in args_sanity_check -- valid_micro_num: 4
+2023-12-04 21:29:13,420	INFO launch.py:150 in args_sanity_check -- valid_every: 500
+2023-12-04 21:29:13,420	INFO launch.py:203 in args_sanity_check -- +++++++++++++++ Ckpt Info +++++++++++++++
+2023-12-04 21:29:13,420	INFO launch.py:204 in args_sanity_check -- is enable save ckpt: False
+2023-12-04 21:29:13,420	INFO launch.py:205 in args_sanity_check -- save_ckpt_folder: None
+2023-12-04 21:29:13,420	INFO launch.py:206 in args_sanity_check -- checkpoint_every: inf
+2023-12-04 21:29:13,420	INFO launch.py:221 in args_sanity_check -- tensorboard_folder: None
+2023-12-04 21:29:13,420	INFO launch.py:222 in args_sanity_check -- resume_tb_folder: None
+2023-12-04 21:29:13,420	INFO launch.py:230 in args_sanity_check -- +++++++++++++++ Other Info +++++++++++++++
+2023-12-04 21:29:13,420	INFO launch.py:231 in args_sanity_check -- cudnn.benchmark: False
+2023-12-04 21:29:13,420	INFO launch.py:232 in args_sanity_check -- cudnn.deterministic: False
+2023-12-04 21:29:13,420	INFO launch.py:233 in args_sanity_check -- clip_grad_norm: 1.0
+2023-12-04 21:29:13,420	INFO launch.py:270 in args_sanity_check -- +++++++++++++++ Model Info +++++++++++++++
+2023-12-04 21:29:13,420	INFO parallel_context.py:593 in set_seed -- initialized seed on rank 4, numpy: 1024, python random: 1024, ParallelMode.DATA: 1024, ParallelMode.DUMMY: 1024, ParallelMode.TENSOR: 1024,the default parallel seed is ParallelMode.DATA.
+2023-12-04 21:29:13,420	INFO launch.py:271 in args_sanity_check -- Model: {'checkpoint': 0, 'num_attention_heads': 32, 'embed_split_hidden': True, 'vocab_size': 103168, 'embed_grad_scale': 1, 'parallel_output': True, 'hidden_size': 4096, 'num_layers': 32, 'mlp_ratio': 2.6666666666666665, 'apply_post_layer_norm': False, 'dtype': torch.float16, 'norm_type': 'rmsnorm', 'layer_norm_epsilon': 1e-06, 'use_flash_attn': True, 'num_chunks': 1}
+2023-12-04 21:29:13,420	INFO launch.py:273 in args_sanity_check -- +++++++++++++++ grad_scaler Info +++++++++++++++
+2023-12-04 21:29:13,420	INFO launch.py:274 in args_sanity_check -- grad_scaler: {'fp16': {'initial_scale': 65536, 'min_scale': 1, 'growth_interval': 1000}, 'growth_factor': 2, 'backoff_factor': 0.5, 'max_scale': 16777216, 'hysteresis': 2}
+2023-12-04 21:29:13,420	INFO launch.py:276 in args_sanity_check -- +++++++++++++++ hybrid_zero_optimizer Info +++++++++++++++
+2023-12-04 21:29:13,420	INFO launch.py:277 in args_sanity_check -- hybrid_zero_optimizer: {'overlap_sync_grad': True, 'overlap_sync_param': False, 'reduce_bucket_size': 536870912, 'clip_grad_norm': 1.0}
+2023-12-04 21:29:13,420	INFO launch.py:279 in args_sanity_check -- +++++++++++++++ adam Info +++++++++++++++
+2023-12-04 21:29:13,420	INFO launch.py:280 in args_sanity_check -- adam: {'lr': 2e-06, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_beta2_c': 0, 'adam_eps': 1e-08, 'weight_decay': 0.01}
+2023-12-04 21:29:13,420	INFO launch.py:282 in args_sanity_check -- +++++++++++++++ beta2_scheduler Info +++++++++++++++
+2023-12-04 21:29:13,421	INFO launch.py:283 in args_sanity_check -- beta2_scheduler: {'init_beta2': 0.95, 'c': 0, 'cur_iter': -1}
+2023-12-04 21:29:13,421	INFO launch.py:340 in args_sanity_check -- overlap_sync_grad:True, overlap_sync_param:False
+2023-12-04 21:29:13,420	INFO parallel_context.py:593 in set_seed -- initialized seed on rank 2, numpy: 1024, python random: 1024, ParallelMode.DATA: 1024, ParallelMode.DUMMY: 1024, ParallelMode.TENSOR: 1024,the default parallel seed is ParallelMode.DATA.
+2023-12-04 21:29:13,420	INFO parallel_context.py:593 in set_seed -- initialized seed on rank 5, numpy: 1024, python random: 1024, ParallelMode.DATA: 1024, ParallelMode.DUMMY: 1024, ParallelMode.TENSOR: 1024,the default parallel seed is ParallelMode.DATA.
+2023-12-04 21:29:13,420	INFO parallel_context.py:593 in set_seed -- initialized seed on rank 7, numpy: 1024, python random: 1024, ParallelMode.DATA: 1024, ParallelMode.DUMMY: 1024, ParallelMode.TENSOR: 1024,the default parallel seed is ParallelMode.DATA.
+2023-12-04 21:29:13,420	INFO parallel_context.py:593 in set_seed -- initialized seed on rank 1, numpy: 1024, python random: 1024, ParallelMode.DATA: 1024, ParallelMode.DUMMY: 1024, ParallelMode.TENSOR: 1024,the default parallel seed is ParallelMode.DATA.
+2023-12-04 21:29:13,471	INFO modeling_internlm.py:459 in _build_generic_model_1d -- The layer sharding is [[(0, 32)]].
+2023-12-04 21:29:19,251	INFO packed_dataset.py:378 in get_packed_dataset_without_short_length -- Reading /mnt/petrelfs/share_data/lijiaxing/wikitext-2-tokenize/train...
+
                  
+
  0%|          | 0/2 [00:00<?, ?it/s]
                                     
+2023-12-04 21:29:19,282	INFO training_internlm.py:334 in get_validation_data_loader -- load validation dataset en with valid batch size 16 and samples 11.
+2023-12-04 21:29:19,340	INFO hybrid_zero_optim.py:268 in _partition_param_list -- Number of elements on ranks: [907415552, 907411456, 910163968, 910163968, 921698304, 921698304, 921698304, 921698304], rank:0
+2023-12-04 21:29:35,932	INFO hybrid_zero_optim.py:268 in _partition_param_list -- Number of elements on ranks: [0, 0, 0, 0, 0, 0, 0, 0], rank:0
+2023-12-04 21:29:35,937	WARNING model_checkpoint.py:786 in __init__ -- no set stop_file_path, quit_signal_handler is disable
+2023-12-04 21:29:35,937	INFO model_checkpoint.py:208 in try_load_internlm_ckpt -- Try load_ckpt_folder: /mnt/petrelfs/share/lijiaxing/Huawei_init_ckpt
+2023-12-04 21:29:35,937	WARNING storage_manager.py:285 in try_get_storage_backend -- path: '/mnt/petrelfs/share/lijiaxing/Huawei_init_ckpt' not start with backend prefix, guess it is the backend of local.
+2023-12-04 21:29:35,941	WARNING storage_manager.py:285 in try_get_storage_backend -- path: '/mnt/petrelfs/share/lijiaxing/Huawei_init_ckpt/model_tp0_pp0.pt' not start with backend prefix, guess it is the backend of local.
+2023-12-04 21:29:53,062	WARNING storage_manager.py:285 in try_get_storage_backend -- path: '/mnt/petrelfs/share/lijiaxing/Huawei_init_ckpt/context.pt' not start with backend prefix, guess it is the backend of local.
+2023-12-04 21:29:53,063	INFO model_checkpoint.py:664 in load_context -- reload train_state:{
+    "batch_count": 1,
+    "inf_nan_skip_batches": 0,
+    "num_consumed_samples_in_epoch": 0,
+    "num_consumed_tokens": 0,
+    "step_count": 0
+}
+2023-12-04 21:29:53,063	WARNING storage_manager.py:285 in try_get_storage_backend -- path: '/mnt/petrelfs/share/lijiaxing/Huawei_init_ckpt' not start with backend prefix, guess it is the backend of local.
+2023-12-04 21:29:53,071	WARNING storage_manager.py:285 in try_get_storage_backend -- path: '/mnt/petrelfs/share/lijiaxing/Huawei_init_ckpt/optimizer_tp0_pp0_zo0.pt' not start with backend prefix, guess it is the backend of local.
+2023-12-04 21:29:56,663	WARNING storage_manager.py:285 in try_get_storage_backend -- path: '/mnt/petrelfs/share/lijiaxing/Huawei_init_ckpt/gpus-8_pp-0_tp-0_zo-0.pt' not start with backend prefix, guess it is the backend of local.
+2023-12-04 21:29:56,699	WARNING storage_manager.py:285 in try_get_storage_backend -- path: '/mnt/petrelfs/share/lijiaxing/Huawei_init_ckpt/schedulder.pt' not start with backend prefix, guess it is the backend of local.
+2023-12-04 21:29:56,704	INFO model_checkpoint.py:696 in load_scheduler -- reload load_scheduler:{
+    "_get_lr_called_within_step": false,
+    "_init_steps": 0,
+    "_last_lr": [
+        6.666666666666667e-08,
+        6.666666666666667e-08
+    ],
+    "_step_count": 1,
+    "_warmup_steps": 30,
+    "after_scheduler_dict": {
+        "T_max": 970,
+        "_get_lr_called_within_step": false,
+        "_last_lr": [
+            2e-06,
+            2e-06
+        ],
+        "_step_count": 1,
+        "base_lrs": [
+            2e-06,
+            2e-06
+        ],
+        "eta_min": 1e-05,
+        "last_epoch": 0,
+        "verbose": false
+    },
+    "after_scheduler_type": "CosineAnnealingLR",
+    "base_lrs": [
+        2e-06,
+        2e-06
+    ],
+    "finished": false,
+    "last_epoch": 1,
+    "verbose": false,
+    "warmup_epochs": 30
+}
+2023-12-04 21:29:56,704	WARNING storage_manager.py:285 in try_get_storage_backend -- path: '/mnt/petrelfs/share/lijiaxing/Huawei_init_ckpt/sampler.pt' not start with backend prefix, guess it is the backend of local.
+2023-12-04 21:29:56,725	INFO model_checkpoint.py:656 in load_sampler -- reload sampler_states:{'batch_size': 1, 'raw_rampup_batch_size': '1 1 1', 'epoch': 0, 'seed': 1024, 'data_world_size': 8, 'num_consumed_samples_in_epoch': 0, 'batch_count': 0}
+2023-12-04 21:29:56,725	WARNING model_checkpoint.py:256 in try_load_internlm_ckpt -- CheckpointManager has no 'data_state_dict', skip reload data_state_dict checkpoint!
+2023-12-04 21:29:56,725	INFO model_checkpoint.py:1012 in try_resume_training -- load_ckpt_info : {'path': '/mnt/petrelfs/share/lijiaxing/Huawei_init_ckpt', 'content': {'optimizer', 'scheduler', 'sampler', 'model'}., 'ckpt_type': <CheckpointLoadType.INTERNLM: 'internlm'>}
+2023-12-04 21:29:56,725	INFO model_checkpoint.py:1013 in try_resume_training -- ===========Resume training from `/mnt/petrelfs/share/lijiaxing/Huawei_init_ckpt` Dec04_21-29-13 on host:HOST-10-140-60-82===========
+2023-12-04 21:29:56,725	INFO model_checkpoint.py:1018 in try_resume_training -- ===========Load contents are: model, optimizer, scheduler, sampler, 
+2023-12-04T21:29:56.726+08:00 INFO [writer.py, line 57, in init_tb_writer] - pid=13773 : Login tensorboard logs to: hw_7B_wiki/Dec04_21-29-13/tensorboards
+fwd hidden_states embedding output: tensor([[[ 0.0032, -0.0058,  0.0047,  ...,  0.0020, -0.0094, -0.0037],
+         [ 0.0022,  0.0049, -0.0017,  ...,  0.0052,  0.0074,  0.0013],
+         [ 0.0035,  0.0006, -0.0002,  ...,  0.0065, -0.0012,  0.0007],
+         ...,
+         [-0.0060,  0.0068,  0.0008,  ..., -0.0005,  0.0059, -0.0005],
+         [-0.0060,  0.0068,  0.0008,  ..., -0.0005,  0.0059, -0.0005],
+         [-0.0060,  0.0068,  0.0008,  ..., -0.0005,  0.0059, -0.0005]]],
+       device='cuda:0', dtype=torch.float16), shape:torch.Size([1, 8192, 4096])
+/mnt/petrelfs/share_data/llm_env/miniconda3-py39_4/envs/llm-flash2.0/lib/python3.10/site-packages/torch/utils/checkpoint.py:31: UserWarning: None of the inputs have requires_grad=True. Gradients will be None
+  warnings.warn("None of the inputs have requires_grad=True. Gradients will be None")
+fwd block 0 dropout1+norm1: tensor([[ 0.5889, -1.0645,  0.8765,  ...,  0.3772, -1.7451, -0.6895],
+        [ 0.4214,  0.9590, -0.3362,  ...,  1.0176,  1.4424,  0.2605],
+        [ 0.6533,  0.1079, -0.0458,  ...,  1.2100, -0.2299,  0.1229],
+        ...,
+        [-1.1475,  1.2939,  0.1488,  ..., -0.0983,  1.1221, -0.0894],
+        [-1.1475,  1.2939,  0.1488,  ..., -0.0983,  1.1221, -0.0894],
+        [-1.1475,  1.2939,  0.1488,  ..., -0.0983,  1.1221, -0.0894]],
+       device='cuda:0', dtype=torch.float16), shape:torch.Size([8192, 4096])
+fwd block 0 qkv: tensor([[ 0.0233,  0.1367,  0.4971,  ...,  0.9688, -0.5342,  0.0586],
+        [-0.6387, -0.0337, -0.0786,  ..., -0.6523, -0.4207, -0.0371],
+        [-0.2412, -0.4653, -0.1932,  ..., -0.1326, -0.3643,  0.1268],
+        ...,
+        [ 0.4270,  0.5815, -0.5161,  ..., -0.0453,  0.0557,  0.1555],
+        [ 0.4270,  0.5815, -0.5161,  ..., -0.0453,  0.0557,  0.1555],
+        [ 0.4270,  0.5815, -0.5161,  ..., -0.0453,  0.0557,  0.1555]],
+       device='cuda:0', dtype=torch.float16), shape: torch.Size([8192, 12288])
+fwd block 0 rotary_emb: tensor([[[[ 2.3315e-02,  1.3672e-01,  4.9707e-01,  ..., -7.3340e-01,
+           -1.3013e-01, -6.0596e-01],
+          [ 3.7744e-01, -2.0618e-01,  3.7964e-02,  ..., -1.9128e-01,
+           -1.0815e-01, -2.7740e-02],
+          [-3.4668e-01, -5.4248e-01,  9.3994e-02,  ...,  3.0591e-01,
+            2.3315e-01,  5.7764e-01],
+          ...,
+          [ 7.6318e-01,  3.5132e-01, -2.1387e-01,  ...,  6.4502e-01,
+            4.2847e-01,  3.4448e-01],
+          [-3.4888e-01,  6.0352e-01,  6.1310e-02,  ..., -2.1228e-01,
+            3.9478e-01, -2.7393e-01],
+          [-1.3904e-01, -1.0815e-01, -3.6206e-01,  ..., -6.6040e-02,
+            3.6694e-01,  5.1178e-02]],
+
+         [[ 4.3365e-02,  4.4629e-01, -7.2559e-01,  ...,  1.4905e-01,
+            4.1211e-01,  2.5098e-01],
+          [ 9.1400e-03,  1.3086e-01,  4.7998e-01,  ..., -4.4702e-01,
+           -5.9375e-01,  1.8774e-01],
+          [ 1.2732e-01,  4.9976e-01,  1.2158e-01,  ...,  8.2031e-02,
+            3.7292e-02, -5.3271e-01],
+          ...,
+          [-3.2886e-01,  2.0435e-01, -1.6479e-01,  ..., -1.2231e-01,
+           -3.4473e-01, -3.7427e-01],
+          [-1.8689e-01, -3.5645e-01, -2.5513e-01,  ..., -3.3594e-01,
+           -5.3418e-01,  1.0291e-01],
+          [ 1.1407e-01,  2.1436e-01, -2.9922e-02,  ...,  1.4563e-01,
+           -1.1530e-01, -1.8875e-02]],
+
+         [[-2.4438e-01, -1.0455e-01,  6.7432e-01,  ..., -1.6272e-01,
+            1.6626e-01, -4.7729e-01],
+          [ 9.6680e-02, -1.6663e-01, -1.8176e-01,  ...,  5.1660e-01,
+           -1.5051e-01, -3.8483e-02],
+          [-8.3545e-01, -4.9805e-02,  4.9048e-01,  ...,  2.1558e-01,
+           -4.5435e-01,  2.7246e-01],
+          ...,
+          [ 6.4636e-02,  5.4932e-01, -1.3489e-01,  ..., -3.2666e-01,
+            8.6914e-01, -4.6851e-01],
+          [ 5.0146e-01, -1.7175e-01,  1.2947e-02,  ...,  2.4194e-01,
+            1.0381e+00, -9.9268e-01],
+          [-2.0740e-01, -1.8738e-01, -4.6143e-02,  ...,  9.6875e-01,
+           -5.3418e-01,  5.8563e-02]]],
+
+
+        [[[-9.1260e-01, -4.0430e-01, -6.7932e-02,  ...,  4.1284e-01,
+           -2.3608e-01,  1.0321e-01],
+          [ 3.5376e-01, -5.5664e-02, -1.7358e-01,  ...,  3.3252e-01,
+           -7.1239e-04, -4.6338e-01],
+          [-5.9229e-01, -6.0156e-01,  3.7964e-01,  ..., -1.9788e-01,
+            2.1863e-01,  4.4922e-01],
+          ...,
+          [ 9.6436e-02,  3.3643e-01,  2.5244e-01,  ...,  7.6074e-01,
+            4.8242e-01, -2.0435e-01],
+          [ 2.6172e-01,  2.7808e-01, -6.1768e-01,  ...,  2.4500e-01,
+            2.8052e-01,  1.1230e+00],
+          [-7.6562e-01, -5.5762e-01, -1.5454e-01,  ..., -1.6394e-01,
+           -2.9810e-01,  5.3174e-01]],
+
+         [[ 2.2888e-02,  5.1855e-01, -4.6216e-01,  ...,  2.9297e-01,
+            2.6367e-01,  1.6821e-01],
+          [-1.7944e-01, -1.8835e-01,  1.0059e-01,  ..., -2.5024e-01,
+           -1.4551e-01, -4.7583e-01],
+          [-5.1208e-02, -7.2705e-01,  5.0293e-01,  ...,  8.8135e-02,
+           -4.2505e-01,  1.8628e-01],
+          ...,
+          [-4.6948e-01, -2.5317e-01, -4.1577e-01,  ..., -5.4688e-01,
+            3.1348e-01, -2.8418e-01],
+          [-4.9934e-03,  4.9463e-01, -2.9980e-01,  ..., -3.9111e-01,
+            2.1988e-02,  1.1804e-01],
+          [ 2.1960e-01, -3.7622e-01,  2.9272e-01,  ...,  1.0098e+00,
+            1.6089e-01,  4.6338e-01]],
+
+         [[ 3.5858e-02,  6.5771e-01,  3.9966e-01,  ..., -8.8574e-01,
+            3.0930e-02, -9.5398e-02],
+          [ 5.8545e-01,  4.2236e-01, -2.6221e-01,  ..., -2.7856e-01,
+           -1.6418e-01, -3.5571e-01],
+          [ 1.5295e-01,  1.5601e-01,  1.3867e-01,  ..., -6.7285e-01,
+           -2.8394e-01, -1.1748e+00],
+          ...,
+          [ 1.5820e-01,  1.0748e-01,  5.3174e-01,  ..., -1.1115e-01,
+           -4.9268e-01, -6.9153e-02],
+          [ 7.7246e-01,  1.6174e-01, -2.7759e-01,  ...,  5.1660e-01,
+            8.2275e-02, -8.0273e-01],
+          [-2.0276e-01,  6.0400e-01, -3.4760e-02,  ..., -6.5234e-01,
+           -4.2065e-01, -3.7079e-02]]],
+
+
+        [[[ 1.8219e-02,  8.2336e-02, -5.6543e-01,  ...,  3.6133e-01,
+           -5.1660e-01, -4.4952e-02],
+          [ 6.0889e-01,  4.1534e-02,  3.8086e-01,  ..., -1.1401e-01,
+            8.3057e-01, -2.1716e-01],
+          [-1.2891e-01,  2.0300e-01, -5.6396e-02,  ...,  2.1265e-01,
+           -1.8823e-01, -5.8252e-01],
+          ...,
+          [-6.6455e-01,  3.0591e-01,  1.7224e-01,  ..., -3.7939e-01,
+            6.4111e-01, -3.6523e-01],
+          [ 4.3579e-01, -7.4121e-01,  6.0150e-02,  ...,  2.2205e-01,
+            8.7967e-03,  2.7817e-02],
+          [-2.2095e-02,  3.7524e-01, -6.2891e-01,  ...,  1.4214e-02,
+            4.2676e-01,  8.1482e-03]],
+
+         [[ 1.0944e-01,  3.4277e-01, -4.1382e-01,  ..., -3.4448e-01,
+           -2.5195e-01, -7.0264e-01],
+          [-1.8478e-02, -5.6122e-02,  3.1152e-01,  ..., -2.6660e-01,
+           -5.4736e-01, -5.4102e-01],
+          [-8.6792e-02, -2.8784e-01,  2.7759e-01,  ...,  3.2739e-01,
+           -1.1694e-01,  3.1152e-01],
+          ...,
+          [-4.0405e-01, -5.0293e-01, -6.2939e-01,  ...,  1.3147e-01,
+           -1.6833e-01, -3.3472e-01],
+          [-5.4102e-01,  3.8062e-01, -2.5952e-01,  ...,  4.1553e-01,
+           -7.0374e-02, -8.4717e-02],
+          [ 5.2795e-02, -9.5276e-02,  5.0537e-02,  ..., -3.4741e-01,
+           -2.9443e-01,  4.2017e-01]],
+
+         [[-3.2202e-01,  2.3694e-01, -1.3379e-01,  ..., -2.9907e-01,
+           -5.1904e-01, -6.5674e-01],
+          [-1.0273e+00, -3.4521e-01,  2.4268e-01,  ...,  3.4961e-01,
+           -3.6865e-01,  1.6492e-01],
+          [-3.7402e-01,  1.7322e-01,  3.9014e-01,  ...,  1.9128e-01,
+           -7.3242e-01, -3.4814e-01],
+          ...,
+          [-3.6182e-01,  1.7563e-02, -5.7666e-01,  ..., -1.5759e-01,
+            4.2261e-01,  2.8595e-02],
+          [-1.3329e-02, -2.3425e-01,  3.9844e-01,  ...,  1.9519e-01,
+           -1.2903e-01,  6.5674e-02],
+          [ 2.6465e-01, -1.8774e-01, -2.8046e-02,  ..., -1.3257e-01,
+           -3.6426e-01,  1.2683e-01]]],
+
+
+        ...,
+
+
+        [[[ 1.5857e-01,  6.2305e-01,  4.9268e-01,  ..., -1.2830e-01,
+           -3.4637e-02, -1.6296e-01],
+          [ 1.6281e-02,  2.2083e-01,  3.3081e-01,  ..., -5.8990e-02,
+           -6.9824e-02,  6.0547e-01],
+          [ 1.3770e-01,  6.6309e-01, -4.0234e-01,  ...,  2.1378e-02,
+            1.6931e-01,  1.4771e-01],
+          ...,
+          [ 4.8999e-01, -8.6914e-02, -2.4255e-01,  ...,  1.0919e-01,
+            1.9958e-01,  2.6538e-01],
+          [-1.1060e-01, -5.9668e-01,  1.7126e-01,  ..., -3.0444e-01,
+           -2.2046e-01, -6.6406e-02],
+          [ 3.5498e-01, -4.1089e-01, -3.9502e-01,  ...,  2.1582e-01,
+           -3.3521e-01, -2.7664e-02]],
+
+         [[-2.2632e-01,  4.2798e-01, -1.2231e-01,  ...,  1.3989e-01,
+            2.1191e-01,  2.5464e-01],
+          [ 2.5732e-01, -1.1481e-01, -1.4685e-01,  ..., -7.9150e-01,
+            5.3894e-02, -4.5410e-01],
+          [ 1.9751e-01,  1.1151e-01,  1.3878e-02,  ...,  5.6396e-01,
+           -5.0537e-01,  7.4890e-02],
+          ...,
+          [-6.9458e-02,  9.2468e-02,  6.1816e-01,  ...,  7.2632e-02,
+            6.3281e-01, -8.3557e-02],
+          [ 8.4521e-01,  7.1533e-02,  1.4087e-01,  ..., -1.5686e-01,
+           -3.5693e-01,  2.9956e-01],
+          [ 5.8545e-01,  2.6758e-01,  1.4717e-02,  ...,  6.6452e-03,
+           -3.9307e-02,  3.0151e-01]],
+
+         [[-1.6223e-01,  7.2693e-02, -5.1514e-01,  ...,  2.1289e-01,
+           -1.4880e-01,  2.5220e-01],
+          [ 3.7036e-01, -1.2732e-01,  2.5391e-01,  ...,  3.2324e-01,
+           -5.5939e-02,  7.5293e-01],
+          [ 3.8330e-01,  9.2676e-01, -2.7173e-01,  ..., -1.5823e-02,
+            1.0553e-01, -5.1221e-01],
+          ...,
+          [-3.7549e-01, -4.5337e-01, -2.2729e-01,  ..., -4.9219e-01,
+           -4.7974e-01, -3.4204e-01],
+          [ 4.5581e-01, -3.8501e-01, -9.6497e-02,  ...,  2.0401e-02,
+           -8.5266e-02,  3.2739e-01],
+          [-2.0386e-01,  2.3010e-01, -5.5713e-01,  ..., -4.5288e-02,
+            5.5725e-02,  1.5552e-01]]],
+
+
+        [[[ 4.4873e-01,  1.8652e-01,  2.2693e-01,  ..., -1.2842e-01,
+           -3.4637e-02, -1.6333e-01],
+          [-2.2864e-01,  5.4785e-01,  1.3708e-01,  ..., -5.8960e-02,
+           -6.9824e-02,  6.0547e-01],
+          [ 1.4038e-01,  4.8389e-01, -2.7002e-01,  ...,  2.1408e-02,
+            1.6931e-01,  1.4758e-01],
+          ...,
+          [-1.7529e-01,  3.9575e-01, -1.2128e-01,  ...,  1.0913e-01,
+            1.9958e-01,  2.6562e-01],
+          [-2.5589e-02, -1.7371e-01, -1.2610e-01,  ..., -3.0469e-01,
+           -2.2046e-01, -6.6895e-02],
+          [ 8.1726e-02, -5.2637e-01, -5.1172e-01,  ...,  2.1594e-01,
+           -3.3521e-01, -2.7802e-02]],
+
+         [[-2.7878e-02, -3.1006e-01,  4.6600e-02,  ...,  1.3989e-01,
+            2.1191e-01,  2.5488e-01],
+          [ 2.5513e-01,  2.8711e-01,  7.0740e-02,  ..., -7.9150e-01,
+            5.3894e-02, -4.5459e-01],
+          [ 4.8035e-02,  3.6328e-01, -1.0815e-01,  ...,  5.6348e-01,
+           -5.0537e-01,  7.5317e-02],
+          ...,
+          [-2.9907e-01, -5.0732e-01,  6.7480e-01,  ...,  7.2693e-02,
+            6.3281e-01, -8.3618e-02],
+          [ 8.1934e-01, -2.1838e-01,  5.5078e-01,  ..., -1.5686e-01,
+           -3.5693e-01,  2.9907e-01],
+          [ 2.3840e-01,  5.1074e-01, -4.0796e-01,  ...,  6.6452e-03,
+           -3.9307e-02,  3.0200e-01]],
+
+         [[-1.6223e-01,  7.2693e-02, -5.1514e-01,  ...,  2.1289e-01,
+           -1.4880e-01,  2.5220e-01],
+          [ 3.7036e-01, -1.2732e-01,  2.5391e-01,  ...,  3.2324e-01,
+           -5.5939e-02,  7.5293e-01],
+          [ 3.8330e-01,  9.2676e-01, -2.7173e-01,  ..., -1.5823e-02,
+            1.0553e-01, -5.1221e-01],
+          ...,
+          [-3.7549e-01, -4.5337e-01, -2.2729e-01,  ..., -4.9219e-01,
+           -4.7974e-01, -3.4204e-01],
+          [ 4.5581e-01, -3.8501e-01, -9.6497e-02,  ...,  2.0401e-02,
+           -8.5266e-02,  3.2739e-01],
+          [-2.0386e-01,  2.3010e-01, -5.5713e-01,  ..., -4.5288e-02,
+            5.5725e-02,  1.5552e-01]]],
+
+
+        [[[ 3.2617e-01, -3.8110e-01, -1.6077e-01,  ..., -1.2854e-01,
+           -3.4637e-02, -1.6333e-01],
+          [-2.6343e-01,  4.8877e-01, -1.3025e-01,  ..., -5.8899e-02,
+           -6.9824e-02,  6.0547e-01],
+          [ 1.3931e-02, -3.6224e-02,  7.0648e-03,  ...,  2.1423e-02,
+            1.6931e-01,  1.4758e-01],
+          ...,
+          [-6.7969e-01,  5.9961e-01,  6.5063e-02,  ...,  1.0907e-01,
+            1.9958e-01,  2.6562e-01],
+          [ 8.3008e-02,  3.7158e-01, -3.5596e-01,  ..., -3.0469e-01,
+           -2.2046e-01, -6.6895e-02],
+          [-2.6685e-01, -2.7124e-01, -3.5400e-01,  ...,  2.1594e-01,
+           -3.3521e-01, -2.7802e-02]],
+
+         [[ 1.9629e-01, -8.2959e-01,  1.9055e-01,  ...,  1.3989e-01,
+            2.1191e-01,  2.5488e-01],
+          [ 1.8204e-02,  4.8657e-01,  2.5049e-01,  ..., -7.9150e-01,
+            5.3894e-02, -4.5459e-01],
+          [-1.4563e-01,  3.5938e-01, -1.7224e-01,  ...,  5.6348e-01,
+           -5.0537e-01,  7.5317e-02],
+          ...,
+          [-2.5391e-01, -7.5000e-01,  3.6963e-01,  ...,  7.2815e-02,
+            6.3281e-01, -8.3618e-02],
+          [ 4.0161e-02, -3.5449e-01,  6.6504e-01,  ..., -1.5686e-01,
+           -3.5693e-01,  2.9907e-01],
+          [-3.2788e-01,  3.9404e-01, -6.1182e-01,  ...,  6.6452e-03,
+           -3.9307e-02,  3.0200e-01]],
+
+         [[-1.6223e-01,  7.2693e-02, -5.1514e-01,  ...,  2.1289e-01,
+           -1.4880e-01,  2.5220e-01],
+          [ 3.7036e-01, -1.2732e-01,  2.5391e-01,  ...,  3.2324e-01,
+           -5.5939e-02,  7.5293e-01],
+          [ 3.8330e-01,  9.2676e-01, -2.7173e-01,  ..., -1.5823e-02,
+            1.0553e-01, -5.1221e-01],
+          ...,
+          [-3.7549e-01, -4.5337e-01, -2.2729e-01,  ..., -4.9219e-01,
+           -4.7974e-01, -3.4204e-01],
+          [ 4.5581e-01, -3.8501e-01, -9.6497e-02,  ...,  2.0401e-02,
+           -8.5266e-02,  3.2739e-01],
+          [-2.0386e-01,  2.3010e-01, -5.5713e-01,  ..., -4.5288e-02,
+            5.5725e-02,  1.5552e-01]]]], device='cuda:0', dtype=torch.float16), shape: torch.Size([8192, 3, 32, 128])
+fwd block 0 inner_attn: tensor([[[-0.2444, -0.1046,  0.6743,  ..., -0.1627,  0.1663, -0.4773],
+         [ 0.0967, -0.1666, -0.1818,  ...,  0.5166, -0.1505, -0.0385],
+         [-0.8354, -0.0498,  0.4905,  ...,  0.2156, -0.4543,  0.2725],
+         ...,
+         [ 0.0646,  0.5493, -0.1349,  ..., -0.3267,  0.8691, -0.4685],
+         [ 0.5015, -0.1718,  0.0129,  ...,  0.2419,  1.0381, -0.9927],
+         [-0.2074, -0.1874, -0.0461,  ...,  0.9688, -0.5342,  0.0586]],
+
+        [[-0.0975,  0.2952,  0.5303,  ..., -0.5420,  0.0953, -0.2771],
+         [ 0.3682,  0.1605, -0.2264,  ...,  0.0749, -0.1581, -0.2147],
+         [-0.3630,  0.0486,  0.3223,  ..., -0.2091, -0.3728, -0.4192],
+         ...,
+         [ 0.1113,  0.3291,  0.1971,  ..., -0.2192,  0.1907, -0.2695],
+         [ 0.6147, -0.0324, -0.1085,  ...,  0.3567,  0.6387, -0.9136],
+         [-0.2048,  0.2603, -0.0397,  ...,  0.0516, -0.4700,  0.0045]],
+
+        [[-0.1688,  0.2776,  0.3176,  ..., -0.4653, -0.1013, -0.3977],
+         [-0.1248, -0.0288, -0.0629,  ...,  0.1901, -0.2302, -0.0758],
+         [-0.3474,  0.0880,  0.3350,  ..., -0.1119, -0.4719, -0.4265],
+         ...,
+         [-0.0091,  0.2426,  0.0078,  ..., -0.2003,  0.2305, -0.1870],
+         [ 0.4124, -0.0836,  0.0508,  ...,  0.3162,  0.3188, -0.5635],
+         [-0.0726,  0.1346, -0.0364,  ..., -0.0011, -0.4402,  0.0388]],
+
+        ...,
+
+        [[-0.1622,  0.0727, -0.5151,  ...,  0.2129, -0.1488,  0.2522],
+         [ 0.3704, -0.1273,  0.2539,  ...,  0.3232, -0.0559,  0.7529],
+         [ 0.3833,  0.9268, -0.2717,  ..., -0.0158,  0.1055, -0.5122],
+         ...,
+         [-0.3755, -0.4534, -0.2273,  ..., -0.4922, -0.4797, -0.3420],
+         [ 0.4558, -0.3850, -0.0965,  ...,  0.0204, -0.0853,  0.3274],
+         [-0.2039,  0.2301, -0.5571,  ..., -0.0453,  0.0557,  0.1555]],
+
+        [[-0.1622,  0.0727, -0.5151,  ...,  0.2129, -0.1488,  0.2522],
+         [ 0.3704, -0.1273,  0.2539,  ...,  0.3232, -0.0559,  0.7529],
+         [ 0.3833,  0.9268, -0.2717,  ..., -0.0158,  0.1055, -0.5122],
+         ...,
+         [-0.3755, -0.4534, -0.2273,  ..., -0.4922, -0.4797, -0.3420],
+         [ 0.4558, -0.3850, -0.0965,  ...,  0.0204, -0.0853,  0.3274],
+         [-0.2039,  0.2301, -0.5571,  ..., -0.0453,  0.0557,  0.1555]],
+
+        [[-0.1622,  0.0727, -0.5151,  ...,  0.2129, -0.1488,  0.2522],
+         [ 0.3704, -0.1273,  0.2539,  ...,  0.3232, -0.0559,  0.7529],
+         [ 0.3833,  0.9268, -0.2717,  ..., -0.0158,  0.1055, -0.5122],
+         ...,
+         [-0.3755, -0.4534, -0.2273,  ..., -0.4922, -0.4797, -0.3420],
+         [ 0.4558, -0.3850, -0.0965,  ...,  0.0204, -0.0853,  0.3274],
+         [-0.2039,  0.2301, -0.5571,  ..., -0.0453,  0.0557,  0.1555]]],
+       device='cuda:0', dtype=torch.float16), shape: torch.Size([8192, 32, 128])
+fwd block 0 out_proj: tensor([[-0.0449,  0.2668,  0.0318,  ..., -0.1669,  0.0184,  0.0685],
+        [-0.0646,  0.1694,  0.0032,  ..., -0.0605,  0.0173,  0.0800],
+        [-0.0704,  0.1581,  0.0136,  ..., -0.0740,  0.0375,  0.0423],
+        ...,
+        [-0.0076,  0.1115, -0.1059,  ...,  0.0121, -0.0580,  0.1011],
+        [-0.0076,  0.1115, -0.1059,  ...,  0.0121, -0.0580,  0.1011],
+        [-0.0076,  0.1115, -0.1059,  ...,  0.0121, -0.0580,  0.1011]],
+       device='cuda:0', dtype=torch.float16), shape: torch.Size([8192, 4096])
+fwd block 0 dropout2+norm2: tensor([[-0.4109,  2.5684,  0.3599,  ..., -1.6221,  0.0884,  0.6382],
+        [-0.8530,  2.3789,  0.0201,  ..., -0.7539,  0.3372,  1.1104],
+        [-1.1201,  2.6582,  0.2231,  ..., -1.1309,  0.6074,  0.7192],
+        ...,
+        [-0.1346,  1.1719, -1.0400,  ...,  0.1146, -0.5156,  0.9966],
+        [-0.1346,  1.1719, -1.0400,  ...,  0.1146, -0.5156,  0.9966],
+        [-0.1346,  1.1719, -1.0400,  ...,  0.1146, -0.5156,  0.9966]],
+       device='cuda:0', dtype=torch.float16), shape:torch.Size([8192, 4096])
+fwd block 0 mlp: tensor([[-0.0121, -0.0072, -0.0016,  ..., -0.0024,  0.0053,  0.0030],
+        [-0.0165, -0.0130,  0.0150,  ..., -0.0087, -0.0036, -0.0043],
+        [-0.0029, -0.0028,  0.0016,  ..., -0.0005, -0.0079, -0.0018],
+        ...,
+        [-0.0006, -0.0102, -0.0053,  ..., -0.0108, -0.0120,  0.0058],
+        [-0.0006, -0.0102, -0.0053,  ..., -0.0108, -0.0120,  0.0058],
+        [-0.0006, -0.0102, -0.0053,  ..., -0.0108, -0.0120,  0.0058]],
+       device='cuda:0', dtype=torch.float16), shape:torch.Size([8192, 4096])
+fwd norm: tensor([[-1.7852,  0.8657,  0.0756,  ..., -1.1943, -0.4158,  0.7007],
+        [-2.0840,  0.4084,  0.6230,  ..., -0.8672,  0.0288,  0.6421],
+        [-1.8848,  0.4941,  1.0088,  ..., -0.6123,  0.3828,  0.2512],
+        ...,
+        [ 0.3508,  0.8120,  0.2200,  ...,  1.3623,  1.0596, -0.2998],
+        [ 0.3508,  0.8120,  0.2200,  ...,  1.3623,  1.0596, -0.2998],
+        [ 0.3508,  0.8120,  0.2200,  ...,  1.3623,  1.0596, -0.2998]],
+       device='cuda:0', dtype=torch.float16), shapetorch.Size([8192, 4096])
+fwd head: tensor([[ 0.1843,  0.4094, -0.2091,  ...,  0.0714,  0.5176,  0.2795],
+        [ 0.1895,  0.5454, -0.5142,  ...,  0.1703,  0.4844,  0.3513],
+        [ 0.2000,  0.5674, -0.6069,  ...,  0.2366,  0.4680,  0.2693],
+        ...,
+        [-0.0163, -0.4883,  0.5146,  ..., -0.0645, -0.2766,  0.1595],
+        [-0.0163, -0.4883,  0.5146,  ..., -0.0645, -0.2766,  0.1595],
+        [-0.0163, -0.4883,  0.5146,  ..., -0.0645, -0.2766,  0.1595]],
+       device='cuda:0', dtype=torch.float16), shapetorch.Size([8192, 103168])
+/mnt/petrelfs/share_data/llm_env/miniconda3-py39_4/envs/llm-flash2.0/lib/python3.10/site-packages/torch/utils/checkpoint.py:31: UserWarning: None of the inputs have requires_grad=True. Gradients will be None
+  warnings.warn("None of the inputs have requires_grad=True. Gradients will be None")
+/mnt/petrelfs/share_data/llm_env/miniconda3-py39_4/envs/llm-flash2.0/lib/python3.10/site-packages/torch/utils/checkpoint.py:31: UserWarning: None of the inputs have requires_grad=True. Gradients will be None
+  warnings.warn("None of the inputs have requires_grad=True. Gradients will be None")
+/mnt/petrelfs/share_data/llm_env/miniconda3-py39_4/envs/llm-flash2.0/lib/python3.10/site-packages/torch/utils/checkpoint.py:31: UserWarning: None of the inputs have requires_grad=True. Gradients will be None
+  warnings.warn("None of the inputs have requires_grad=True. Gradients will be None")
+/mnt/petrelfs/share_data/llm_env/miniconda3-py39_4/envs/llm-flash2.0/lib/python3.10/site-packages/torch/utils/checkpoint.py:31: UserWarning: None of the inputs have requires_grad=True. Gradients will be None
+  warnings.warn("None of the inputs have requires_grad=True. Gradients will be None")
+/mnt/petrelfs/share_data/llm_env/miniconda3-py39_4/envs/llm-flash2.0/lib/python3.10/site-packages/torch/utils/checkpoint.py:31: UserWarning: None of the inputs have requires_grad=True. Gradients will be None
+  warnings.warn("None of the inputs have requires_grad=True. Gradients will be None")
+/mnt/petrelfs/share_data/llm_env/miniconda3-py39_4/envs/llm-flash2.0/lib/python3.10/site-packages/torch/utils/checkpoint.py:31: UserWarning: None of the inputs have requires_grad=True. Gradients will be None
+  warnings.warn("None of the inputs have requires_grad=True. Gradients will be None")
+/mnt/petrelfs/share_data/llm_env/miniconda3-py39_4/envs/llm-flash2.0/lib/python3.10/site-packages/torch/utils/checkpoint.py:31: UserWarning: None of the inputs have requires_grad=True. Gradients will be None
+  warnings.warn("None of the inputs have requires_grad=True. Gradients will be None")

qkv_dp_0_step_0_block_0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:544c6a941b414a3a07c85e7016eadbf96d164208a83d02268b24228fb8d9406a
+size 201327451

rotary_emb_dp_0_step_0_block_0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:49ec102881395bcc8b421da19a15e5b29915a453db47c1a0718653687f594f13
+size 201327536