diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1025/config.json b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1025/config.json new file mode 100644 index 0000000000000000000000000000000000000000..18bd58d561964a16b60687f1b04aefdfed0b9b20 --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1025/config.json @@ -0,0 +1,39 @@ +{ + "architectures": [ + "LlamaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "dtype": "bfloat16", + "eos_token_id": [ + 128001, + 128008, + 128009 + ], + "head_dim": 64, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 8192, + "max_position_embeddings": 131072, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 32, + "num_hidden_layers": 16, + "num_key_value_heads": 8, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": { + "factor": 32.0, + "high_freq_factor": 4.0, + "low_freq_factor": 1.0, + "original_max_position_embeddings": 8192, + "rope_type": "llama3" + }, + "rope_theta": 500000.0, + "tie_word_embeddings": true, + "transformers_version": "4.57.1", + "use_cache": false, + "vocab_size": 128256 +} diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1025/generation_config.json b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1025/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..125cc74759bc705ab8a0c48d07d7eac083f56011 --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1025/generation_config.json @@ -0,0 +1,12 @@ +{ + "bos_token_id": 128000, + "do_sample": true, + "eos_token_id": [ + 128001, + 128008, + 128009 + ], + "temperature": 0.6, + "top_p": 0.9, + "transformers_version": "4.57.1" +} diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1025/latest b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1025/latest new file mode 100644 index 0000000000000000000000000000000000000000..892e29ec53a91c9d3867ac21ded15f032e4965cf --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1025/latest @@ -0,0 +1 @@ +global_step1025 \ No newline at end of file diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1025/model.safetensors b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1025/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8fb143e2d40dfcd8bda1fa5de65527683e1d5e08 --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1025/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8a8893a0d101e747951d9f57c04f3fb50d463eb951ca1afaf23685df2bb15205 +size 2996982344 diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1025/scheduler.pt b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1025/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..8b9714385211d6c1a9e2fb12fed5dc3a4d28903d --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1025/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3bfc7021ac48ba0d3d01a50fe2ad034e0b238728f73fd320e110d6f90de4b81a +size 1465 diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1025/trainer_state.json b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1025/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..2cb1220f4bd65837ebd7aee84ed78f96692f92c8 --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1025/trainer_state.json @@ -0,0 +1,748 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.2486288848263254, + "eval_steps": 500, + "global_step": 1025, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.01218769043266301, + "grad_norm": 0.36486390233039856, + "learning_rate": 5e-05, + "loss": 1.2213, + "step": 10 + }, + { + "epoch": 0.02437538086532602, + "grad_norm": 0.2953564524650574, + "learning_rate": 5e-05, + "loss": 1.0223, + "step": 20 + }, + { + "epoch": 0.03656307129798903, + "grad_norm": 0.2766735851764679, + "learning_rate": 5e-05, + "loss": 0.9589, + "step": 30 + }, + { + "epoch": 0.04875076173065204, + "grad_norm": 0.32770997285842896, + "learning_rate": 5e-05, + "loss": 0.9081, + "step": 40 + }, + { + "epoch": 0.06093845216331505, + "grad_norm": 0.3052276372909546, + "learning_rate": 5e-05, + "loss": 0.8752, + "step": 50 + }, + { + "epoch": 0.07312614259597806, + "grad_norm": 0.240146666765213, + "learning_rate": 5e-05, + "loss": 0.873, + "step": 60 + }, + { + "epoch": 0.08531383302864107, + "grad_norm": 0.23872187733650208, + "learning_rate": 5e-05, + "loss": 0.8587, + "step": 70 + }, + { + "epoch": 0.09750152346130408, + "grad_norm": 0.32638487219810486, + "learning_rate": 5e-05, + "loss": 0.8656, + "step": 80 + }, + { + "epoch": 0.10968921389396709, + "grad_norm": 0.25019028782844543, + "learning_rate": 5e-05, + "loss": 0.8685, + "step": 90 + }, + { + "epoch": 0.1218769043266301, + "grad_norm": 0.2507675886154175, + "learning_rate": 5e-05, + "loss": 0.8662, + "step": 100 + }, + { + "epoch": 0.1340645947592931, + "grad_norm": 0.27966755628585815, + "learning_rate": 5e-05, + "loss": 0.8428, + "step": 110 + }, + { + "epoch": 0.14625228519195613, + "grad_norm": 0.2622673213481903, + "learning_rate": 5e-05, + "loss": 0.8416, + "step": 120 + }, + { + "epoch": 0.15843997562461914, + "grad_norm": 0.3003966510295868, + "learning_rate": 5e-05, + "loss": 0.8528, + "step": 130 + }, + { + "epoch": 0.17062766605728213, + "grad_norm": 0.32308319211006165, + "learning_rate": 5e-05, + "loss": 0.8357, + "step": 140 + }, + { + "epoch": 0.18281535648994515, + "grad_norm": 0.27279096841812134, + "learning_rate": 5e-05, + "loss": 0.8432, + "step": 150 + }, + { + "epoch": 0.19500304692260817, + "grad_norm": 0.27976104617118835, + "learning_rate": 5e-05, + "loss": 0.8119, + "step": 160 + }, + { + "epoch": 0.2071907373552712, + "grad_norm": 0.290866881608963, + "learning_rate": 5e-05, + "loss": 0.8096, + "step": 170 + }, + { + "epoch": 0.21937842778793418, + "grad_norm": 0.29430949687957764, + "learning_rate": 5e-05, + "loss": 0.819, + "step": 180 + }, + { + "epoch": 0.2315661182205972, + "grad_norm": 0.27714136242866516, + "learning_rate": 5e-05, + "loss": 0.8187, + "step": 190 + }, + { + "epoch": 0.2437538086532602, + "grad_norm": 0.2997029423713684, + "learning_rate": 5e-05, + "loss": 0.808, + "step": 200 + }, + { + "epoch": 0.25594149908592323, + "grad_norm": 0.2539573609828949, + "learning_rate": 5e-05, + "loss": 0.7943, + "step": 210 + }, + { + "epoch": 0.2681291895185862, + "grad_norm": 0.23294171690940857, + "learning_rate": 5e-05, + "loss": 0.8188, + "step": 220 + }, + { + "epoch": 0.28031687995124926, + "grad_norm": 0.25016292929649353, + "learning_rate": 5e-05, + "loss": 0.8035, + "step": 230 + }, + { + "epoch": 0.29250457038391225, + "grad_norm": 0.37125322222709656, + "learning_rate": 5e-05, + "loss": 0.785, + "step": 240 + }, + { + "epoch": 0.30469226081657524, + "grad_norm": 0.21520452201366425, + "learning_rate": 5e-05, + "loss": 0.8088, + "step": 250 + }, + { + "epoch": 0.3168799512492383, + "grad_norm": 0.20990602672100067, + "learning_rate": 5e-05, + "loss": 0.7813, + "step": 260 + }, + { + "epoch": 0.3290676416819013, + "grad_norm": 0.24193961918354034, + "learning_rate": 5e-05, + "loss": 0.7979, + "step": 270 + }, + { + "epoch": 0.34125533211456427, + "grad_norm": 0.22766050696372986, + "learning_rate": 5e-05, + "loss": 0.785, + "step": 280 + }, + { + "epoch": 0.3534430225472273, + "grad_norm": 0.2662702202796936, + "learning_rate": 5e-05, + "loss": 0.7766, + "step": 290 + }, + { + "epoch": 0.3656307129798903, + "grad_norm": 0.23020495474338531, + "learning_rate": 5e-05, + "loss": 0.7687, + "step": 300 + }, + { + "epoch": 0.3778184034125533, + "grad_norm": 0.21770520508289337, + "learning_rate": 5e-05, + "loss": 0.7815, + "step": 310 + }, + { + "epoch": 0.39000609384521634, + "grad_norm": 0.22677403688430786, + "learning_rate": 5e-05, + "loss": 0.7999, + "step": 320 + }, + { + "epoch": 0.40219378427787933, + "grad_norm": 0.24891860783100128, + "learning_rate": 5e-05, + "loss": 0.7946, + "step": 330 + }, + { + "epoch": 0.4143814747105424, + "grad_norm": 0.2506854832172394, + "learning_rate": 5e-05, + "loss": 0.7889, + "step": 340 + }, + { + "epoch": 0.42656916514320536, + "grad_norm": 0.24388180673122406, + "learning_rate": 5e-05, + "loss": 0.79, + "step": 350 + }, + { + "epoch": 0.43875685557586835, + "grad_norm": 0.22329650819301605, + "learning_rate": 5e-05, + "loss": 0.7632, + "step": 360 + }, + { + "epoch": 0.4509445460085314, + "grad_norm": 0.2240927815437317, + "learning_rate": 5e-05, + "loss": 0.8042, + "step": 370 + }, + { + "epoch": 0.4631322364411944, + "grad_norm": 0.2578203082084656, + "learning_rate": 5e-05, + "loss": 0.7667, + "step": 380 + }, + { + "epoch": 0.4753199268738574, + "grad_norm": 0.25349217653274536, + "learning_rate": 5e-05, + "loss": 0.7911, + "step": 390 + }, + { + "epoch": 0.4875076173065204, + "grad_norm": 0.254446804523468, + "learning_rate": 5e-05, + "loss": 0.7909, + "step": 400 + }, + { + "epoch": 0.4996953077391834, + "grad_norm": 0.24253171682357788, + "learning_rate": 5e-05, + "loss": 0.7896, + "step": 410 + }, + { + "epoch": 0.5118829981718465, + "grad_norm": 0.27901458740234375, + "learning_rate": 5e-05, + "loss": 0.7828, + "step": 420 + }, + { + "epoch": 0.5240706886045094, + "grad_norm": 0.31884434819221497, + "learning_rate": 5e-05, + "loss": 0.7919, + "step": 430 + }, + { + "epoch": 0.5362583790371724, + "grad_norm": 0.22920799255371094, + "learning_rate": 5e-05, + "loss": 0.7777, + "step": 440 + }, + { + "epoch": 0.5484460694698354, + "grad_norm": 0.22606618702411652, + "learning_rate": 5e-05, + "loss": 0.7661, + "step": 450 + }, + { + "epoch": 0.5606337599024985, + "grad_norm": 0.21604052186012268, + "learning_rate": 5e-05, + "loss": 0.769, + "step": 460 + }, + { + "epoch": 0.5728214503351615, + "grad_norm": 0.2296101152896881, + "learning_rate": 5e-05, + "loss": 0.7626, + "step": 470 + }, + { + "epoch": 0.5850091407678245, + "grad_norm": 0.24162150919437408, + "learning_rate": 5e-05, + "loss": 0.7831, + "step": 480 + }, + { + "epoch": 0.5971968312004875, + "grad_norm": 0.22364799678325653, + "learning_rate": 5e-05, + "loss": 0.7737, + "step": 490 + }, + { + "epoch": 0.6093845216331505, + "grad_norm": 0.23953403532505035, + "learning_rate": 5e-05, + "loss": 0.787, + "step": 500 + }, + { + "epoch": 0.6215722120658135, + "grad_norm": 0.22349311411380768, + "learning_rate": 5e-05, + "loss": 0.7763, + "step": 510 + }, + { + "epoch": 0.6337599024984766, + "grad_norm": 0.21996377408504486, + "learning_rate": 5e-05, + "loss": 0.7596, + "step": 520 + }, + { + "epoch": 0.6459475929311396, + "grad_norm": 0.245538130402565, + "learning_rate": 5e-05, + "loss": 0.7785, + "step": 530 + }, + { + "epoch": 0.6581352833638026, + "grad_norm": 0.23442520201206207, + "learning_rate": 5e-05, + "loss": 0.7633, + "step": 540 + }, + { + "epoch": 0.6703229737964655, + "grad_norm": 0.22274106740951538, + "learning_rate": 5e-05, + "loss": 0.7756, + "step": 550 + }, + { + "epoch": 0.6825106642291285, + "grad_norm": 0.21717530488967896, + "learning_rate": 5e-05, + "loss": 0.7726, + "step": 560 + }, + { + "epoch": 0.6946983546617916, + "grad_norm": 0.2465955913066864, + "learning_rate": 5e-05, + "loss": 0.7476, + "step": 570 + }, + { + "epoch": 0.7068860450944546, + "grad_norm": 0.2318575084209442, + "learning_rate": 5e-05, + "loss": 0.7584, + "step": 580 + }, + { + "epoch": 0.7190737355271176, + "grad_norm": 0.25114378333091736, + "learning_rate": 5e-05, + "loss": 0.7625, + "step": 590 + }, + { + "epoch": 0.7312614259597806, + "grad_norm": 0.19145755469799042, + "learning_rate": 5e-05, + "loss": 0.7577, + "step": 600 + }, + { + "epoch": 0.7434491163924436, + "grad_norm": 0.19129066169261932, + "learning_rate": 5e-05, + "loss": 0.7772, + "step": 610 + }, + { + "epoch": 0.7556368068251066, + "grad_norm": 0.2014475166797638, + "learning_rate": 5e-05, + "loss": 0.7601, + "step": 620 + }, + { + "epoch": 0.7678244972577697, + "grad_norm": 0.21480509638786316, + "learning_rate": 5e-05, + "loss": 0.7665, + "step": 630 + }, + { + "epoch": 0.7800121876904327, + "grad_norm": 0.2633228302001953, + "learning_rate": 5e-05, + "loss": 0.7565, + "step": 640 + }, + { + "epoch": 0.7921998781230957, + "grad_norm": 0.24633203446865082, + "learning_rate": 5e-05, + "loss": 0.7602, + "step": 650 + }, + { + "epoch": 0.8043875685557587, + "grad_norm": 0.19770534336566925, + "learning_rate": 5e-05, + "loss": 0.7685, + "step": 660 + }, + { + "epoch": 0.8165752589884216, + "grad_norm": 0.2267930805683136, + "learning_rate": 5e-05, + "loss": 0.7516, + "step": 670 + }, + { + "epoch": 0.8287629494210847, + "grad_norm": 0.238111674785614, + "learning_rate": 5e-05, + "loss": 0.7487, + "step": 680 + }, + { + "epoch": 0.8409506398537477, + "grad_norm": 0.19528904557228088, + "learning_rate": 5e-05, + "loss": 0.7467, + "step": 690 + }, + { + "epoch": 0.8531383302864107, + "grad_norm": 0.21305914223194122, + "learning_rate": 5e-05, + "loss": 0.7527, + "step": 700 + }, + { + "epoch": 0.8653260207190737, + "grad_norm": 0.2205205112695694, + "learning_rate": 5e-05, + "loss": 0.7639, + "step": 710 + }, + { + "epoch": 0.8775137111517367, + "grad_norm": 0.22429127991199493, + "learning_rate": 5e-05, + "loss": 0.7744, + "step": 720 + }, + { + "epoch": 0.8897014015843998, + "grad_norm": 0.23755772411823273, + "learning_rate": 5e-05, + "loss": 0.7446, + "step": 730 + }, + { + "epoch": 0.9018890920170628, + "grad_norm": 0.17592693865299225, + "learning_rate": 5e-05, + "loss": 0.7442, + "step": 740 + }, + { + "epoch": 0.9140767824497258, + "grad_norm": 0.2266898900270462, + "learning_rate": 5e-05, + "loss": 0.7326, + "step": 750 + }, + { + "epoch": 0.9262644728823888, + "grad_norm": 0.23541569709777832, + "learning_rate": 5e-05, + "loss": 0.7338, + "step": 760 + }, + { + "epoch": 0.9384521633150518, + "grad_norm": 0.22233888506889343, + "learning_rate": 5e-05, + "loss": 0.7385, + "step": 770 + }, + { + "epoch": 0.9506398537477148, + "grad_norm": 0.23714767396450043, + "learning_rate": 5e-05, + "loss": 0.7306, + "step": 780 + }, + { + "epoch": 0.9628275441803779, + "grad_norm": 0.21191060543060303, + "learning_rate": 5e-05, + "loss": 0.736, + "step": 790 + }, + { + "epoch": 0.9750152346130408, + "grad_norm": 0.22759106755256653, + "learning_rate": 5e-05, + "loss": 0.7505, + "step": 800 + }, + { + "epoch": 0.9872029250457038, + "grad_norm": 0.19895903766155243, + "learning_rate": 5e-05, + "loss": 0.7536, + "step": 810 + }, + { + "epoch": 0.9993906154783668, + "grad_norm": 0.3061685860157013, + "learning_rate": 5e-05, + "loss": 0.7493, + "step": 820 + }, + { + "epoch": 1.0109689213893966, + "grad_norm": 0.18868015706539154, + "learning_rate": 5e-05, + "loss": 0.7342, + "step": 830 + }, + { + "epoch": 1.0231566118220596, + "grad_norm": 0.1797628402709961, + "learning_rate": 5e-05, + "loss": 0.7192, + "step": 840 + }, + { + "epoch": 1.0353443022547226, + "grad_norm": 0.19511596858501434, + "learning_rate": 5e-05, + "loss": 0.7002, + "step": 850 + }, + { + "epoch": 1.0475319926873858, + "grad_norm": 0.2123185396194458, + "learning_rate": 5e-05, + "loss": 0.7236, + "step": 860 + }, + { + "epoch": 1.0597196831200488, + "grad_norm": 0.2047470211982727, + "learning_rate": 5e-05, + "loss": 0.711, + "step": 870 + }, + { + "epoch": 1.0719073735527118, + "grad_norm": 0.18809868395328522, + "learning_rate": 5e-05, + "loss": 0.7253, + "step": 880 + }, + { + "epoch": 1.0840950639853748, + "grad_norm": 0.2207012176513672, + "learning_rate": 5e-05, + "loss": 0.6971, + "step": 890 + }, + { + "epoch": 1.0962827544180378, + "grad_norm": 0.1912035495042801, + "learning_rate": 5e-05, + "loss": 0.7239, + "step": 900 + }, + { + "epoch": 1.1084704448507008, + "grad_norm": 0.23486360907554626, + "learning_rate": 5e-05, + "loss": 0.7335, + "step": 910 + }, + { + "epoch": 1.1206581352833638, + "grad_norm": 0.22189787030220032, + "learning_rate": 5e-05, + "loss": 0.7149, + "step": 920 + }, + { + "epoch": 1.1328458257160268, + "grad_norm": 0.25818824768066406, + "learning_rate": 5e-05, + "loss": 0.7322, + "step": 930 + }, + { + "epoch": 1.1450335161486898, + "grad_norm": 0.2088775783777237, + "learning_rate": 5e-05, + "loss": 0.7382, + "step": 940 + }, + { + "epoch": 1.1572212065813527, + "grad_norm": 0.19445833563804626, + "learning_rate": 5e-05, + "loss": 0.7223, + "step": 950 + }, + { + "epoch": 1.1694088970140157, + "grad_norm": 0.21489456295967102, + "learning_rate": 5e-05, + "loss": 0.7317, + "step": 960 + }, + { + "epoch": 1.181596587446679, + "grad_norm": 0.24050794541835785, + "learning_rate": 5e-05, + "loss": 0.7275, + "step": 970 + }, + { + "epoch": 1.193784277879342, + "grad_norm": 0.18142102658748627, + "learning_rate": 5e-05, + "loss": 0.7046, + "step": 980 + }, + { + "epoch": 1.205971968312005, + "grad_norm": 0.4781077802181244, + "learning_rate": 5e-05, + "loss": 0.7433, + "step": 990 + }, + { + "epoch": 1.218159658744668, + "grad_norm": 0.20628023147583008, + "learning_rate": 5e-05, + "loss": 0.7403, + "step": 1000 + }, + { + "epoch": 1.230347349177331, + "grad_norm": 0.19117140769958496, + "learning_rate": 5e-05, + "loss": 0.7308, + "step": 1010 + }, + { + "epoch": 1.242535039609994, + "grad_norm": 0.2107745110988617, + "learning_rate": 5e-05, + "loss": 0.7201, + "step": 1020 + } + ], + "logging_steps": 10, + "max_steps": 2463, + "num_input_tokens_seen": 0, + "num_train_epochs": 3, + "save_steps": 205, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 4.207123313442423e+18, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1025/training_args.bin b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1025/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..23ae0ec8a4226be5457f61c137abab2b2c10a0cf --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1025/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56896ea21940e33d7e2934267dff2a5b4a4a80195a0f165245f6617a0f88b90b +size 7377 diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1025/zero_to_fp32.py b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1025/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..5995d6e6f04e43b989587aa9022a3aef0c66d694 --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1025/zero_to_fp32.py @@ -0,0 +1,760 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: +# python zero_to_fp32.py . output_dir/ +# or +# python zero_to_fp32.py . output_dir/ --safe_serialization + +import argparse +import torch +import glob +import math +import os +import re +import gc +import json +import numpy as np +from tqdm import tqdm +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device, weights_only=False) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + total_files = len(files) + state_dicts = [] + for f in tqdm(files, desc='Loading checkpoint shards'): + state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + model_files = get_model_state_files(ds_checkpoint_dir) + + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +class GatheredTensor: + """ + A pseudo tensor that collects partitioned weights. + It is more memory efficient when there are multiple groups. + """ + + def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape): + self.flat_groups = flat_groups + self.flat_groups_offset = flat_groups_offset + self.offset = offset + self.partitioned_numel = partitioned_numel + self.shape = shape + self.dtype = self.flat_groups[0][0].dtype + + def contiguous(self): + """ + Merge partitioned weights from flat_groups into a single tensor. + """ + end_idx = self.offset + self.partitioned_numel + world_size = len(self.flat_groups) + pad_flat_param_chunks = [] + + for rank_i in range(world_size): + # for each rank, we need to collect weights from related group/groups + flat_groups_at_rank_i = self.flat_groups[rank_i] + start_group_id = None + end_group_id = None + for group_id in range(len(self.flat_groups_offset)): + if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]: + start_group_id = group_id + if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]: + end_group_id = group_id + break + # collect weights from related group/groups + for group_id in range(start_group_id, end_group_id + 1): + flat_tensor = flat_groups_at_rank_i[group_id] + start_offset = self.offset - self.flat_groups_offset[group_id] + end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id] + pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset]) + + # collect weights from all ranks + pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0) + param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous() + return param + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size + + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]])) + for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'): + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # memory efficient tensor + tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape) + state_dict[name] = tensor + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def to_torch_tensor(state_dict, return_empty_tensor=False): + """ + Convert state_dict of GatheredTensor to torch tensor + """ + torch_state_dict = {} + converted_tensors = {} + for name, tensor in state_dict.items(): + tensor_id = id(tensor) + if tensor_id in converted_tensors: # shared tensors + shared_tensor = torch_state_dict[converted_tensors[tensor_id]] + torch_state_dict[name] = shared_tensor + else: + converted_tensors[tensor_id] = name + if return_empty_tensor: + torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype) + else: + torch_state_dict[name] = tensor.contiguous() + return torch_state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag=None, + exclude_frozen_parameters=False, + lazy_mode=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient. + Convert the pesduo tensor to torch tensor by ``.contiguous()`` + + Returns: + - pytorch ``state_dict`` + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + Note: the above usage may not work if your application doesn't have sufficient free CPU memory. + You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. Or you can load state_dict in lazy mode :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu + for name, lazy_tensor in state_dict.item(): + tensor = lazy_tensor.contiguous() # to cpu + print(name, tensor) + # del tensor to release memory if it no longer in use + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + if lazy_mode: + return state_dict + else: + return to_torch_tensor(state_dict) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, + output_dir, + max_shard_size="5GB", + safe_serialization=False, + tag=None, + exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_dir``: directory to the pytorch fp32 state_dict output files + - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB + - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + + # Dependency pre-check + if safe_serialization: + try: + from safetensors.torch import save_file + except ImportError: + print('If you want to use `safe_serialization`, please `pip install safetensors`') + raise + if max_shard_size is not None: + try: + from huggingface_hub import split_torch_state_dict_into_shards + except ImportError: + print('If you want to use `max_shard_size`, please `pip install huggingface_hub`') + raise + + # Convert zero checkpoint to state_dict + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag, + exclude_frozen_parameters, + lazy_mode=True) + + # Shard the model if it is too big. + weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin" + if max_shard_size is not None: + filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors") + # an memory-efficient approach for sharding + empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True) + state_dict_split = split_torch_state_dict_into_shards(empty_state_dict, + filename_pattern=filename_pattern, + max_shard_size=max_shard_size) + else: + from collections import namedtuple + StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"]) + state_dict_split = StateDictSplit(is_sharded=False, + filename_to_tensors={weights_name: list(state_dict.keys())}) + + # Save the model by shard + os.makedirs(output_dir, exist_ok=True) + filename_to_tensors = state_dict_split.filename_to_tensors.items() + for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"): + shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors} + shard_state_dict = to_torch_tensor(shard_state_dict) + output_path = os.path.join(output_dir, shard_file) + if safe_serialization: + save_file(shard_state_dict, output_path, metadata={"format": "pt"}) + else: + torch.save(shard_state_dict, output_path) + # release the memory of current shard + for tensor_name in list(shard_state_dict.keys()): + del state_dict[tensor_name] + del shard_state_dict[tensor_name] + del shard_state_dict + gc.collect() + + # Save index if sharded + if state_dict_split.is_sharded: + index = { + "metadata": state_dict_split.metadata, + "weight_map": state_dict_split.tensor_to_filename, + } + save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json" + save_index_file = os.path.join(output_dir, save_index_file) + with open(save_index_file, "w", encoding="utf-8") as f: + content = json.dumps(index, indent=2, sort_keys=True) + "\n" + f.write(content) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info("Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info("Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument("output_dir", + type=str, + help="directory to the pytorch fp32 state_dict output files" + "(e.g. path/checkpoint-12-output/)") + parser.add_argument( + "--max_shard_size", + type=str, + default="5GB", + help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size" + "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`" + "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances" + "without CPU OOM issues.") + parser.add_argument( + "--safe_serialization", + default=False, + action='store_true', + help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_dir, + max_shard_size=args.max_shard_size, + safe_serialization=args.safe_serialization, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1230/config.json b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1230/config.json new file mode 100644 index 0000000000000000000000000000000000000000..18bd58d561964a16b60687f1b04aefdfed0b9b20 --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1230/config.json @@ -0,0 +1,39 @@ +{ + "architectures": [ + "LlamaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "dtype": "bfloat16", + "eos_token_id": [ + 128001, + 128008, + 128009 + ], + "head_dim": 64, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 8192, + "max_position_embeddings": 131072, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 32, + "num_hidden_layers": 16, + "num_key_value_heads": 8, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": { + "factor": 32.0, + "high_freq_factor": 4.0, + "low_freq_factor": 1.0, + "original_max_position_embeddings": 8192, + "rope_type": "llama3" + }, + "rope_theta": 500000.0, + "tie_word_embeddings": true, + "transformers_version": "4.57.1", + "use_cache": false, + "vocab_size": 128256 +} diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1230/generation_config.json b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1230/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..125cc74759bc705ab8a0c48d07d7eac083f56011 --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1230/generation_config.json @@ -0,0 +1,12 @@ +{ + "bos_token_id": 128000, + "do_sample": true, + "eos_token_id": [ + 128001, + 128008, + 128009 + ], + "temperature": 0.6, + "top_p": 0.9, + "transformers_version": "4.57.1" +} diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1230/latest b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1230/latest new file mode 100644 index 0000000000000000000000000000000000000000..236b0c2c3ac1f7ea0dbf331b74b124d48ca715e0 --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1230/latest @@ -0,0 +1 @@ +global_step1230 \ No newline at end of file diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1230/model.safetensors b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1230/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..b6c1296a1931e7000bf4e102c8fd863877de101c --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1230/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a48d40b419c750e901d492309f615ed524a2647afd9b8afb05d5516ce0df53b6 +size 2996982344 diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1230/scheduler.pt b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1230/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..8728c9eea356062610f862bacfeaad7882a3b614 --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1230/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7e8477210f775a7c1c3ea7cbcf76343d146771b5b05309a94b7d69ce3332955 +size 1465 diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1230/trainer_state.json b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1230/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..9e9f0cbbb805b23dc0af7a3f1a814600c1f2f4a4 --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1230/trainer_state.json @@ -0,0 +1,895 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.4984765386959171, + "eval_steps": 500, + "global_step": 1230, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.01218769043266301, + "grad_norm": 0.36486390233039856, + "learning_rate": 5e-05, + "loss": 1.2213, + "step": 10 + }, + { + "epoch": 0.02437538086532602, + "grad_norm": 0.2953564524650574, + "learning_rate": 5e-05, + "loss": 1.0223, + "step": 20 + }, + { + "epoch": 0.03656307129798903, + "grad_norm": 0.2766735851764679, + "learning_rate": 5e-05, + "loss": 0.9589, + "step": 30 + }, + { + "epoch": 0.04875076173065204, + "grad_norm": 0.32770997285842896, + "learning_rate": 5e-05, + "loss": 0.9081, + "step": 40 + }, + { + "epoch": 0.06093845216331505, + "grad_norm": 0.3052276372909546, + "learning_rate": 5e-05, + "loss": 0.8752, + "step": 50 + }, + { + "epoch": 0.07312614259597806, + "grad_norm": 0.240146666765213, + "learning_rate": 5e-05, + "loss": 0.873, + "step": 60 + }, + { + "epoch": 0.08531383302864107, + "grad_norm": 0.23872187733650208, + "learning_rate": 5e-05, + "loss": 0.8587, + "step": 70 + }, + { + "epoch": 0.09750152346130408, + "grad_norm": 0.32638487219810486, + "learning_rate": 5e-05, + "loss": 0.8656, + "step": 80 + }, + { + "epoch": 0.10968921389396709, + "grad_norm": 0.25019028782844543, + "learning_rate": 5e-05, + "loss": 0.8685, + "step": 90 + }, + { + "epoch": 0.1218769043266301, + "grad_norm": 0.2507675886154175, + "learning_rate": 5e-05, + "loss": 0.8662, + "step": 100 + }, + { + "epoch": 0.1340645947592931, + "grad_norm": 0.27966755628585815, + "learning_rate": 5e-05, + "loss": 0.8428, + "step": 110 + }, + { + "epoch": 0.14625228519195613, + "grad_norm": 0.2622673213481903, + "learning_rate": 5e-05, + "loss": 0.8416, + "step": 120 + }, + { + "epoch": 0.15843997562461914, + "grad_norm": 0.3003966510295868, + "learning_rate": 5e-05, + "loss": 0.8528, + "step": 130 + }, + { + "epoch": 0.17062766605728213, + "grad_norm": 0.32308319211006165, + "learning_rate": 5e-05, + "loss": 0.8357, + "step": 140 + }, + { + "epoch": 0.18281535648994515, + "grad_norm": 0.27279096841812134, + "learning_rate": 5e-05, + "loss": 0.8432, + "step": 150 + }, + { + "epoch": 0.19500304692260817, + "grad_norm": 0.27976104617118835, + "learning_rate": 5e-05, + "loss": 0.8119, + "step": 160 + }, + { + "epoch": 0.2071907373552712, + "grad_norm": 0.290866881608963, + "learning_rate": 5e-05, + "loss": 0.8096, + "step": 170 + }, + { + "epoch": 0.21937842778793418, + "grad_norm": 0.29430949687957764, + "learning_rate": 5e-05, + "loss": 0.819, + "step": 180 + }, + { + "epoch": 0.2315661182205972, + "grad_norm": 0.27714136242866516, + "learning_rate": 5e-05, + "loss": 0.8187, + "step": 190 + }, + { + "epoch": 0.2437538086532602, + "grad_norm": 0.2997029423713684, + "learning_rate": 5e-05, + "loss": 0.808, + "step": 200 + }, + { + "epoch": 0.25594149908592323, + "grad_norm": 0.2539573609828949, + "learning_rate": 5e-05, + "loss": 0.7943, + "step": 210 + }, + { + "epoch": 0.2681291895185862, + "grad_norm": 0.23294171690940857, + "learning_rate": 5e-05, + "loss": 0.8188, + "step": 220 + }, + { + "epoch": 0.28031687995124926, + "grad_norm": 0.25016292929649353, + "learning_rate": 5e-05, + "loss": 0.8035, + "step": 230 + }, + { + "epoch": 0.29250457038391225, + "grad_norm": 0.37125322222709656, + "learning_rate": 5e-05, + "loss": 0.785, + "step": 240 + }, + { + "epoch": 0.30469226081657524, + "grad_norm": 0.21520452201366425, + "learning_rate": 5e-05, + "loss": 0.8088, + "step": 250 + }, + { + "epoch": 0.3168799512492383, + "grad_norm": 0.20990602672100067, + "learning_rate": 5e-05, + "loss": 0.7813, + "step": 260 + }, + { + "epoch": 0.3290676416819013, + "grad_norm": 0.24193961918354034, + "learning_rate": 5e-05, + "loss": 0.7979, + "step": 270 + }, + { + "epoch": 0.34125533211456427, + "grad_norm": 0.22766050696372986, + "learning_rate": 5e-05, + "loss": 0.785, + "step": 280 + }, + { + "epoch": 0.3534430225472273, + "grad_norm": 0.2662702202796936, + "learning_rate": 5e-05, + "loss": 0.7766, + "step": 290 + }, + { + "epoch": 0.3656307129798903, + "grad_norm": 0.23020495474338531, + "learning_rate": 5e-05, + "loss": 0.7687, + "step": 300 + }, + { + "epoch": 0.3778184034125533, + "grad_norm": 0.21770520508289337, + "learning_rate": 5e-05, + "loss": 0.7815, + "step": 310 + }, + { + "epoch": 0.39000609384521634, + "grad_norm": 0.22677403688430786, + "learning_rate": 5e-05, + "loss": 0.7999, + "step": 320 + }, + { + "epoch": 0.40219378427787933, + "grad_norm": 0.24891860783100128, + "learning_rate": 5e-05, + "loss": 0.7946, + "step": 330 + }, + { + "epoch": 0.4143814747105424, + "grad_norm": 0.2506854832172394, + "learning_rate": 5e-05, + "loss": 0.7889, + "step": 340 + }, + { + "epoch": 0.42656916514320536, + "grad_norm": 0.24388180673122406, + "learning_rate": 5e-05, + "loss": 0.79, + "step": 350 + }, + { + "epoch": 0.43875685557586835, + "grad_norm": 0.22329650819301605, + "learning_rate": 5e-05, + "loss": 0.7632, + "step": 360 + }, + { + "epoch": 0.4509445460085314, + "grad_norm": 0.2240927815437317, + "learning_rate": 5e-05, + "loss": 0.8042, + "step": 370 + }, + { + "epoch": 0.4631322364411944, + "grad_norm": 0.2578203082084656, + "learning_rate": 5e-05, + "loss": 0.7667, + "step": 380 + }, + { + "epoch": 0.4753199268738574, + "grad_norm": 0.25349217653274536, + "learning_rate": 5e-05, + "loss": 0.7911, + "step": 390 + }, + { + "epoch": 0.4875076173065204, + "grad_norm": 0.254446804523468, + "learning_rate": 5e-05, + "loss": 0.7909, + "step": 400 + }, + { + "epoch": 0.4996953077391834, + "grad_norm": 0.24253171682357788, + "learning_rate": 5e-05, + "loss": 0.7896, + "step": 410 + }, + { + "epoch": 0.5118829981718465, + "grad_norm": 0.27901458740234375, + "learning_rate": 5e-05, + "loss": 0.7828, + "step": 420 + }, + { + "epoch": 0.5240706886045094, + "grad_norm": 0.31884434819221497, + "learning_rate": 5e-05, + "loss": 0.7919, + "step": 430 + }, + { + "epoch": 0.5362583790371724, + "grad_norm": 0.22920799255371094, + "learning_rate": 5e-05, + "loss": 0.7777, + "step": 440 + }, + { + "epoch": 0.5484460694698354, + "grad_norm": 0.22606618702411652, + "learning_rate": 5e-05, + "loss": 0.7661, + "step": 450 + }, + { + "epoch": 0.5606337599024985, + "grad_norm": 0.21604052186012268, + "learning_rate": 5e-05, + "loss": 0.769, + "step": 460 + }, + { + "epoch": 0.5728214503351615, + "grad_norm": 0.2296101152896881, + "learning_rate": 5e-05, + "loss": 0.7626, + "step": 470 + }, + { + "epoch": 0.5850091407678245, + "grad_norm": 0.24162150919437408, + "learning_rate": 5e-05, + "loss": 0.7831, + "step": 480 + }, + { + "epoch": 0.5971968312004875, + "grad_norm": 0.22364799678325653, + "learning_rate": 5e-05, + "loss": 0.7737, + "step": 490 + }, + { + "epoch": 0.6093845216331505, + "grad_norm": 0.23953403532505035, + "learning_rate": 5e-05, + "loss": 0.787, + "step": 500 + }, + { + "epoch": 0.6215722120658135, + "grad_norm": 0.22349311411380768, + "learning_rate": 5e-05, + "loss": 0.7763, + "step": 510 + }, + { + "epoch": 0.6337599024984766, + "grad_norm": 0.21996377408504486, + "learning_rate": 5e-05, + "loss": 0.7596, + "step": 520 + }, + { + "epoch": 0.6459475929311396, + "grad_norm": 0.245538130402565, + "learning_rate": 5e-05, + "loss": 0.7785, + "step": 530 + }, + { + "epoch": 0.6581352833638026, + "grad_norm": 0.23442520201206207, + "learning_rate": 5e-05, + "loss": 0.7633, + "step": 540 + }, + { + "epoch": 0.6703229737964655, + "grad_norm": 0.22274106740951538, + "learning_rate": 5e-05, + "loss": 0.7756, + "step": 550 + }, + { + "epoch": 0.6825106642291285, + "grad_norm": 0.21717530488967896, + "learning_rate": 5e-05, + "loss": 0.7726, + "step": 560 + }, + { + "epoch": 0.6946983546617916, + "grad_norm": 0.2465955913066864, + "learning_rate": 5e-05, + "loss": 0.7476, + "step": 570 + }, + { + "epoch": 0.7068860450944546, + "grad_norm": 0.2318575084209442, + "learning_rate": 5e-05, + "loss": 0.7584, + "step": 580 + }, + { + "epoch": 0.7190737355271176, + "grad_norm": 0.25114378333091736, + "learning_rate": 5e-05, + "loss": 0.7625, + "step": 590 + }, + { + "epoch": 0.7312614259597806, + "grad_norm": 0.19145755469799042, + "learning_rate": 5e-05, + "loss": 0.7577, + "step": 600 + }, + { + "epoch": 0.7434491163924436, + "grad_norm": 0.19129066169261932, + "learning_rate": 5e-05, + "loss": 0.7772, + "step": 610 + }, + { + "epoch": 0.7556368068251066, + "grad_norm": 0.2014475166797638, + "learning_rate": 5e-05, + "loss": 0.7601, + "step": 620 + }, + { + "epoch": 0.7678244972577697, + "grad_norm": 0.21480509638786316, + "learning_rate": 5e-05, + "loss": 0.7665, + "step": 630 + }, + { + "epoch": 0.7800121876904327, + "grad_norm": 0.2633228302001953, + "learning_rate": 5e-05, + "loss": 0.7565, + "step": 640 + }, + { + "epoch": 0.7921998781230957, + "grad_norm": 0.24633203446865082, + "learning_rate": 5e-05, + "loss": 0.7602, + "step": 650 + }, + { + "epoch": 0.8043875685557587, + "grad_norm": 0.19770534336566925, + "learning_rate": 5e-05, + "loss": 0.7685, + "step": 660 + }, + { + "epoch": 0.8165752589884216, + "grad_norm": 0.2267930805683136, + "learning_rate": 5e-05, + "loss": 0.7516, + "step": 670 + }, + { + "epoch": 0.8287629494210847, + "grad_norm": 0.238111674785614, + "learning_rate": 5e-05, + "loss": 0.7487, + "step": 680 + }, + { + "epoch": 0.8409506398537477, + "grad_norm": 0.19528904557228088, + "learning_rate": 5e-05, + "loss": 0.7467, + "step": 690 + }, + { + "epoch": 0.8531383302864107, + "grad_norm": 0.21305914223194122, + "learning_rate": 5e-05, + "loss": 0.7527, + "step": 700 + }, + { + "epoch": 0.8653260207190737, + "grad_norm": 0.2205205112695694, + "learning_rate": 5e-05, + "loss": 0.7639, + "step": 710 + }, + { + "epoch": 0.8775137111517367, + "grad_norm": 0.22429127991199493, + "learning_rate": 5e-05, + "loss": 0.7744, + "step": 720 + }, + { + "epoch": 0.8897014015843998, + "grad_norm": 0.23755772411823273, + "learning_rate": 5e-05, + "loss": 0.7446, + "step": 730 + }, + { + "epoch": 0.9018890920170628, + "grad_norm": 0.17592693865299225, + "learning_rate": 5e-05, + "loss": 0.7442, + "step": 740 + }, + { + "epoch": 0.9140767824497258, + "grad_norm": 0.2266898900270462, + "learning_rate": 5e-05, + "loss": 0.7326, + "step": 750 + }, + { + "epoch": 0.9262644728823888, + "grad_norm": 0.23541569709777832, + "learning_rate": 5e-05, + "loss": 0.7338, + "step": 760 + }, + { + "epoch": 0.9384521633150518, + "grad_norm": 0.22233888506889343, + "learning_rate": 5e-05, + "loss": 0.7385, + "step": 770 + }, + { + "epoch": 0.9506398537477148, + "grad_norm": 0.23714767396450043, + "learning_rate": 5e-05, + "loss": 0.7306, + "step": 780 + }, + { + "epoch": 0.9628275441803779, + "grad_norm": 0.21191060543060303, + "learning_rate": 5e-05, + "loss": 0.736, + "step": 790 + }, + { + "epoch": 0.9750152346130408, + "grad_norm": 0.22759106755256653, + "learning_rate": 5e-05, + "loss": 0.7505, + "step": 800 + }, + { + "epoch": 0.9872029250457038, + "grad_norm": 0.19895903766155243, + "learning_rate": 5e-05, + "loss": 0.7536, + "step": 810 + }, + { + "epoch": 0.9993906154783668, + "grad_norm": 0.3061685860157013, + "learning_rate": 5e-05, + "loss": 0.7493, + "step": 820 + }, + { + "epoch": 1.0109689213893966, + "grad_norm": 0.18868015706539154, + "learning_rate": 5e-05, + "loss": 0.7342, + "step": 830 + }, + { + "epoch": 1.0231566118220596, + "grad_norm": 0.1797628402709961, + "learning_rate": 5e-05, + "loss": 0.7192, + "step": 840 + }, + { + "epoch": 1.0353443022547226, + "grad_norm": 0.19511596858501434, + "learning_rate": 5e-05, + "loss": 0.7002, + "step": 850 + }, + { + "epoch": 1.0475319926873858, + "grad_norm": 0.2123185396194458, + "learning_rate": 5e-05, + "loss": 0.7236, + "step": 860 + }, + { + "epoch": 1.0597196831200488, + "grad_norm": 0.2047470211982727, + "learning_rate": 5e-05, + "loss": 0.711, + "step": 870 + }, + { + "epoch": 1.0719073735527118, + "grad_norm": 0.18809868395328522, + "learning_rate": 5e-05, + "loss": 0.7253, + "step": 880 + }, + { + "epoch": 1.0840950639853748, + "grad_norm": 0.2207012176513672, + "learning_rate": 5e-05, + "loss": 0.6971, + "step": 890 + }, + { + "epoch": 1.0962827544180378, + "grad_norm": 0.1912035495042801, + "learning_rate": 5e-05, + "loss": 0.7239, + "step": 900 + }, + { + "epoch": 1.1084704448507008, + "grad_norm": 0.23486360907554626, + "learning_rate": 5e-05, + "loss": 0.7335, + "step": 910 + }, + { + "epoch": 1.1206581352833638, + "grad_norm": 0.22189787030220032, + "learning_rate": 5e-05, + "loss": 0.7149, + "step": 920 + }, + { + "epoch": 1.1328458257160268, + "grad_norm": 0.25818824768066406, + "learning_rate": 5e-05, + "loss": 0.7322, + "step": 930 + }, + { + "epoch": 1.1450335161486898, + "grad_norm": 0.2088775783777237, + "learning_rate": 5e-05, + "loss": 0.7382, + "step": 940 + }, + { + "epoch": 1.1572212065813527, + "grad_norm": 0.19445833563804626, + "learning_rate": 5e-05, + "loss": 0.7223, + "step": 950 + }, + { + "epoch": 1.1694088970140157, + "grad_norm": 0.21489456295967102, + "learning_rate": 5e-05, + "loss": 0.7317, + "step": 960 + }, + { + "epoch": 1.181596587446679, + "grad_norm": 0.24050794541835785, + "learning_rate": 5e-05, + "loss": 0.7275, + "step": 970 + }, + { + "epoch": 1.193784277879342, + "grad_norm": 0.18142102658748627, + "learning_rate": 5e-05, + "loss": 0.7046, + "step": 980 + }, + { + "epoch": 1.205971968312005, + "grad_norm": 0.4781077802181244, + "learning_rate": 5e-05, + "loss": 0.7433, + "step": 990 + }, + { + "epoch": 1.218159658744668, + "grad_norm": 0.20628023147583008, + "learning_rate": 5e-05, + "loss": 0.7403, + "step": 1000 + }, + { + "epoch": 1.230347349177331, + "grad_norm": 0.19117140769958496, + "learning_rate": 5e-05, + "loss": 0.7308, + "step": 1010 + }, + { + "epoch": 1.242535039609994, + "grad_norm": 0.2107745110988617, + "learning_rate": 5e-05, + "loss": 0.7201, + "step": 1020 + }, + { + "epoch": 1.2547227300426569, + "grad_norm": 0.20551154017448425, + "learning_rate": 5e-05, + "loss": 0.7038, + "step": 1030 + }, + { + "epoch": 1.2669104204753199, + "grad_norm": 0.26577022671699524, + "learning_rate": 5e-05, + "loss": 0.7096, + "step": 1040 + }, + { + "epoch": 1.2790981109079829, + "grad_norm": 0.22094209492206573, + "learning_rate": 5e-05, + "loss": 0.718, + "step": 1050 + }, + { + "epoch": 1.291285801340646, + "grad_norm": 0.18224076926708221, + "learning_rate": 5e-05, + "loss": 0.7261, + "step": 1060 + }, + { + "epoch": 1.303473491773309, + "grad_norm": 0.226251482963562, + "learning_rate": 5e-05, + "loss": 0.7372, + "step": 1070 + }, + { + "epoch": 1.315661182205972, + "grad_norm": 0.20186461508274078, + "learning_rate": 5e-05, + "loss": 0.7074, + "step": 1080 + }, + { + "epoch": 1.327848872638635, + "grad_norm": 0.22229762375354767, + "learning_rate": 5e-05, + "loss": 0.7069, + "step": 1090 + }, + { + "epoch": 1.340036563071298, + "grad_norm": 0.1776452213525772, + "learning_rate": 5e-05, + "loss": 0.714, + "step": 1100 + }, + { + "epoch": 1.352224253503961, + "grad_norm": 0.20680229365825653, + "learning_rate": 5e-05, + "loss": 0.7066, + "step": 1110 + }, + { + "epoch": 1.364411943936624, + "grad_norm": 0.21842031180858612, + "learning_rate": 5e-05, + "loss": 0.7281, + "step": 1120 + }, + { + "epoch": 1.376599634369287, + "grad_norm": 0.28037774562835693, + "learning_rate": 5e-05, + "loss": 0.7149, + "step": 1130 + }, + { + "epoch": 1.38878732480195, + "grad_norm": 0.20962920784950256, + "learning_rate": 5e-05, + "loss": 0.6951, + "step": 1140 + }, + { + "epoch": 1.400975015234613, + "grad_norm": 0.18751871585845947, + "learning_rate": 5e-05, + "loss": 0.7101, + "step": 1150 + }, + { + "epoch": 1.413162705667276, + "grad_norm": 0.2722964584827423, + "learning_rate": 5e-05, + "loss": 0.7205, + "step": 1160 + }, + { + "epoch": 1.425350396099939, + "grad_norm": 0.20932041108608246, + "learning_rate": 5e-05, + "loss": 0.7082, + "step": 1170 + }, + { + "epoch": 1.437538086532602, + "grad_norm": 0.17836257815361023, + "learning_rate": 5e-05, + "loss": 0.7232, + "step": 1180 + }, + { + "epoch": 1.449725776965265, + "grad_norm": 0.3180703818798065, + "learning_rate": 5e-05, + "loss": 0.7245, + "step": 1190 + }, + { + "epoch": 1.4619134673979282, + "grad_norm": 0.201424703001976, + "learning_rate": 5e-05, + "loss": 0.7232, + "step": 1200 + }, + { + "epoch": 1.4741011578305911, + "grad_norm": 0.20228999853134155, + "learning_rate": 5e-05, + "loss": 0.7169, + "step": 1210 + }, + { + "epoch": 1.4862888482632541, + "grad_norm": 0.17478394508361816, + "learning_rate": 5e-05, + "loss": 0.704, + "step": 1220 + }, + { + "epoch": 1.4984765386959171, + "grad_norm": 0.20061351358890533, + "learning_rate": 5e-05, + "loss": 0.6792, + "step": 1230 + } + ], + "logging_steps": 10, + "max_steps": 2463, + "num_input_tokens_seen": 0, + "num_train_epochs": 3, + "save_steps": 205, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 5.045988908773409e+18, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1230/training_args.bin b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1230/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..23ae0ec8a4226be5457f61c137abab2b2c10a0cf --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1230/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56896ea21940e33d7e2934267dff2a5b4a4a80195a0f165245f6617a0f88b90b +size 7377 diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1230/zero_to_fp32.py b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1230/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..5995d6e6f04e43b989587aa9022a3aef0c66d694 --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1230/zero_to_fp32.py @@ -0,0 +1,760 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: +# python zero_to_fp32.py . output_dir/ +# or +# python zero_to_fp32.py . output_dir/ --safe_serialization + +import argparse +import torch +import glob +import math +import os +import re +import gc +import json +import numpy as np +from tqdm import tqdm +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device, weights_only=False) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + total_files = len(files) + state_dicts = [] + for f in tqdm(files, desc='Loading checkpoint shards'): + state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + model_files = get_model_state_files(ds_checkpoint_dir) + + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +class GatheredTensor: + """ + A pseudo tensor that collects partitioned weights. + It is more memory efficient when there are multiple groups. + """ + + def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape): + self.flat_groups = flat_groups + self.flat_groups_offset = flat_groups_offset + self.offset = offset + self.partitioned_numel = partitioned_numel + self.shape = shape + self.dtype = self.flat_groups[0][0].dtype + + def contiguous(self): + """ + Merge partitioned weights from flat_groups into a single tensor. + """ + end_idx = self.offset + self.partitioned_numel + world_size = len(self.flat_groups) + pad_flat_param_chunks = [] + + for rank_i in range(world_size): + # for each rank, we need to collect weights from related group/groups + flat_groups_at_rank_i = self.flat_groups[rank_i] + start_group_id = None + end_group_id = None + for group_id in range(len(self.flat_groups_offset)): + if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]: + start_group_id = group_id + if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]: + end_group_id = group_id + break + # collect weights from related group/groups + for group_id in range(start_group_id, end_group_id + 1): + flat_tensor = flat_groups_at_rank_i[group_id] + start_offset = self.offset - self.flat_groups_offset[group_id] + end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id] + pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset]) + + # collect weights from all ranks + pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0) + param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous() + return param + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size + + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]])) + for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'): + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # memory efficient tensor + tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape) + state_dict[name] = tensor + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def to_torch_tensor(state_dict, return_empty_tensor=False): + """ + Convert state_dict of GatheredTensor to torch tensor + """ + torch_state_dict = {} + converted_tensors = {} + for name, tensor in state_dict.items(): + tensor_id = id(tensor) + if tensor_id in converted_tensors: # shared tensors + shared_tensor = torch_state_dict[converted_tensors[tensor_id]] + torch_state_dict[name] = shared_tensor + else: + converted_tensors[tensor_id] = name + if return_empty_tensor: + torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype) + else: + torch_state_dict[name] = tensor.contiguous() + return torch_state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag=None, + exclude_frozen_parameters=False, + lazy_mode=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient. + Convert the pesduo tensor to torch tensor by ``.contiguous()`` + + Returns: + - pytorch ``state_dict`` + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + Note: the above usage may not work if your application doesn't have sufficient free CPU memory. + You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. Or you can load state_dict in lazy mode :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu + for name, lazy_tensor in state_dict.item(): + tensor = lazy_tensor.contiguous() # to cpu + print(name, tensor) + # del tensor to release memory if it no longer in use + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + if lazy_mode: + return state_dict + else: + return to_torch_tensor(state_dict) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, + output_dir, + max_shard_size="5GB", + safe_serialization=False, + tag=None, + exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_dir``: directory to the pytorch fp32 state_dict output files + - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB + - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + + # Dependency pre-check + if safe_serialization: + try: + from safetensors.torch import save_file + except ImportError: + print('If you want to use `safe_serialization`, please `pip install safetensors`') + raise + if max_shard_size is not None: + try: + from huggingface_hub import split_torch_state_dict_into_shards + except ImportError: + print('If you want to use `max_shard_size`, please `pip install huggingface_hub`') + raise + + # Convert zero checkpoint to state_dict + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag, + exclude_frozen_parameters, + lazy_mode=True) + + # Shard the model if it is too big. + weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin" + if max_shard_size is not None: + filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors") + # an memory-efficient approach for sharding + empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True) + state_dict_split = split_torch_state_dict_into_shards(empty_state_dict, + filename_pattern=filename_pattern, + max_shard_size=max_shard_size) + else: + from collections import namedtuple + StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"]) + state_dict_split = StateDictSplit(is_sharded=False, + filename_to_tensors={weights_name: list(state_dict.keys())}) + + # Save the model by shard + os.makedirs(output_dir, exist_ok=True) + filename_to_tensors = state_dict_split.filename_to_tensors.items() + for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"): + shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors} + shard_state_dict = to_torch_tensor(shard_state_dict) + output_path = os.path.join(output_dir, shard_file) + if safe_serialization: + save_file(shard_state_dict, output_path, metadata={"format": "pt"}) + else: + torch.save(shard_state_dict, output_path) + # release the memory of current shard + for tensor_name in list(shard_state_dict.keys()): + del state_dict[tensor_name] + del shard_state_dict[tensor_name] + del shard_state_dict + gc.collect() + + # Save index if sharded + if state_dict_split.is_sharded: + index = { + "metadata": state_dict_split.metadata, + "weight_map": state_dict_split.tensor_to_filename, + } + save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json" + save_index_file = os.path.join(output_dir, save_index_file) + with open(save_index_file, "w", encoding="utf-8") as f: + content = json.dumps(index, indent=2, sort_keys=True) + "\n" + f.write(content) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info("Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info("Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument("output_dir", + type=str, + help="directory to the pytorch fp32 state_dict output files" + "(e.g. path/checkpoint-12-output/)") + parser.add_argument( + "--max_shard_size", + type=str, + default="5GB", + help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size" + "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`" + "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances" + "without CPU OOM issues.") + parser.add_argument( + "--safe_serialization", + default=False, + action='store_true', + help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_dir, + max_shard_size=args.max_shard_size, + safe_serialization=args.safe_serialization, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1435/config.json b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1435/config.json new file mode 100644 index 0000000000000000000000000000000000000000..18bd58d561964a16b60687f1b04aefdfed0b9b20 --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1435/config.json @@ -0,0 +1,39 @@ +{ + "architectures": [ + "LlamaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "dtype": "bfloat16", + "eos_token_id": [ + 128001, + 128008, + 128009 + ], + "head_dim": 64, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 8192, + "max_position_embeddings": 131072, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 32, + "num_hidden_layers": 16, + "num_key_value_heads": 8, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": { + "factor": 32.0, + "high_freq_factor": 4.0, + "low_freq_factor": 1.0, + "original_max_position_embeddings": 8192, + "rope_type": "llama3" + }, + "rope_theta": 500000.0, + "tie_word_embeddings": true, + "transformers_version": "4.57.1", + "use_cache": false, + "vocab_size": 128256 +} diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1435/generation_config.json b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1435/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..125cc74759bc705ab8a0c48d07d7eac083f56011 --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1435/generation_config.json @@ -0,0 +1,12 @@ +{ + "bos_token_id": 128000, + "do_sample": true, + "eos_token_id": [ + 128001, + 128008, + 128009 + ], + "temperature": 0.6, + "top_p": 0.9, + "transformers_version": "4.57.1" +} diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1435/latest b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1435/latest new file mode 100644 index 0000000000000000000000000000000000000000..7afd093895a4424f229eb688b400d98d1a01114b --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1435/latest @@ -0,0 +1 @@ +global_step1435 \ No newline at end of file diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1435/model.safetensors b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1435/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..02b7c2509890110d9bfa73ec332a6b8c80275088 --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1435/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cbe31e0f0b3fbf43f52fe7b150aee5c28e90a332cb8459fea906932f5e368c93 +size 2996982344 diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1435/scheduler.pt b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1435/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..89655e0f7e3ab7732d9f305d800fa59c75a5a852 --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1435/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e5c4ae215497f7a140302ca16ad638046b865f4344b7be03b9098b7c29a892c +size 1465 diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1435/trainer_state.json b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1435/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..b2d96f4e728f828a3fa71c7392db366346b60cad --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1435/trainer_state.json @@ -0,0 +1,1035 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.7483241925655089, + "eval_steps": 500, + "global_step": 1435, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.01218769043266301, + "grad_norm": 0.36486390233039856, + "learning_rate": 5e-05, + "loss": 1.2213, + "step": 10 + }, + { + "epoch": 0.02437538086532602, + "grad_norm": 0.2953564524650574, + "learning_rate": 5e-05, + "loss": 1.0223, + "step": 20 + }, + { + "epoch": 0.03656307129798903, + "grad_norm": 0.2766735851764679, + "learning_rate": 5e-05, + "loss": 0.9589, + "step": 30 + }, + { + "epoch": 0.04875076173065204, + "grad_norm": 0.32770997285842896, + "learning_rate": 5e-05, + "loss": 0.9081, + "step": 40 + }, + { + "epoch": 0.06093845216331505, + "grad_norm": 0.3052276372909546, + "learning_rate": 5e-05, + "loss": 0.8752, + "step": 50 + }, + { + "epoch": 0.07312614259597806, + "grad_norm": 0.240146666765213, + "learning_rate": 5e-05, + "loss": 0.873, + "step": 60 + }, + { + "epoch": 0.08531383302864107, + "grad_norm": 0.23872187733650208, + "learning_rate": 5e-05, + "loss": 0.8587, + "step": 70 + }, + { + "epoch": 0.09750152346130408, + "grad_norm": 0.32638487219810486, + "learning_rate": 5e-05, + "loss": 0.8656, + "step": 80 + }, + { + "epoch": 0.10968921389396709, + "grad_norm": 0.25019028782844543, + "learning_rate": 5e-05, + "loss": 0.8685, + "step": 90 + }, + { + "epoch": 0.1218769043266301, + "grad_norm": 0.2507675886154175, + "learning_rate": 5e-05, + "loss": 0.8662, + "step": 100 + }, + { + "epoch": 0.1340645947592931, + "grad_norm": 0.27966755628585815, + "learning_rate": 5e-05, + "loss": 0.8428, + "step": 110 + }, + { + "epoch": 0.14625228519195613, + "grad_norm": 0.2622673213481903, + "learning_rate": 5e-05, + "loss": 0.8416, + "step": 120 + }, + { + "epoch": 0.15843997562461914, + "grad_norm": 0.3003966510295868, + "learning_rate": 5e-05, + "loss": 0.8528, + "step": 130 + }, + { + "epoch": 0.17062766605728213, + "grad_norm": 0.32308319211006165, + "learning_rate": 5e-05, + "loss": 0.8357, + "step": 140 + }, + { + "epoch": 0.18281535648994515, + "grad_norm": 0.27279096841812134, + "learning_rate": 5e-05, + "loss": 0.8432, + "step": 150 + }, + { + "epoch": 0.19500304692260817, + "grad_norm": 0.27976104617118835, + "learning_rate": 5e-05, + "loss": 0.8119, + "step": 160 + }, + { + "epoch": 0.2071907373552712, + "grad_norm": 0.290866881608963, + "learning_rate": 5e-05, + "loss": 0.8096, + "step": 170 + }, + { + "epoch": 0.21937842778793418, + "grad_norm": 0.29430949687957764, + "learning_rate": 5e-05, + "loss": 0.819, + "step": 180 + }, + { + "epoch": 0.2315661182205972, + "grad_norm": 0.27714136242866516, + "learning_rate": 5e-05, + "loss": 0.8187, + "step": 190 + }, + { + "epoch": 0.2437538086532602, + "grad_norm": 0.2997029423713684, + "learning_rate": 5e-05, + "loss": 0.808, + "step": 200 + }, + { + "epoch": 0.25594149908592323, + "grad_norm": 0.2539573609828949, + "learning_rate": 5e-05, + "loss": 0.7943, + "step": 210 + }, + { + "epoch": 0.2681291895185862, + "grad_norm": 0.23294171690940857, + "learning_rate": 5e-05, + "loss": 0.8188, + "step": 220 + }, + { + "epoch": 0.28031687995124926, + "grad_norm": 0.25016292929649353, + "learning_rate": 5e-05, + "loss": 0.8035, + "step": 230 + }, + { + "epoch": 0.29250457038391225, + "grad_norm": 0.37125322222709656, + "learning_rate": 5e-05, + "loss": 0.785, + "step": 240 + }, + { + "epoch": 0.30469226081657524, + "grad_norm": 0.21520452201366425, + "learning_rate": 5e-05, + "loss": 0.8088, + "step": 250 + }, + { + "epoch": 0.3168799512492383, + "grad_norm": 0.20990602672100067, + "learning_rate": 5e-05, + "loss": 0.7813, + "step": 260 + }, + { + "epoch": 0.3290676416819013, + "grad_norm": 0.24193961918354034, + "learning_rate": 5e-05, + "loss": 0.7979, + "step": 270 + }, + { + "epoch": 0.34125533211456427, + "grad_norm": 0.22766050696372986, + "learning_rate": 5e-05, + "loss": 0.785, + "step": 280 + }, + { + "epoch": 0.3534430225472273, + "grad_norm": 0.2662702202796936, + "learning_rate": 5e-05, + "loss": 0.7766, + "step": 290 + }, + { + "epoch": 0.3656307129798903, + "grad_norm": 0.23020495474338531, + "learning_rate": 5e-05, + "loss": 0.7687, + "step": 300 + }, + { + "epoch": 0.3778184034125533, + "grad_norm": 0.21770520508289337, + "learning_rate": 5e-05, + "loss": 0.7815, + "step": 310 + }, + { + "epoch": 0.39000609384521634, + "grad_norm": 0.22677403688430786, + "learning_rate": 5e-05, + "loss": 0.7999, + "step": 320 + }, + { + "epoch": 0.40219378427787933, + "grad_norm": 0.24891860783100128, + "learning_rate": 5e-05, + "loss": 0.7946, + "step": 330 + }, + { + "epoch": 0.4143814747105424, + "grad_norm": 0.2506854832172394, + "learning_rate": 5e-05, + "loss": 0.7889, + "step": 340 + }, + { + "epoch": 0.42656916514320536, + "grad_norm": 0.24388180673122406, + "learning_rate": 5e-05, + "loss": 0.79, + "step": 350 + }, + { + "epoch": 0.43875685557586835, + "grad_norm": 0.22329650819301605, + "learning_rate": 5e-05, + "loss": 0.7632, + "step": 360 + }, + { + "epoch": 0.4509445460085314, + "grad_norm": 0.2240927815437317, + "learning_rate": 5e-05, + "loss": 0.8042, + "step": 370 + }, + { + "epoch": 0.4631322364411944, + "grad_norm": 0.2578203082084656, + "learning_rate": 5e-05, + "loss": 0.7667, + "step": 380 + }, + { + "epoch": 0.4753199268738574, + "grad_norm": 0.25349217653274536, + "learning_rate": 5e-05, + "loss": 0.7911, + "step": 390 + }, + { + "epoch": 0.4875076173065204, + "grad_norm": 0.254446804523468, + "learning_rate": 5e-05, + "loss": 0.7909, + "step": 400 + }, + { + "epoch": 0.4996953077391834, + "grad_norm": 0.24253171682357788, + "learning_rate": 5e-05, + "loss": 0.7896, + "step": 410 + }, + { + "epoch": 0.5118829981718465, + "grad_norm": 0.27901458740234375, + "learning_rate": 5e-05, + "loss": 0.7828, + "step": 420 + }, + { + "epoch": 0.5240706886045094, + "grad_norm": 0.31884434819221497, + "learning_rate": 5e-05, + "loss": 0.7919, + "step": 430 + }, + { + "epoch": 0.5362583790371724, + "grad_norm": 0.22920799255371094, + "learning_rate": 5e-05, + "loss": 0.7777, + "step": 440 + }, + { + "epoch": 0.5484460694698354, + "grad_norm": 0.22606618702411652, + "learning_rate": 5e-05, + "loss": 0.7661, + "step": 450 + }, + { + "epoch": 0.5606337599024985, + "grad_norm": 0.21604052186012268, + "learning_rate": 5e-05, + "loss": 0.769, + "step": 460 + }, + { + "epoch": 0.5728214503351615, + "grad_norm": 0.2296101152896881, + "learning_rate": 5e-05, + "loss": 0.7626, + "step": 470 + }, + { + "epoch": 0.5850091407678245, + "grad_norm": 0.24162150919437408, + "learning_rate": 5e-05, + "loss": 0.7831, + "step": 480 + }, + { + "epoch": 0.5971968312004875, + "grad_norm": 0.22364799678325653, + "learning_rate": 5e-05, + "loss": 0.7737, + "step": 490 + }, + { + "epoch": 0.6093845216331505, + "grad_norm": 0.23953403532505035, + "learning_rate": 5e-05, + "loss": 0.787, + "step": 500 + }, + { + "epoch": 0.6215722120658135, + "grad_norm": 0.22349311411380768, + "learning_rate": 5e-05, + "loss": 0.7763, + "step": 510 + }, + { + "epoch": 0.6337599024984766, + "grad_norm": 0.21996377408504486, + "learning_rate": 5e-05, + "loss": 0.7596, + "step": 520 + }, + { + "epoch": 0.6459475929311396, + "grad_norm": 0.245538130402565, + "learning_rate": 5e-05, + "loss": 0.7785, + "step": 530 + }, + { + "epoch": 0.6581352833638026, + "grad_norm": 0.23442520201206207, + "learning_rate": 5e-05, + "loss": 0.7633, + "step": 540 + }, + { + "epoch": 0.6703229737964655, + "grad_norm": 0.22274106740951538, + "learning_rate": 5e-05, + "loss": 0.7756, + "step": 550 + }, + { + "epoch": 0.6825106642291285, + "grad_norm": 0.21717530488967896, + "learning_rate": 5e-05, + "loss": 0.7726, + "step": 560 + }, + { + "epoch": 0.6946983546617916, + "grad_norm": 0.2465955913066864, + "learning_rate": 5e-05, + "loss": 0.7476, + "step": 570 + }, + { + "epoch": 0.7068860450944546, + "grad_norm": 0.2318575084209442, + "learning_rate": 5e-05, + "loss": 0.7584, + "step": 580 + }, + { + "epoch": 0.7190737355271176, + "grad_norm": 0.25114378333091736, + "learning_rate": 5e-05, + "loss": 0.7625, + "step": 590 + }, + { + "epoch": 0.7312614259597806, + "grad_norm": 0.19145755469799042, + "learning_rate": 5e-05, + "loss": 0.7577, + "step": 600 + }, + { + "epoch": 0.7434491163924436, + "grad_norm": 0.19129066169261932, + "learning_rate": 5e-05, + "loss": 0.7772, + "step": 610 + }, + { + "epoch": 0.7556368068251066, + "grad_norm": 0.2014475166797638, + "learning_rate": 5e-05, + "loss": 0.7601, + "step": 620 + }, + { + "epoch": 0.7678244972577697, + "grad_norm": 0.21480509638786316, + "learning_rate": 5e-05, + "loss": 0.7665, + "step": 630 + }, + { + "epoch": 0.7800121876904327, + "grad_norm": 0.2633228302001953, + "learning_rate": 5e-05, + "loss": 0.7565, + "step": 640 + }, + { + "epoch": 0.7921998781230957, + "grad_norm": 0.24633203446865082, + "learning_rate": 5e-05, + "loss": 0.7602, + "step": 650 + }, + { + "epoch": 0.8043875685557587, + "grad_norm": 0.19770534336566925, + "learning_rate": 5e-05, + "loss": 0.7685, + "step": 660 + }, + { + "epoch": 0.8165752589884216, + "grad_norm": 0.2267930805683136, + "learning_rate": 5e-05, + "loss": 0.7516, + "step": 670 + }, + { + "epoch": 0.8287629494210847, + "grad_norm": 0.238111674785614, + "learning_rate": 5e-05, + "loss": 0.7487, + "step": 680 + }, + { + "epoch": 0.8409506398537477, + "grad_norm": 0.19528904557228088, + "learning_rate": 5e-05, + "loss": 0.7467, + "step": 690 + }, + { + "epoch": 0.8531383302864107, + "grad_norm": 0.21305914223194122, + "learning_rate": 5e-05, + "loss": 0.7527, + "step": 700 + }, + { + "epoch": 0.8653260207190737, + "grad_norm": 0.2205205112695694, + "learning_rate": 5e-05, + "loss": 0.7639, + "step": 710 + }, + { + "epoch": 0.8775137111517367, + "grad_norm": 0.22429127991199493, + "learning_rate": 5e-05, + "loss": 0.7744, + "step": 720 + }, + { + "epoch": 0.8897014015843998, + "grad_norm": 0.23755772411823273, + "learning_rate": 5e-05, + "loss": 0.7446, + "step": 730 + }, + { + "epoch": 0.9018890920170628, + "grad_norm": 0.17592693865299225, + "learning_rate": 5e-05, + "loss": 0.7442, + "step": 740 + }, + { + "epoch": 0.9140767824497258, + "grad_norm": 0.2266898900270462, + "learning_rate": 5e-05, + "loss": 0.7326, + "step": 750 + }, + { + "epoch": 0.9262644728823888, + "grad_norm": 0.23541569709777832, + "learning_rate": 5e-05, + "loss": 0.7338, + "step": 760 + }, + { + "epoch": 0.9384521633150518, + "grad_norm": 0.22233888506889343, + "learning_rate": 5e-05, + "loss": 0.7385, + "step": 770 + }, + { + "epoch": 0.9506398537477148, + "grad_norm": 0.23714767396450043, + "learning_rate": 5e-05, + "loss": 0.7306, + "step": 780 + }, + { + "epoch": 0.9628275441803779, + "grad_norm": 0.21191060543060303, + "learning_rate": 5e-05, + "loss": 0.736, + "step": 790 + }, + { + "epoch": 0.9750152346130408, + "grad_norm": 0.22759106755256653, + "learning_rate": 5e-05, + "loss": 0.7505, + "step": 800 + }, + { + "epoch": 0.9872029250457038, + "grad_norm": 0.19895903766155243, + "learning_rate": 5e-05, + "loss": 0.7536, + "step": 810 + }, + { + "epoch": 0.9993906154783668, + "grad_norm": 0.3061685860157013, + "learning_rate": 5e-05, + "loss": 0.7493, + "step": 820 + }, + { + "epoch": 1.0109689213893966, + "grad_norm": 0.18868015706539154, + "learning_rate": 5e-05, + "loss": 0.7342, + "step": 830 + }, + { + "epoch": 1.0231566118220596, + "grad_norm": 0.1797628402709961, + "learning_rate": 5e-05, + "loss": 0.7192, + "step": 840 + }, + { + "epoch": 1.0353443022547226, + "grad_norm": 0.19511596858501434, + "learning_rate": 5e-05, + "loss": 0.7002, + "step": 850 + }, + { + "epoch": 1.0475319926873858, + "grad_norm": 0.2123185396194458, + "learning_rate": 5e-05, + "loss": 0.7236, + "step": 860 + }, + { + "epoch": 1.0597196831200488, + "grad_norm": 0.2047470211982727, + "learning_rate": 5e-05, + "loss": 0.711, + "step": 870 + }, + { + "epoch": 1.0719073735527118, + "grad_norm": 0.18809868395328522, + "learning_rate": 5e-05, + "loss": 0.7253, + "step": 880 + }, + { + "epoch": 1.0840950639853748, + "grad_norm": 0.2207012176513672, + "learning_rate": 5e-05, + "loss": 0.6971, + "step": 890 + }, + { + "epoch": 1.0962827544180378, + "grad_norm": 0.1912035495042801, + "learning_rate": 5e-05, + "loss": 0.7239, + "step": 900 + }, + { + "epoch": 1.1084704448507008, + "grad_norm": 0.23486360907554626, + "learning_rate": 5e-05, + "loss": 0.7335, + "step": 910 + }, + { + "epoch": 1.1206581352833638, + "grad_norm": 0.22189787030220032, + "learning_rate": 5e-05, + "loss": 0.7149, + "step": 920 + }, + { + "epoch": 1.1328458257160268, + "grad_norm": 0.25818824768066406, + "learning_rate": 5e-05, + "loss": 0.7322, + "step": 930 + }, + { + "epoch": 1.1450335161486898, + "grad_norm": 0.2088775783777237, + "learning_rate": 5e-05, + "loss": 0.7382, + "step": 940 + }, + { + "epoch": 1.1572212065813527, + "grad_norm": 0.19445833563804626, + "learning_rate": 5e-05, + "loss": 0.7223, + "step": 950 + }, + { + "epoch": 1.1694088970140157, + "grad_norm": 0.21489456295967102, + "learning_rate": 5e-05, + "loss": 0.7317, + "step": 960 + }, + { + "epoch": 1.181596587446679, + "grad_norm": 0.24050794541835785, + "learning_rate": 5e-05, + "loss": 0.7275, + "step": 970 + }, + { + "epoch": 1.193784277879342, + "grad_norm": 0.18142102658748627, + "learning_rate": 5e-05, + "loss": 0.7046, + "step": 980 + }, + { + "epoch": 1.205971968312005, + "grad_norm": 0.4781077802181244, + "learning_rate": 5e-05, + "loss": 0.7433, + "step": 990 + }, + { + "epoch": 1.218159658744668, + "grad_norm": 0.20628023147583008, + "learning_rate": 5e-05, + "loss": 0.7403, + "step": 1000 + }, + { + "epoch": 1.230347349177331, + "grad_norm": 0.19117140769958496, + "learning_rate": 5e-05, + "loss": 0.7308, + "step": 1010 + }, + { + "epoch": 1.242535039609994, + "grad_norm": 0.2107745110988617, + "learning_rate": 5e-05, + "loss": 0.7201, + "step": 1020 + }, + { + "epoch": 1.2547227300426569, + "grad_norm": 0.20551154017448425, + "learning_rate": 5e-05, + "loss": 0.7038, + "step": 1030 + }, + { + "epoch": 1.2669104204753199, + "grad_norm": 0.26577022671699524, + "learning_rate": 5e-05, + "loss": 0.7096, + "step": 1040 + }, + { + "epoch": 1.2790981109079829, + "grad_norm": 0.22094209492206573, + "learning_rate": 5e-05, + "loss": 0.718, + "step": 1050 + }, + { + "epoch": 1.291285801340646, + "grad_norm": 0.18224076926708221, + "learning_rate": 5e-05, + "loss": 0.7261, + "step": 1060 + }, + { + "epoch": 1.303473491773309, + "grad_norm": 0.226251482963562, + "learning_rate": 5e-05, + "loss": 0.7372, + "step": 1070 + }, + { + "epoch": 1.315661182205972, + "grad_norm": 0.20186461508274078, + "learning_rate": 5e-05, + "loss": 0.7074, + "step": 1080 + }, + { + "epoch": 1.327848872638635, + "grad_norm": 0.22229762375354767, + "learning_rate": 5e-05, + "loss": 0.7069, + "step": 1090 + }, + { + "epoch": 1.340036563071298, + "grad_norm": 0.1776452213525772, + "learning_rate": 5e-05, + "loss": 0.714, + "step": 1100 + }, + { + "epoch": 1.352224253503961, + "grad_norm": 0.20680229365825653, + "learning_rate": 5e-05, + "loss": 0.7066, + "step": 1110 + }, + { + "epoch": 1.364411943936624, + "grad_norm": 0.21842031180858612, + "learning_rate": 5e-05, + "loss": 0.7281, + "step": 1120 + }, + { + "epoch": 1.376599634369287, + "grad_norm": 0.28037774562835693, + "learning_rate": 5e-05, + "loss": 0.7149, + "step": 1130 + }, + { + "epoch": 1.38878732480195, + "grad_norm": 0.20962920784950256, + "learning_rate": 5e-05, + "loss": 0.6951, + "step": 1140 + }, + { + "epoch": 1.400975015234613, + "grad_norm": 0.18751871585845947, + "learning_rate": 5e-05, + "loss": 0.7101, + "step": 1150 + }, + { + "epoch": 1.413162705667276, + "grad_norm": 0.2722964584827423, + "learning_rate": 5e-05, + "loss": 0.7205, + "step": 1160 + }, + { + "epoch": 1.425350396099939, + "grad_norm": 0.20932041108608246, + "learning_rate": 5e-05, + "loss": 0.7082, + "step": 1170 + }, + { + "epoch": 1.437538086532602, + "grad_norm": 0.17836257815361023, + "learning_rate": 5e-05, + "loss": 0.7232, + "step": 1180 + }, + { + "epoch": 1.449725776965265, + "grad_norm": 0.3180703818798065, + "learning_rate": 5e-05, + "loss": 0.7245, + "step": 1190 + }, + { + "epoch": 1.4619134673979282, + "grad_norm": 0.201424703001976, + "learning_rate": 5e-05, + "loss": 0.7232, + "step": 1200 + }, + { + "epoch": 1.4741011578305911, + "grad_norm": 0.20228999853134155, + "learning_rate": 5e-05, + "loss": 0.7169, + "step": 1210 + }, + { + "epoch": 1.4862888482632541, + "grad_norm": 0.17478394508361816, + "learning_rate": 5e-05, + "loss": 0.704, + "step": 1220 + }, + { + "epoch": 1.4984765386959171, + "grad_norm": 0.20061351358890533, + "learning_rate": 5e-05, + "loss": 0.6792, + "step": 1230 + }, + { + "epoch": 1.5106642291285801, + "grad_norm": 0.1857377588748932, + "learning_rate": 5e-05, + "loss": 0.709, + "step": 1240 + }, + { + "epoch": 1.522851919561243, + "grad_norm": 0.19873379170894623, + "learning_rate": 5e-05, + "loss": 0.7007, + "step": 1250 + }, + { + "epoch": 1.5350396099939063, + "grad_norm": 0.19807074964046478, + "learning_rate": 5e-05, + "loss": 0.7122, + "step": 1260 + }, + { + "epoch": 1.5472273004265693, + "grad_norm": 0.16746796667575836, + "learning_rate": 5e-05, + "loss": 0.7005, + "step": 1270 + }, + { + "epoch": 1.5594149908592323, + "grad_norm": 0.2750326991081238, + "learning_rate": 5e-05, + "loss": 0.7166, + "step": 1280 + }, + { + "epoch": 1.5716026812918953, + "grad_norm": 0.2776738107204437, + "learning_rate": 5e-05, + "loss": 0.7405, + "step": 1290 + }, + { + "epoch": 1.5837903717245583, + "grad_norm": 0.3494517207145691, + "learning_rate": 5e-05, + "loss": 0.7192, + "step": 1300 + }, + { + "epoch": 1.5959780621572213, + "grad_norm": 0.25028494000434875, + "learning_rate": 5e-05, + "loss": 0.741, + "step": 1310 + }, + { + "epoch": 1.6081657525898843, + "grad_norm": 0.23914143443107605, + "learning_rate": 5e-05, + "loss": 0.72, + "step": 1320 + }, + { + "epoch": 1.6203534430225472, + "grad_norm": 0.2052862048149109, + "learning_rate": 5e-05, + "loss": 0.7112, + "step": 1330 + }, + { + "epoch": 1.6325411334552102, + "grad_norm": 0.22285936772823334, + "learning_rate": 5e-05, + "loss": 0.7165, + "step": 1340 + }, + { + "epoch": 1.6447288238878732, + "grad_norm": 0.20810848474502563, + "learning_rate": 5e-05, + "loss": 0.7122, + "step": 1350 + }, + { + "epoch": 1.6569165143205362, + "grad_norm": 0.20422884821891785, + "learning_rate": 5e-05, + "loss": 0.7082, + "step": 1360 + }, + { + "epoch": 1.6691042047531992, + "grad_norm": 0.18164314329624176, + "learning_rate": 5e-05, + "loss": 0.7131, + "step": 1370 + }, + { + "epoch": 1.6812918951858622, + "grad_norm": 0.19893014430999756, + "learning_rate": 5e-05, + "loss": 0.7055, + "step": 1380 + }, + { + "epoch": 1.6934795856185252, + "grad_norm": 0.24865971505641937, + "learning_rate": 5e-05, + "loss": 0.6994, + "step": 1390 + }, + { + "epoch": 1.7056672760511882, + "grad_norm": 0.18078123033046722, + "learning_rate": 5e-05, + "loss": 0.7162, + "step": 1400 + }, + { + "epoch": 1.7178549664838512, + "grad_norm": 0.1973421722650528, + "learning_rate": 5e-05, + "loss": 0.7136, + "step": 1410 + }, + { + "epoch": 1.7300426569165142, + "grad_norm": 0.2057911604642868, + "learning_rate": 5e-05, + "loss": 0.7004, + "step": 1420 + }, + { + "epoch": 1.7422303473491774, + "grad_norm": 0.18640495836734772, + "learning_rate": 5e-05, + "loss": 0.7102, + "step": 1430 + } + ], + "logging_steps": 10, + "max_steps": 2463, + "num_input_tokens_seen": 0, + "num_train_epochs": 3, + "save_steps": 205, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 5.8872689779519e+18, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1435/training_args.bin b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1435/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..23ae0ec8a4226be5457f61c137abab2b2c10a0cf --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1435/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56896ea21940e33d7e2934267dff2a5b4a4a80195a0f165245f6617a0f88b90b +size 7377 diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1435/zero_to_fp32.py b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1435/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..5995d6e6f04e43b989587aa9022a3aef0c66d694 --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1435/zero_to_fp32.py @@ -0,0 +1,760 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: +# python zero_to_fp32.py . output_dir/ +# or +# python zero_to_fp32.py . output_dir/ --safe_serialization + +import argparse +import torch +import glob +import math +import os +import re +import gc +import json +import numpy as np +from tqdm import tqdm +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device, weights_only=False) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + total_files = len(files) + state_dicts = [] + for f in tqdm(files, desc='Loading checkpoint shards'): + state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + model_files = get_model_state_files(ds_checkpoint_dir) + + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +class GatheredTensor: + """ + A pseudo tensor that collects partitioned weights. + It is more memory efficient when there are multiple groups. + """ + + def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape): + self.flat_groups = flat_groups + self.flat_groups_offset = flat_groups_offset + self.offset = offset + self.partitioned_numel = partitioned_numel + self.shape = shape + self.dtype = self.flat_groups[0][0].dtype + + def contiguous(self): + """ + Merge partitioned weights from flat_groups into a single tensor. + """ + end_idx = self.offset + self.partitioned_numel + world_size = len(self.flat_groups) + pad_flat_param_chunks = [] + + for rank_i in range(world_size): + # for each rank, we need to collect weights from related group/groups + flat_groups_at_rank_i = self.flat_groups[rank_i] + start_group_id = None + end_group_id = None + for group_id in range(len(self.flat_groups_offset)): + if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]: + start_group_id = group_id + if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]: + end_group_id = group_id + break + # collect weights from related group/groups + for group_id in range(start_group_id, end_group_id + 1): + flat_tensor = flat_groups_at_rank_i[group_id] + start_offset = self.offset - self.flat_groups_offset[group_id] + end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id] + pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset]) + + # collect weights from all ranks + pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0) + param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous() + return param + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size + + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]])) + for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'): + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # memory efficient tensor + tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape) + state_dict[name] = tensor + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def to_torch_tensor(state_dict, return_empty_tensor=False): + """ + Convert state_dict of GatheredTensor to torch tensor + """ + torch_state_dict = {} + converted_tensors = {} + for name, tensor in state_dict.items(): + tensor_id = id(tensor) + if tensor_id in converted_tensors: # shared tensors + shared_tensor = torch_state_dict[converted_tensors[tensor_id]] + torch_state_dict[name] = shared_tensor + else: + converted_tensors[tensor_id] = name + if return_empty_tensor: + torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype) + else: + torch_state_dict[name] = tensor.contiguous() + return torch_state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag=None, + exclude_frozen_parameters=False, + lazy_mode=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient. + Convert the pesduo tensor to torch tensor by ``.contiguous()`` + + Returns: + - pytorch ``state_dict`` + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + Note: the above usage may not work if your application doesn't have sufficient free CPU memory. + You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. Or you can load state_dict in lazy mode :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu + for name, lazy_tensor in state_dict.item(): + tensor = lazy_tensor.contiguous() # to cpu + print(name, tensor) + # del tensor to release memory if it no longer in use + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + if lazy_mode: + return state_dict + else: + return to_torch_tensor(state_dict) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, + output_dir, + max_shard_size="5GB", + safe_serialization=False, + tag=None, + exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_dir``: directory to the pytorch fp32 state_dict output files + - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB + - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + + # Dependency pre-check + if safe_serialization: + try: + from safetensors.torch import save_file + except ImportError: + print('If you want to use `safe_serialization`, please `pip install safetensors`') + raise + if max_shard_size is not None: + try: + from huggingface_hub import split_torch_state_dict_into_shards + except ImportError: + print('If you want to use `max_shard_size`, please `pip install huggingface_hub`') + raise + + # Convert zero checkpoint to state_dict + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag, + exclude_frozen_parameters, + lazy_mode=True) + + # Shard the model if it is too big. + weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin" + if max_shard_size is not None: + filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors") + # an memory-efficient approach for sharding + empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True) + state_dict_split = split_torch_state_dict_into_shards(empty_state_dict, + filename_pattern=filename_pattern, + max_shard_size=max_shard_size) + else: + from collections import namedtuple + StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"]) + state_dict_split = StateDictSplit(is_sharded=False, + filename_to_tensors={weights_name: list(state_dict.keys())}) + + # Save the model by shard + os.makedirs(output_dir, exist_ok=True) + filename_to_tensors = state_dict_split.filename_to_tensors.items() + for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"): + shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors} + shard_state_dict = to_torch_tensor(shard_state_dict) + output_path = os.path.join(output_dir, shard_file) + if safe_serialization: + save_file(shard_state_dict, output_path, metadata={"format": "pt"}) + else: + torch.save(shard_state_dict, output_path) + # release the memory of current shard + for tensor_name in list(shard_state_dict.keys()): + del state_dict[tensor_name] + del shard_state_dict[tensor_name] + del shard_state_dict + gc.collect() + + # Save index if sharded + if state_dict_split.is_sharded: + index = { + "metadata": state_dict_split.metadata, + "weight_map": state_dict_split.tensor_to_filename, + } + save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json" + save_index_file = os.path.join(output_dir, save_index_file) + with open(save_index_file, "w", encoding="utf-8") as f: + content = json.dumps(index, indent=2, sort_keys=True) + "\n" + f.write(content) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info("Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info("Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument("output_dir", + type=str, + help="directory to the pytorch fp32 state_dict output files" + "(e.g. path/checkpoint-12-output/)") + parser.add_argument( + "--max_shard_size", + type=str, + default="5GB", + help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size" + "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`" + "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances" + "without CPU OOM issues.") + parser.add_argument( + "--safe_serialization", + default=False, + action='store_true', + help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_dir, + max_shard_size=args.max_shard_size, + safe_serialization=args.safe_serialization, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1640/config.json b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1640/config.json new file mode 100644 index 0000000000000000000000000000000000000000..18bd58d561964a16b60687f1b04aefdfed0b9b20 --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1640/config.json @@ -0,0 +1,39 @@ +{ + "architectures": [ + "LlamaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "dtype": "bfloat16", + "eos_token_id": [ + 128001, + 128008, + 128009 + ], + "head_dim": 64, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 8192, + "max_position_embeddings": 131072, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 32, + "num_hidden_layers": 16, + "num_key_value_heads": 8, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": { + "factor": 32.0, + "high_freq_factor": 4.0, + "low_freq_factor": 1.0, + "original_max_position_embeddings": 8192, + "rope_type": "llama3" + }, + "rope_theta": 500000.0, + "tie_word_embeddings": true, + "transformers_version": "4.57.1", + "use_cache": false, + "vocab_size": 128256 +} diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1640/generation_config.json b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1640/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..125cc74759bc705ab8a0c48d07d7eac083f56011 --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1640/generation_config.json @@ -0,0 +1,12 @@ +{ + "bos_token_id": 128000, + "do_sample": true, + "eos_token_id": [ + 128001, + 128008, + 128009 + ], + "temperature": 0.6, + "top_p": 0.9, + "transformers_version": "4.57.1" +} diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1640/latest b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1640/latest new file mode 100644 index 0000000000000000000000000000000000000000..f32b044f59453e5657540017a239aabc81c068fd --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1640/latest @@ -0,0 +1 @@ +global_step1640 \ No newline at end of file diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1640/model.safetensors b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1640/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..abc8871d9f24d656cfa5ae4f3fc0fa7861f00b94 --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1640/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8257b1ffe27bccf980bead367bedf21a6cbdab1f058457747062cf9c26601b19 +size 2996982344 diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1640/scheduler.pt b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1640/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..c73e1b17ae0f050b53cb817aa77df32d4717fae9 --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1640/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:871123a3f5e5af48623dacbf8d53b1f1d8ad18307a63bd7259d3c594eb7e872a +size 1465 diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1640/trainer_state.json b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1640/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..faa08d8412988f914efa7e6c3c0e610f0f1e2fcc --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1640/trainer_state.json @@ -0,0 +1,1182 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.9981718464351006, + "eval_steps": 500, + "global_step": 1640, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.01218769043266301, + "grad_norm": 0.36486390233039856, + "learning_rate": 5e-05, + "loss": 1.2213, + "step": 10 + }, + { + "epoch": 0.02437538086532602, + "grad_norm": 0.2953564524650574, + "learning_rate": 5e-05, + "loss": 1.0223, + "step": 20 + }, + { + "epoch": 0.03656307129798903, + "grad_norm": 0.2766735851764679, + "learning_rate": 5e-05, + "loss": 0.9589, + "step": 30 + }, + { + "epoch": 0.04875076173065204, + "grad_norm": 0.32770997285842896, + "learning_rate": 5e-05, + "loss": 0.9081, + "step": 40 + }, + { + "epoch": 0.06093845216331505, + "grad_norm": 0.3052276372909546, + "learning_rate": 5e-05, + "loss": 0.8752, + "step": 50 + }, + { + "epoch": 0.07312614259597806, + "grad_norm": 0.240146666765213, + "learning_rate": 5e-05, + "loss": 0.873, + "step": 60 + }, + { + "epoch": 0.08531383302864107, + "grad_norm": 0.23872187733650208, + "learning_rate": 5e-05, + "loss": 0.8587, + "step": 70 + }, + { + "epoch": 0.09750152346130408, + "grad_norm": 0.32638487219810486, + "learning_rate": 5e-05, + "loss": 0.8656, + "step": 80 + }, + { + "epoch": 0.10968921389396709, + "grad_norm": 0.25019028782844543, + "learning_rate": 5e-05, + "loss": 0.8685, + "step": 90 + }, + { + "epoch": 0.1218769043266301, + "grad_norm": 0.2507675886154175, + "learning_rate": 5e-05, + "loss": 0.8662, + "step": 100 + }, + { + "epoch": 0.1340645947592931, + "grad_norm": 0.27966755628585815, + "learning_rate": 5e-05, + "loss": 0.8428, + "step": 110 + }, + { + "epoch": 0.14625228519195613, + "grad_norm": 0.2622673213481903, + "learning_rate": 5e-05, + "loss": 0.8416, + "step": 120 + }, + { + "epoch": 0.15843997562461914, + "grad_norm": 0.3003966510295868, + "learning_rate": 5e-05, + "loss": 0.8528, + "step": 130 + }, + { + "epoch": 0.17062766605728213, + "grad_norm": 0.32308319211006165, + "learning_rate": 5e-05, + "loss": 0.8357, + "step": 140 + }, + { + "epoch": 0.18281535648994515, + "grad_norm": 0.27279096841812134, + "learning_rate": 5e-05, + "loss": 0.8432, + "step": 150 + }, + { + "epoch": 0.19500304692260817, + "grad_norm": 0.27976104617118835, + "learning_rate": 5e-05, + "loss": 0.8119, + "step": 160 + }, + { + "epoch": 0.2071907373552712, + "grad_norm": 0.290866881608963, + "learning_rate": 5e-05, + "loss": 0.8096, + "step": 170 + }, + { + "epoch": 0.21937842778793418, + "grad_norm": 0.29430949687957764, + "learning_rate": 5e-05, + "loss": 0.819, + "step": 180 + }, + { + "epoch": 0.2315661182205972, + "grad_norm": 0.27714136242866516, + "learning_rate": 5e-05, + "loss": 0.8187, + "step": 190 + }, + { + "epoch": 0.2437538086532602, + "grad_norm": 0.2997029423713684, + "learning_rate": 5e-05, + "loss": 0.808, + "step": 200 + }, + { + "epoch": 0.25594149908592323, + "grad_norm": 0.2539573609828949, + "learning_rate": 5e-05, + "loss": 0.7943, + "step": 210 + }, + { + "epoch": 0.2681291895185862, + "grad_norm": 0.23294171690940857, + "learning_rate": 5e-05, + "loss": 0.8188, + "step": 220 + }, + { + "epoch": 0.28031687995124926, + "grad_norm": 0.25016292929649353, + "learning_rate": 5e-05, + "loss": 0.8035, + "step": 230 + }, + { + "epoch": 0.29250457038391225, + "grad_norm": 0.37125322222709656, + "learning_rate": 5e-05, + "loss": 0.785, + "step": 240 + }, + { + "epoch": 0.30469226081657524, + "grad_norm": 0.21520452201366425, + "learning_rate": 5e-05, + "loss": 0.8088, + "step": 250 + }, + { + "epoch": 0.3168799512492383, + "grad_norm": 0.20990602672100067, + "learning_rate": 5e-05, + "loss": 0.7813, + "step": 260 + }, + { + "epoch": 0.3290676416819013, + "grad_norm": 0.24193961918354034, + "learning_rate": 5e-05, + "loss": 0.7979, + "step": 270 + }, + { + "epoch": 0.34125533211456427, + "grad_norm": 0.22766050696372986, + "learning_rate": 5e-05, + "loss": 0.785, + "step": 280 + }, + { + "epoch": 0.3534430225472273, + "grad_norm": 0.2662702202796936, + "learning_rate": 5e-05, + "loss": 0.7766, + "step": 290 + }, + { + "epoch": 0.3656307129798903, + "grad_norm": 0.23020495474338531, + "learning_rate": 5e-05, + "loss": 0.7687, + "step": 300 + }, + { + "epoch": 0.3778184034125533, + "grad_norm": 0.21770520508289337, + "learning_rate": 5e-05, + "loss": 0.7815, + "step": 310 + }, + { + "epoch": 0.39000609384521634, + "grad_norm": 0.22677403688430786, + "learning_rate": 5e-05, + "loss": 0.7999, + "step": 320 + }, + { + "epoch": 0.40219378427787933, + "grad_norm": 0.24891860783100128, + "learning_rate": 5e-05, + "loss": 0.7946, + "step": 330 + }, + { + "epoch": 0.4143814747105424, + "grad_norm": 0.2506854832172394, + "learning_rate": 5e-05, + "loss": 0.7889, + "step": 340 + }, + { + "epoch": 0.42656916514320536, + "grad_norm": 0.24388180673122406, + "learning_rate": 5e-05, + "loss": 0.79, + "step": 350 + }, + { + "epoch": 0.43875685557586835, + "grad_norm": 0.22329650819301605, + "learning_rate": 5e-05, + "loss": 0.7632, + "step": 360 + }, + { + "epoch": 0.4509445460085314, + "grad_norm": 0.2240927815437317, + "learning_rate": 5e-05, + "loss": 0.8042, + "step": 370 + }, + { + "epoch": 0.4631322364411944, + "grad_norm": 0.2578203082084656, + "learning_rate": 5e-05, + "loss": 0.7667, + "step": 380 + }, + { + "epoch": 0.4753199268738574, + "grad_norm": 0.25349217653274536, + "learning_rate": 5e-05, + "loss": 0.7911, + "step": 390 + }, + { + "epoch": 0.4875076173065204, + "grad_norm": 0.254446804523468, + "learning_rate": 5e-05, + "loss": 0.7909, + "step": 400 + }, + { + "epoch": 0.4996953077391834, + "grad_norm": 0.24253171682357788, + "learning_rate": 5e-05, + "loss": 0.7896, + "step": 410 + }, + { + "epoch": 0.5118829981718465, + "grad_norm": 0.27901458740234375, + "learning_rate": 5e-05, + "loss": 0.7828, + "step": 420 + }, + { + "epoch": 0.5240706886045094, + "grad_norm": 0.31884434819221497, + "learning_rate": 5e-05, + "loss": 0.7919, + "step": 430 + }, + { + "epoch": 0.5362583790371724, + "grad_norm": 0.22920799255371094, + "learning_rate": 5e-05, + "loss": 0.7777, + "step": 440 + }, + { + "epoch": 0.5484460694698354, + "grad_norm": 0.22606618702411652, + "learning_rate": 5e-05, + "loss": 0.7661, + "step": 450 + }, + { + "epoch": 0.5606337599024985, + "grad_norm": 0.21604052186012268, + "learning_rate": 5e-05, + "loss": 0.769, + "step": 460 + }, + { + "epoch": 0.5728214503351615, + "grad_norm": 0.2296101152896881, + "learning_rate": 5e-05, + "loss": 0.7626, + "step": 470 + }, + { + "epoch": 0.5850091407678245, + "grad_norm": 0.24162150919437408, + "learning_rate": 5e-05, + "loss": 0.7831, + "step": 480 + }, + { + "epoch": 0.5971968312004875, + "grad_norm": 0.22364799678325653, + "learning_rate": 5e-05, + "loss": 0.7737, + "step": 490 + }, + { + "epoch": 0.6093845216331505, + "grad_norm": 0.23953403532505035, + "learning_rate": 5e-05, + "loss": 0.787, + "step": 500 + }, + { + "epoch": 0.6215722120658135, + "grad_norm": 0.22349311411380768, + "learning_rate": 5e-05, + "loss": 0.7763, + "step": 510 + }, + { + "epoch": 0.6337599024984766, + "grad_norm": 0.21996377408504486, + "learning_rate": 5e-05, + "loss": 0.7596, + "step": 520 + }, + { + "epoch": 0.6459475929311396, + "grad_norm": 0.245538130402565, + "learning_rate": 5e-05, + "loss": 0.7785, + "step": 530 + }, + { + "epoch": 0.6581352833638026, + "grad_norm": 0.23442520201206207, + "learning_rate": 5e-05, + "loss": 0.7633, + "step": 540 + }, + { + "epoch": 0.6703229737964655, + "grad_norm": 0.22274106740951538, + "learning_rate": 5e-05, + "loss": 0.7756, + "step": 550 + }, + { + "epoch": 0.6825106642291285, + "grad_norm": 0.21717530488967896, + "learning_rate": 5e-05, + "loss": 0.7726, + "step": 560 + }, + { + "epoch": 0.6946983546617916, + "grad_norm": 0.2465955913066864, + "learning_rate": 5e-05, + "loss": 0.7476, + "step": 570 + }, + { + "epoch": 0.7068860450944546, + "grad_norm": 0.2318575084209442, + "learning_rate": 5e-05, + "loss": 0.7584, + "step": 580 + }, + { + "epoch": 0.7190737355271176, + "grad_norm": 0.25114378333091736, + "learning_rate": 5e-05, + "loss": 0.7625, + "step": 590 + }, + { + "epoch": 0.7312614259597806, + "grad_norm": 0.19145755469799042, + "learning_rate": 5e-05, + "loss": 0.7577, + "step": 600 + }, + { + "epoch": 0.7434491163924436, + "grad_norm": 0.19129066169261932, + "learning_rate": 5e-05, + "loss": 0.7772, + "step": 610 + }, + { + "epoch": 0.7556368068251066, + "grad_norm": 0.2014475166797638, + "learning_rate": 5e-05, + "loss": 0.7601, + "step": 620 + }, + { + "epoch": 0.7678244972577697, + "grad_norm": 0.21480509638786316, + "learning_rate": 5e-05, + "loss": 0.7665, + "step": 630 + }, + { + "epoch": 0.7800121876904327, + "grad_norm": 0.2633228302001953, + "learning_rate": 5e-05, + "loss": 0.7565, + "step": 640 + }, + { + "epoch": 0.7921998781230957, + "grad_norm": 0.24633203446865082, + "learning_rate": 5e-05, + "loss": 0.7602, + "step": 650 + }, + { + "epoch": 0.8043875685557587, + "grad_norm": 0.19770534336566925, + "learning_rate": 5e-05, + "loss": 0.7685, + "step": 660 + }, + { + "epoch": 0.8165752589884216, + "grad_norm": 0.2267930805683136, + "learning_rate": 5e-05, + "loss": 0.7516, + "step": 670 + }, + { + "epoch": 0.8287629494210847, + "grad_norm": 0.238111674785614, + "learning_rate": 5e-05, + "loss": 0.7487, + "step": 680 + }, + { + "epoch": 0.8409506398537477, + "grad_norm": 0.19528904557228088, + "learning_rate": 5e-05, + "loss": 0.7467, + "step": 690 + }, + { + "epoch": 0.8531383302864107, + "grad_norm": 0.21305914223194122, + "learning_rate": 5e-05, + "loss": 0.7527, + "step": 700 + }, + { + "epoch": 0.8653260207190737, + "grad_norm": 0.2205205112695694, + "learning_rate": 5e-05, + "loss": 0.7639, + "step": 710 + }, + { + "epoch": 0.8775137111517367, + "grad_norm": 0.22429127991199493, + "learning_rate": 5e-05, + "loss": 0.7744, + "step": 720 + }, + { + "epoch": 0.8897014015843998, + "grad_norm": 0.23755772411823273, + "learning_rate": 5e-05, + "loss": 0.7446, + "step": 730 + }, + { + "epoch": 0.9018890920170628, + "grad_norm": 0.17592693865299225, + "learning_rate": 5e-05, + "loss": 0.7442, + "step": 740 + }, + { + "epoch": 0.9140767824497258, + "grad_norm": 0.2266898900270462, + "learning_rate": 5e-05, + "loss": 0.7326, + "step": 750 + }, + { + "epoch": 0.9262644728823888, + "grad_norm": 0.23541569709777832, + "learning_rate": 5e-05, + "loss": 0.7338, + "step": 760 + }, + { + "epoch": 0.9384521633150518, + "grad_norm": 0.22233888506889343, + "learning_rate": 5e-05, + "loss": 0.7385, + "step": 770 + }, + { + "epoch": 0.9506398537477148, + "grad_norm": 0.23714767396450043, + "learning_rate": 5e-05, + "loss": 0.7306, + "step": 780 + }, + { + "epoch": 0.9628275441803779, + "grad_norm": 0.21191060543060303, + "learning_rate": 5e-05, + "loss": 0.736, + "step": 790 + }, + { + "epoch": 0.9750152346130408, + "grad_norm": 0.22759106755256653, + "learning_rate": 5e-05, + "loss": 0.7505, + "step": 800 + }, + { + "epoch": 0.9872029250457038, + "grad_norm": 0.19895903766155243, + "learning_rate": 5e-05, + "loss": 0.7536, + "step": 810 + }, + { + "epoch": 0.9993906154783668, + "grad_norm": 0.3061685860157013, + "learning_rate": 5e-05, + "loss": 0.7493, + "step": 820 + }, + { + "epoch": 1.0109689213893966, + "grad_norm": 0.18868015706539154, + "learning_rate": 5e-05, + "loss": 0.7342, + "step": 830 + }, + { + "epoch": 1.0231566118220596, + "grad_norm": 0.1797628402709961, + "learning_rate": 5e-05, + "loss": 0.7192, + "step": 840 + }, + { + "epoch": 1.0353443022547226, + "grad_norm": 0.19511596858501434, + "learning_rate": 5e-05, + "loss": 0.7002, + "step": 850 + }, + { + "epoch": 1.0475319926873858, + "grad_norm": 0.2123185396194458, + "learning_rate": 5e-05, + "loss": 0.7236, + "step": 860 + }, + { + "epoch": 1.0597196831200488, + "grad_norm": 0.2047470211982727, + "learning_rate": 5e-05, + "loss": 0.711, + "step": 870 + }, + { + "epoch": 1.0719073735527118, + "grad_norm": 0.18809868395328522, + "learning_rate": 5e-05, + "loss": 0.7253, + "step": 880 + }, + { + "epoch": 1.0840950639853748, + "grad_norm": 0.2207012176513672, + "learning_rate": 5e-05, + "loss": 0.6971, + "step": 890 + }, + { + "epoch": 1.0962827544180378, + "grad_norm": 0.1912035495042801, + "learning_rate": 5e-05, + "loss": 0.7239, + "step": 900 + }, + { + "epoch": 1.1084704448507008, + "grad_norm": 0.23486360907554626, + "learning_rate": 5e-05, + "loss": 0.7335, + "step": 910 + }, + { + "epoch": 1.1206581352833638, + "grad_norm": 0.22189787030220032, + "learning_rate": 5e-05, + "loss": 0.7149, + "step": 920 + }, + { + "epoch": 1.1328458257160268, + "grad_norm": 0.25818824768066406, + "learning_rate": 5e-05, + "loss": 0.7322, + "step": 930 + }, + { + "epoch": 1.1450335161486898, + "grad_norm": 0.2088775783777237, + "learning_rate": 5e-05, + "loss": 0.7382, + "step": 940 + }, + { + "epoch": 1.1572212065813527, + "grad_norm": 0.19445833563804626, + "learning_rate": 5e-05, + "loss": 0.7223, + "step": 950 + }, + { + "epoch": 1.1694088970140157, + "grad_norm": 0.21489456295967102, + "learning_rate": 5e-05, + "loss": 0.7317, + "step": 960 + }, + { + "epoch": 1.181596587446679, + "grad_norm": 0.24050794541835785, + "learning_rate": 5e-05, + "loss": 0.7275, + "step": 970 + }, + { + "epoch": 1.193784277879342, + "grad_norm": 0.18142102658748627, + "learning_rate": 5e-05, + "loss": 0.7046, + "step": 980 + }, + { + "epoch": 1.205971968312005, + "grad_norm": 0.4781077802181244, + "learning_rate": 5e-05, + "loss": 0.7433, + "step": 990 + }, + { + "epoch": 1.218159658744668, + "grad_norm": 0.20628023147583008, + "learning_rate": 5e-05, + "loss": 0.7403, + "step": 1000 + }, + { + "epoch": 1.230347349177331, + "grad_norm": 0.19117140769958496, + "learning_rate": 5e-05, + "loss": 0.7308, + "step": 1010 + }, + { + "epoch": 1.242535039609994, + "grad_norm": 0.2107745110988617, + "learning_rate": 5e-05, + "loss": 0.7201, + "step": 1020 + }, + { + "epoch": 1.2547227300426569, + "grad_norm": 0.20551154017448425, + "learning_rate": 5e-05, + "loss": 0.7038, + "step": 1030 + }, + { + "epoch": 1.2669104204753199, + "grad_norm": 0.26577022671699524, + "learning_rate": 5e-05, + "loss": 0.7096, + "step": 1040 + }, + { + "epoch": 1.2790981109079829, + "grad_norm": 0.22094209492206573, + "learning_rate": 5e-05, + "loss": 0.718, + "step": 1050 + }, + { + "epoch": 1.291285801340646, + "grad_norm": 0.18224076926708221, + "learning_rate": 5e-05, + "loss": 0.7261, + "step": 1060 + }, + { + "epoch": 1.303473491773309, + "grad_norm": 0.226251482963562, + "learning_rate": 5e-05, + "loss": 0.7372, + "step": 1070 + }, + { + "epoch": 1.315661182205972, + "grad_norm": 0.20186461508274078, + "learning_rate": 5e-05, + "loss": 0.7074, + "step": 1080 + }, + { + "epoch": 1.327848872638635, + "grad_norm": 0.22229762375354767, + "learning_rate": 5e-05, + "loss": 0.7069, + "step": 1090 + }, + { + "epoch": 1.340036563071298, + "grad_norm": 0.1776452213525772, + "learning_rate": 5e-05, + "loss": 0.714, + "step": 1100 + }, + { + "epoch": 1.352224253503961, + "grad_norm": 0.20680229365825653, + "learning_rate": 5e-05, + "loss": 0.7066, + "step": 1110 + }, + { + "epoch": 1.364411943936624, + "grad_norm": 0.21842031180858612, + "learning_rate": 5e-05, + "loss": 0.7281, + "step": 1120 + }, + { + "epoch": 1.376599634369287, + "grad_norm": 0.28037774562835693, + "learning_rate": 5e-05, + "loss": 0.7149, + "step": 1130 + }, + { + "epoch": 1.38878732480195, + "grad_norm": 0.20962920784950256, + "learning_rate": 5e-05, + "loss": 0.6951, + "step": 1140 + }, + { + "epoch": 1.400975015234613, + "grad_norm": 0.18751871585845947, + "learning_rate": 5e-05, + "loss": 0.7101, + "step": 1150 + }, + { + "epoch": 1.413162705667276, + "grad_norm": 0.2722964584827423, + "learning_rate": 5e-05, + "loss": 0.7205, + "step": 1160 + }, + { + "epoch": 1.425350396099939, + "grad_norm": 0.20932041108608246, + "learning_rate": 5e-05, + "loss": 0.7082, + "step": 1170 + }, + { + "epoch": 1.437538086532602, + "grad_norm": 0.17836257815361023, + "learning_rate": 5e-05, + "loss": 0.7232, + "step": 1180 + }, + { + "epoch": 1.449725776965265, + "grad_norm": 0.3180703818798065, + "learning_rate": 5e-05, + "loss": 0.7245, + "step": 1190 + }, + { + "epoch": 1.4619134673979282, + "grad_norm": 0.201424703001976, + "learning_rate": 5e-05, + "loss": 0.7232, + "step": 1200 + }, + { + "epoch": 1.4741011578305911, + "grad_norm": 0.20228999853134155, + "learning_rate": 5e-05, + "loss": 0.7169, + "step": 1210 + }, + { + "epoch": 1.4862888482632541, + "grad_norm": 0.17478394508361816, + "learning_rate": 5e-05, + "loss": 0.704, + "step": 1220 + }, + { + "epoch": 1.4984765386959171, + "grad_norm": 0.20061351358890533, + "learning_rate": 5e-05, + "loss": 0.6792, + "step": 1230 + }, + { + "epoch": 1.5106642291285801, + "grad_norm": 0.1857377588748932, + "learning_rate": 5e-05, + "loss": 0.709, + "step": 1240 + }, + { + "epoch": 1.522851919561243, + "grad_norm": 0.19873379170894623, + "learning_rate": 5e-05, + "loss": 0.7007, + "step": 1250 + }, + { + "epoch": 1.5350396099939063, + "grad_norm": 0.19807074964046478, + "learning_rate": 5e-05, + "loss": 0.7122, + "step": 1260 + }, + { + "epoch": 1.5472273004265693, + "grad_norm": 0.16746796667575836, + "learning_rate": 5e-05, + "loss": 0.7005, + "step": 1270 + }, + { + "epoch": 1.5594149908592323, + "grad_norm": 0.2750326991081238, + "learning_rate": 5e-05, + "loss": 0.7166, + "step": 1280 + }, + { + "epoch": 1.5716026812918953, + "grad_norm": 0.2776738107204437, + "learning_rate": 5e-05, + "loss": 0.7405, + "step": 1290 + }, + { + "epoch": 1.5837903717245583, + "grad_norm": 0.3494517207145691, + "learning_rate": 5e-05, + "loss": 0.7192, + "step": 1300 + }, + { + "epoch": 1.5959780621572213, + "grad_norm": 0.25028494000434875, + "learning_rate": 5e-05, + "loss": 0.741, + "step": 1310 + }, + { + "epoch": 1.6081657525898843, + "grad_norm": 0.23914143443107605, + "learning_rate": 5e-05, + "loss": 0.72, + "step": 1320 + }, + { + "epoch": 1.6203534430225472, + "grad_norm": 0.2052862048149109, + "learning_rate": 5e-05, + "loss": 0.7112, + "step": 1330 + }, + { + "epoch": 1.6325411334552102, + "grad_norm": 0.22285936772823334, + "learning_rate": 5e-05, + "loss": 0.7165, + "step": 1340 + }, + { + "epoch": 1.6447288238878732, + "grad_norm": 0.20810848474502563, + "learning_rate": 5e-05, + "loss": 0.7122, + "step": 1350 + }, + { + "epoch": 1.6569165143205362, + "grad_norm": 0.20422884821891785, + "learning_rate": 5e-05, + "loss": 0.7082, + "step": 1360 + }, + { + "epoch": 1.6691042047531992, + "grad_norm": 0.18164314329624176, + "learning_rate": 5e-05, + "loss": 0.7131, + "step": 1370 + }, + { + "epoch": 1.6812918951858622, + "grad_norm": 0.19893014430999756, + "learning_rate": 5e-05, + "loss": 0.7055, + "step": 1380 + }, + { + "epoch": 1.6934795856185252, + "grad_norm": 0.24865971505641937, + "learning_rate": 5e-05, + "loss": 0.6994, + "step": 1390 + }, + { + "epoch": 1.7056672760511882, + "grad_norm": 0.18078123033046722, + "learning_rate": 5e-05, + "loss": 0.7162, + "step": 1400 + }, + { + "epoch": 1.7178549664838512, + "grad_norm": 0.1973421722650528, + "learning_rate": 5e-05, + "loss": 0.7136, + "step": 1410 + }, + { + "epoch": 1.7300426569165142, + "grad_norm": 0.2057911604642868, + "learning_rate": 5e-05, + "loss": 0.7004, + "step": 1420 + }, + { + "epoch": 1.7422303473491774, + "grad_norm": 0.18640495836734772, + "learning_rate": 5e-05, + "loss": 0.7102, + "step": 1430 + }, + { + "epoch": 1.7544180377818404, + "grad_norm": 0.21038417518138885, + "learning_rate": 5e-05, + "loss": 0.6908, + "step": 1440 + }, + { + "epoch": 1.7666057282145033, + "grad_norm": 0.2123890221118927, + "learning_rate": 5e-05, + "loss": 0.6913, + "step": 1450 + }, + { + "epoch": 1.7787934186471663, + "grad_norm": 0.21619991958141327, + "learning_rate": 5e-05, + "loss": 0.7038, + "step": 1460 + }, + { + "epoch": 1.7909811090798293, + "grad_norm": 0.1975220888853073, + "learning_rate": 5e-05, + "loss": 0.7041, + "step": 1470 + }, + { + "epoch": 1.8031687995124925, + "grad_norm": 0.20595207810401917, + "learning_rate": 5e-05, + "loss": 0.7145, + "step": 1480 + }, + { + "epoch": 1.8153564899451555, + "grad_norm": 0.16633334755897522, + "learning_rate": 5e-05, + "loss": 0.6963, + "step": 1490 + }, + { + "epoch": 1.8275441803778185, + "grad_norm": 0.19140620529651642, + "learning_rate": 5e-05, + "loss": 0.6814, + "step": 1500 + }, + { + "epoch": 1.8397318708104815, + "grad_norm": 0.1833905130624771, + "learning_rate": 5e-05, + "loss": 0.7174, + "step": 1510 + }, + { + "epoch": 1.8519195612431445, + "grad_norm": 0.20587337017059326, + "learning_rate": 5e-05, + "loss": 0.7098, + "step": 1520 + }, + { + "epoch": 1.8641072516758075, + "grad_norm": 0.21952909231185913, + "learning_rate": 5e-05, + "loss": 0.6912, + "step": 1530 + }, + { + "epoch": 1.8762949421084705, + "grad_norm": 0.23233620822429657, + "learning_rate": 5e-05, + "loss": 0.6992, + "step": 1540 + }, + { + "epoch": 1.8884826325411335, + "grad_norm": 0.17159809172153473, + "learning_rate": 5e-05, + "loss": 0.7052, + "step": 1550 + }, + { + "epoch": 1.9006703229737965, + "grad_norm": 0.23522134125232697, + "learning_rate": 5e-05, + "loss": 0.6873, + "step": 1560 + }, + { + "epoch": 1.9128580134064594, + "grad_norm": 0.21959669888019562, + "learning_rate": 5e-05, + "loss": 0.6834, + "step": 1570 + }, + { + "epoch": 1.9250457038391224, + "grad_norm": 0.2110578864812851, + "learning_rate": 5e-05, + "loss": 0.7038, + "step": 1580 + }, + { + "epoch": 1.9372333942717854, + "grad_norm": 0.18674372136592865, + "learning_rate": 5e-05, + "loss": 0.6743, + "step": 1590 + }, + { + "epoch": 1.9494210847044484, + "grad_norm": 0.18220046162605286, + "learning_rate": 5e-05, + "loss": 0.6943, + "step": 1600 + }, + { + "epoch": 1.9616087751371114, + "grad_norm": 0.20756569504737854, + "learning_rate": 5e-05, + "loss": 0.7049, + "step": 1610 + }, + { + "epoch": 1.9737964655697744, + "grad_norm": 0.1925211250782013, + "learning_rate": 5e-05, + "loss": 0.7033, + "step": 1620 + }, + { + "epoch": 1.9859841560024374, + "grad_norm": 0.21374447643756866, + "learning_rate": 5e-05, + "loss": 0.7025, + "step": 1630 + }, + { + "epoch": 1.9981718464351006, + "grad_norm": 0.1952880322933197, + "learning_rate": 5e-05, + "loss": 0.6877, + "step": 1640 + } + ], + "logging_steps": 10, + "max_steps": 2463, + "num_input_tokens_seen": 0, + "num_train_epochs": 3, + "save_steps": 205, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 6.729135877101978e+18, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1640/training_args.bin b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1640/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..23ae0ec8a4226be5457f61c137abab2b2c10a0cf --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1640/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56896ea21940e33d7e2934267dff2a5b4a4a80195a0f165245f6617a0f88b90b +size 7377 diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1640/zero_to_fp32.py b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1640/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..5995d6e6f04e43b989587aa9022a3aef0c66d694 --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1640/zero_to_fp32.py @@ -0,0 +1,760 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: +# python zero_to_fp32.py . output_dir/ +# or +# python zero_to_fp32.py . output_dir/ --safe_serialization + +import argparse +import torch +import glob +import math +import os +import re +import gc +import json +import numpy as np +from tqdm import tqdm +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device, weights_only=False) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + total_files = len(files) + state_dicts = [] + for f in tqdm(files, desc='Loading checkpoint shards'): + state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + model_files = get_model_state_files(ds_checkpoint_dir) + + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +class GatheredTensor: + """ + A pseudo tensor that collects partitioned weights. + It is more memory efficient when there are multiple groups. + """ + + def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape): + self.flat_groups = flat_groups + self.flat_groups_offset = flat_groups_offset + self.offset = offset + self.partitioned_numel = partitioned_numel + self.shape = shape + self.dtype = self.flat_groups[0][0].dtype + + def contiguous(self): + """ + Merge partitioned weights from flat_groups into a single tensor. + """ + end_idx = self.offset + self.partitioned_numel + world_size = len(self.flat_groups) + pad_flat_param_chunks = [] + + for rank_i in range(world_size): + # for each rank, we need to collect weights from related group/groups + flat_groups_at_rank_i = self.flat_groups[rank_i] + start_group_id = None + end_group_id = None + for group_id in range(len(self.flat_groups_offset)): + if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]: + start_group_id = group_id + if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]: + end_group_id = group_id + break + # collect weights from related group/groups + for group_id in range(start_group_id, end_group_id + 1): + flat_tensor = flat_groups_at_rank_i[group_id] + start_offset = self.offset - self.flat_groups_offset[group_id] + end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id] + pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset]) + + # collect weights from all ranks + pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0) + param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous() + return param + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size + + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]])) + for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'): + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # memory efficient tensor + tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape) + state_dict[name] = tensor + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def to_torch_tensor(state_dict, return_empty_tensor=False): + """ + Convert state_dict of GatheredTensor to torch tensor + """ + torch_state_dict = {} + converted_tensors = {} + for name, tensor in state_dict.items(): + tensor_id = id(tensor) + if tensor_id in converted_tensors: # shared tensors + shared_tensor = torch_state_dict[converted_tensors[tensor_id]] + torch_state_dict[name] = shared_tensor + else: + converted_tensors[tensor_id] = name + if return_empty_tensor: + torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype) + else: + torch_state_dict[name] = tensor.contiguous() + return torch_state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag=None, + exclude_frozen_parameters=False, + lazy_mode=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient. + Convert the pesduo tensor to torch tensor by ``.contiguous()`` + + Returns: + - pytorch ``state_dict`` + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + Note: the above usage may not work if your application doesn't have sufficient free CPU memory. + You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. Or you can load state_dict in lazy mode :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu + for name, lazy_tensor in state_dict.item(): + tensor = lazy_tensor.contiguous() # to cpu + print(name, tensor) + # del tensor to release memory if it no longer in use + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + if lazy_mode: + return state_dict + else: + return to_torch_tensor(state_dict) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, + output_dir, + max_shard_size="5GB", + safe_serialization=False, + tag=None, + exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_dir``: directory to the pytorch fp32 state_dict output files + - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB + - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + + # Dependency pre-check + if safe_serialization: + try: + from safetensors.torch import save_file + except ImportError: + print('If you want to use `safe_serialization`, please `pip install safetensors`') + raise + if max_shard_size is not None: + try: + from huggingface_hub import split_torch_state_dict_into_shards + except ImportError: + print('If you want to use `max_shard_size`, please `pip install huggingface_hub`') + raise + + # Convert zero checkpoint to state_dict + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag, + exclude_frozen_parameters, + lazy_mode=True) + + # Shard the model if it is too big. + weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin" + if max_shard_size is not None: + filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors") + # an memory-efficient approach for sharding + empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True) + state_dict_split = split_torch_state_dict_into_shards(empty_state_dict, + filename_pattern=filename_pattern, + max_shard_size=max_shard_size) + else: + from collections import namedtuple + StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"]) + state_dict_split = StateDictSplit(is_sharded=False, + filename_to_tensors={weights_name: list(state_dict.keys())}) + + # Save the model by shard + os.makedirs(output_dir, exist_ok=True) + filename_to_tensors = state_dict_split.filename_to_tensors.items() + for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"): + shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors} + shard_state_dict = to_torch_tensor(shard_state_dict) + output_path = os.path.join(output_dir, shard_file) + if safe_serialization: + save_file(shard_state_dict, output_path, metadata={"format": "pt"}) + else: + torch.save(shard_state_dict, output_path) + # release the memory of current shard + for tensor_name in list(shard_state_dict.keys()): + del state_dict[tensor_name] + del shard_state_dict[tensor_name] + del shard_state_dict + gc.collect() + + # Save index if sharded + if state_dict_split.is_sharded: + index = { + "metadata": state_dict_split.metadata, + "weight_map": state_dict_split.tensor_to_filename, + } + save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json" + save_index_file = os.path.join(output_dir, save_index_file) + with open(save_index_file, "w", encoding="utf-8") as f: + content = json.dumps(index, indent=2, sort_keys=True) + "\n" + f.write(content) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info("Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info("Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument("output_dir", + type=str, + help="directory to the pytorch fp32 state_dict output files" + "(e.g. path/checkpoint-12-output/)") + parser.add_argument( + "--max_shard_size", + type=str, + default="5GB", + help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size" + "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`" + "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances" + "without CPU OOM issues.") + parser.add_argument( + "--safe_serialization", + default=False, + action='store_true', + help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_dir, + max_shard_size=args.max_shard_size, + safe_serialization=args.safe_serialization, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1845/config.json b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1845/config.json new file mode 100644 index 0000000000000000000000000000000000000000..18bd58d561964a16b60687f1b04aefdfed0b9b20 --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1845/config.json @@ -0,0 +1,39 @@ +{ + "architectures": [ + "LlamaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "dtype": "bfloat16", + "eos_token_id": [ + 128001, + 128008, + 128009 + ], + "head_dim": 64, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 8192, + "max_position_embeddings": 131072, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 32, + "num_hidden_layers": 16, + "num_key_value_heads": 8, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": { + "factor": 32.0, + "high_freq_factor": 4.0, + "low_freq_factor": 1.0, + "original_max_position_embeddings": 8192, + "rope_type": "llama3" + }, + "rope_theta": 500000.0, + "tie_word_embeddings": true, + "transformers_version": "4.57.1", + "use_cache": false, + "vocab_size": 128256 +} diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1845/generation_config.json b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1845/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..125cc74759bc705ab8a0c48d07d7eac083f56011 --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1845/generation_config.json @@ -0,0 +1,12 @@ +{ + "bos_token_id": 128000, + "do_sample": true, + "eos_token_id": [ + 128001, + 128008, + 128009 + ], + "temperature": 0.6, + "top_p": 0.9, + "transformers_version": "4.57.1" +} diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1845/latest b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1845/latest new file mode 100644 index 0000000000000000000000000000000000000000..241195ab2ceecd960733c23232083ed1cd4663da --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1845/latest @@ -0,0 +1 @@ +global_step1845 \ No newline at end of file diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1845/model.safetensors b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1845/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..962cbe52d54d944814836e6d059ef25864f68c8f --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1845/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c2c95dc323c92307bbea2ca9c79b15a8e18c558f447ae69646eaecd7ca3fec5 +size 2996982344 diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1845/scheduler.pt b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1845/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..51e9499613f9c69889155e7f97e5a8fd4e33cafd --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1845/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:45bf000fd03fbe6a60b9b8cbe4f9d2b1df7587db393fc2d83075b5fc0b286922 +size 1465 diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1845/trainer_state.json b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1845/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..8ae6f58d19ad1026a6074f4c59dc8f77e7f311a9 --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1845/trainer_state.json @@ -0,0 +1,1322 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 2.247410115783059, + "eval_steps": 500, + "global_step": 1845, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.01218769043266301, + "grad_norm": 0.36486390233039856, + "learning_rate": 5e-05, + "loss": 1.2213, + "step": 10 + }, + { + "epoch": 0.02437538086532602, + "grad_norm": 0.2953564524650574, + "learning_rate": 5e-05, + "loss": 1.0223, + "step": 20 + }, + { + "epoch": 0.03656307129798903, + "grad_norm": 0.2766735851764679, + "learning_rate": 5e-05, + "loss": 0.9589, + "step": 30 + }, + { + "epoch": 0.04875076173065204, + "grad_norm": 0.32770997285842896, + "learning_rate": 5e-05, + "loss": 0.9081, + "step": 40 + }, + { + "epoch": 0.06093845216331505, + "grad_norm": 0.3052276372909546, + "learning_rate": 5e-05, + "loss": 0.8752, + "step": 50 + }, + { + "epoch": 0.07312614259597806, + "grad_norm": 0.240146666765213, + "learning_rate": 5e-05, + "loss": 0.873, + "step": 60 + }, + { + "epoch": 0.08531383302864107, + "grad_norm": 0.23872187733650208, + "learning_rate": 5e-05, + "loss": 0.8587, + "step": 70 + }, + { + "epoch": 0.09750152346130408, + "grad_norm": 0.32638487219810486, + "learning_rate": 5e-05, + "loss": 0.8656, + "step": 80 + }, + { + "epoch": 0.10968921389396709, + "grad_norm": 0.25019028782844543, + "learning_rate": 5e-05, + "loss": 0.8685, + "step": 90 + }, + { + "epoch": 0.1218769043266301, + "grad_norm": 0.2507675886154175, + "learning_rate": 5e-05, + "loss": 0.8662, + "step": 100 + }, + { + "epoch": 0.1340645947592931, + "grad_norm": 0.27966755628585815, + "learning_rate": 5e-05, + "loss": 0.8428, + "step": 110 + }, + { + "epoch": 0.14625228519195613, + "grad_norm": 0.2622673213481903, + "learning_rate": 5e-05, + "loss": 0.8416, + "step": 120 + }, + { + "epoch": 0.15843997562461914, + "grad_norm": 0.3003966510295868, + "learning_rate": 5e-05, + "loss": 0.8528, + "step": 130 + }, + { + "epoch": 0.17062766605728213, + "grad_norm": 0.32308319211006165, + "learning_rate": 5e-05, + "loss": 0.8357, + "step": 140 + }, + { + "epoch": 0.18281535648994515, + "grad_norm": 0.27279096841812134, + "learning_rate": 5e-05, + "loss": 0.8432, + "step": 150 + }, + { + "epoch": 0.19500304692260817, + "grad_norm": 0.27976104617118835, + "learning_rate": 5e-05, + "loss": 0.8119, + "step": 160 + }, + { + "epoch": 0.2071907373552712, + "grad_norm": 0.290866881608963, + "learning_rate": 5e-05, + "loss": 0.8096, + "step": 170 + }, + { + "epoch": 0.21937842778793418, + "grad_norm": 0.29430949687957764, + "learning_rate": 5e-05, + "loss": 0.819, + "step": 180 + }, + { + "epoch": 0.2315661182205972, + "grad_norm": 0.27714136242866516, + "learning_rate": 5e-05, + "loss": 0.8187, + "step": 190 + }, + { + "epoch": 0.2437538086532602, + "grad_norm": 0.2997029423713684, + "learning_rate": 5e-05, + "loss": 0.808, + "step": 200 + }, + { + "epoch": 0.25594149908592323, + "grad_norm": 0.2539573609828949, + "learning_rate": 5e-05, + "loss": 0.7943, + "step": 210 + }, + { + "epoch": 0.2681291895185862, + "grad_norm": 0.23294171690940857, + "learning_rate": 5e-05, + "loss": 0.8188, + "step": 220 + }, + { + "epoch": 0.28031687995124926, + "grad_norm": 0.25016292929649353, + "learning_rate": 5e-05, + "loss": 0.8035, + "step": 230 + }, + { + "epoch": 0.29250457038391225, + "grad_norm": 0.37125322222709656, + "learning_rate": 5e-05, + "loss": 0.785, + "step": 240 + }, + { + "epoch": 0.30469226081657524, + "grad_norm": 0.21520452201366425, + "learning_rate": 5e-05, + "loss": 0.8088, + "step": 250 + }, + { + "epoch": 0.3168799512492383, + "grad_norm": 0.20990602672100067, + "learning_rate": 5e-05, + "loss": 0.7813, + "step": 260 + }, + { + "epoch": 0.3290676416819013, + "grad_norm": 0.24193961918354034, + "learning_rate": 5e-05, + "loss": 0.7979, + "step": 270 + }, + { + "epoch": 0.34125533211456427, + "grad_norm": 0.22766050696372986, + "learning_rate": 5e-05, + "loss": 0.785, + "step": 280 + }, + { + "epoch": 0.3534430225472273, + "grad_norm": 0.2662702202796936, + "learning_rate": 5e-05, + "loss": 0.7766, + "step": 290 + }, + { + "epoch": 0.3656307129798903, + "grad_norm": 0.23020495474338531, + "learning_rate": 5e-05, + "loss": 0.7687, + "step": 300 + }, + { + "epoch": 0.3778184034125533, + "grad_norm": 0.21770520508289337, + "learning_rate": 5e-05, + "loss": 0.7815, + "step": 310 + }, + { + "epoch": 0.39000609384521634, + "grad_norm": 0.22677403688430786, + "learning_rate": 5e-05, + "loss": 0.7999, + "step": 320 + }, + { + "epoch": 0.40219378427787933, + "grad_norm": 0.24891860783100128, + "learning_rate": 5e-05, + "loss": 0.7946, + "step": 330 + }, + { + "epoch": 0.4143814747105424, + "grad_norm": 0.2506854832172394, + "learning_rate": 5e-05, + "loss": 0.7889, + "step": 340 + }, + { + "epoch": 0.42656916514320536, + "grad_norm": 0.24388180673122406, + "learning_rate": 5e-05, + "loss": 0.79, + "step": 350 + }, + { + "epoch": 0.43875685557586835, + "grad_norm": 0.22329650819301605, + "learning_rate": 5e-05, + "loss": 0.7632, + "step": 360 + }, + { + "epoch": 0.4509445460085314, + "grad_norm": 0.2240927815437317, + "learning_rate": 5e-05, + "loss": 0.8042, + "step": 370 + }, + { + "epoch": 0.4631322364411944, + "grad_norm": 0.2578203082084656, + "learning_rate": 5e-05, + "loss": 0.7667, + "step": 380 + }, + { + "epoch": 0.4753199268738574, + "grad_norm": 0.25349217653274536, + "learning_rate": 5e-05, + "loss": 0.7911, + "step": 390 + }, + { + "epoch": 0.4875076173065204, + "grad_norm": 0.254446804523468, + "learning_rate": 5e-05, + "loss": 0.7909, + "step": 400 + }, + { + "epoch": 0.4996953077391834, + "grad_norm": 0.24253171682357788, + "learning_rate": 5e-05, + "loss": 0.7896, + "step": 410 + }, + { + "epoch": 0.5118829981718465, + "grad_norm": 0.27901458740234375, + "learning_rate": 5e-05, + "loss": 0.7828, + "step": 420 + }, + { + "epoch": 0.5240706886045094, + "grad_norm": 0.31884434819221497, + "learning_rate": 5e-05, + "loss": 0.7919, + "step": 430 + }, + { + "epoch": 0.5362583790371724, + "grad_norm": 0.22920799255371094, + "learning_rate": 5e-05, + "loss": 0.7777, + "step": 440 + }, + { + "epoch": 0.5484460694698354, + "grad_norm": 0.22606618702411652, + "learning_rate": 5e-05, + "loss": 0.7661, + "step": 450 + }, + { + "epoch": 0.5606337599024985, + "grad_norm": 0.21604052186012268, + "learning_rate": 5e-05, + "loss": 0.769, + "step": 460 + }, + { + "epoch": 0.5728214503351615, + "grad_norm": 0.2296101152896881, + "learning_rate": 5e-05, + "loss": 0.7626, + "step": 470 + }, + { + "epoch": 0.5850091407678245, + "grad_norm": 0.24162150919437408, + "learning_rate": 5e-05, + "loss": 0.7831, + "step": 480 + }, + { + "epoch": 0.5971968312004875, + "grad_norm": 0.22364799678325653, + "learning_rate": 5e-05, + "loss": 0.7737, + "step": 490 + }, + { + "epoch": 0.6093845216331505, + "grad_norm": 0.23953403532505035, + "learning_rate": 5e-05, + "loss": 0.787, + "step": 500 + }, + { + "epoch": 0.6215722120658135, + "grad_norm": 0.22349311411380768, + "learning_rate": 5e-05, + "loss": 0.7763, + "step": 510 + }, + { + "epoch": 0.6337599024984766, + "grad_norm": 0.21996377408504486, + "learning_rate": 5e-05, + "loss": 0.7596, + "step": 520 + }, + { + "epoch": 0.6459475929311396, + "grad_norm": 0.245538130402565, + "learning_rate": 5e-05, + "loss": 0.7785, + "step": 530 + }, + { + "epoch": 0.6581352833638026, + "grad_norm": 0.23442520201206207, + "learning_rate": 5e-05, + "loss": 0.7633, + "step": 540 + }, + { + "epoch": 0.6703229737964655, + "grad_norm": 0.22274106740951538, + "learning_rate": 5e-05, + "loss": 0.7756, + "step": 550 + }, + { + "epoch": 0.6825106642291285, + "grad_norm": 0.21717530488967896, + "learning_rate": 5e-05, + "loss": 0.7726, + "step": 560 + }, + { + "epoch": 0.6946983546617916, + "grad_norm": 0.2465955913066864, + "learning_rate": 5e-05, + "loss": 0.7476, + "step": 570 + }, + { + "epoch": 0.7068860450944546, + "grad_norm": 0.2318575084209442, + "learning_rate": 5e-05, + "loss": 0.7584, + "step": 580 + }, + { + "epoch": 0.7190737355271176, + "grad_norm": 0.25114378333091736, + "learning_rate": 5e-05, + "loss": 0.7625, + "step": 590 + }, + { + "epoch": 0.7312614259597806, + "grad_norm": 0.19145755469799042, + "learning_rate": 5e-05, + "loss": 0.7577, + "step": 600 + }, + { + "epoch": 0.7434491163924436, + "grad_norm": 0.19129066169261932, + "learning_rate": 5e-05, + "loss": 0.7772, + "step": 610 + }, + { + "epoch": 0.7556368068251066, + "grad_norm": 0.2014475166797638, + "learning_rate": 5e-05, + "loss": 0.7601, + "step": 620 + }, + { + "epoch": 0.7678244972577697, + "grad_norm": 0.21480509638786316, + "learning_rate": 5e-05, + "loss": 0.7665, + "step": 630 + }, + { + "epoch": 0.7800121876904327, + "grad_norm": 0.2633228302001953, + "learning_rate": 5e-05, + "loss": 0.7565, + "step": 640 + }, + { + "epoch": 0.7921998781230957, + "grad_norm": 0.24633203446865082, + "learning_rate": 5e-05, + "loss": 0.7602, + "step": 650 + }, + { + "epoch": 0.8043875685557587, + "grad_norm": 0.19770534336566925, + "learning_rate": 5e-05, + "loss": 0.7685, + "step": 660 + }, + { + "epoch": 0.8165752589884216, + "grad_norm": 0.2267930805683136, + "learning_rate": 5e-05, + "loss": 0.7516, + "step": 670 + }, + { + "epoch": 0.8287629494210847, + "grad_norm": 0.238111674785614, + "learning_rate": 5e-05, + "loss": 0.7487, + "step": 680 + }, + { + "epoch": 0.8409506398537477, + "grad_norm": 0.19528904557228088, + "learning_rate": 5e-05, + "loss": 0.7467, + "step": 690 + }, + { + "epoch": 0.8531383302864107, + "grad_norm": 0.21305914223194122, + "learning_rate": 5e-05, + "loss": 0.7527, + "step": 700 + }, + { + "epoch": 0.8653260207190737, + "grad_norm": 0.2205205112695694, + "learning_rate": 5e-05, + "loss": 0.7639, + "step": 710 + }, + { + "epoch": 0.8775137111517367, + "grad_norm": 0.22429127991199493, + "learning_rate": 5e-05, + "loss": 0.7744, + "step": 720 + }, + { + "epoch": 0.8897014015843998, + "grad_norm": 0.23755772411823273, + "learning_rate": 5e-05, + "loss": 0.7446, + "step": 730 + }, + { + "epoch": 0.9018890920170628, + "grad_norm": 0.17592693865299225, + "learning_rate": 5e-05, + "loss": 0.7442, + "step": 740 + }, + { + "epoch": 0.9140767824497258, + "grad_norm": 0.2266898900270462, + "learning_rate": 5e-05, + "loss": 0.7326, + "step": 750 + }, + { + "epoch": 0.9262644728823888, + "grad_norm": 0.23541569709777832, + "learning_rate": 5e-05, + "loss": 0.7338, + "step": 760 + }, + { + "epoch": 0.9384521633150518, + "grad_norm": 0.22233888506889343, + "learning_rate": 5e-05, + "loss": 0.7385, + "step": 770 + }, + { + "epoch": 0.9506398537477148, + "grad_norm": 0.23714767396450043, + "learning_rate": 5e-05, + "loss": 0.7306, + "step": 780 + }, + { + "epoch": 0.9628275441803779, + "grad_norm": 0.21191060543060303, + "learning_rate": 5e-05, + "loss": 0.736, + "step": 790 + }, + { + "epoch": 0.9750152346130408, + "grad_norm": 0.22759106755256653, + "learning_rate": 5e-05, + "loss": 0.7505, + "step": 800 + }, + { + "epoch": 0.9872029250457038, + "grad_norm": 0.19895903766155243, + "learning_rate": 5e-05, + "loss": 0.7536, + "step": 810 + }, + { + "epoch": 0.9993906154783668, + "grad_norm": 0.3061685860157013, + "learning_rate": 5e-05, + "loss": 0.7493, + "step": 820 + }, + { + "epoch": 1.0109689213893966, + "grad_norm": 0.18868015706539154, + "learning_rate": 5e-05, + "loss": 0.7342, + "step": 830 + }, + { + "epoch": 1.0231566118220596, + "grad_norm": 0.1797628402709961, + "learning_rate": 5e-05, + "loss": 0.7192, + "step": 840 + }, + { + "epoch": 1.0353443022547226, + "grad_norm": 0.19511596858501434, + "learning_rate": 5e-05, + "loss": 0.7002, + "step": 850 + }, + { + "epoch": 1.0475319926873858, + "grad_norm": 0.2123185396194458, + "learning_rate": 5e-05, + "loss": 0.7236, + "step": 860 + }, + { + "epoch": 1.0597196831200488, + "grad_norm": 0.2047470211982727, + "learning_rate": 5e-05, + "loss": 0.711, + "step": 870 + }, + { + "epoch": 1.0719073735527118, + "grad_norm": 0.18809868395328522, + "learning_rate": 5e-05, + "loss": 0.7253, + "step": 880 + }, + { + "epoch": 1.0840950639853748, + "grad_norm": 0.2207012176513672, + "learning_rate": 5e-05, + "loss": 0.6971, + "step": 890 + }, + { + "epoch": 1.0962827544180378, + "grad_norm": 0.1912035495042801, + "learning_rate": 5e-05, + "loss": 0.7239, + "step": 900 + }, + { + "epoch": 1.1084704448507008, + "grad_norm": 0.23486360907554626, + "learning_rate": 5e-05, + "loss": 0.7335, + "step": 910 + }, + { + "epoch": 1.1206581352833638, + "grad_norm": 0.22189787030220032, + "learning_rate": 5e-05, + "loss": 0.7149, + "step": 920 + }, + { + "epoch": 1.1328458257160268, + "grad_norm": 0.25818824768066406, + "learning_rate": 5e-05, + "loss": 0.7322, + "step": 930 + }, + { + "epoch": 1.1450335161486898, + "grad_norm": 0.2088775783777237, + "learning_rate": 5e-05, + "loss": 0.7382, + "step": 940 + }, + { + "epoch": 1.1572212065813527, + "grad_norm": 0.19445833563804626, + "learning_rate": 5e-05, + "loss": 0.7223, + "step": 950 + }, + { + "epoch": 1.1694088970140157, + "grad_norm": 0.21489456295967102, + "learning_rate": 5e-05, + "loss": 0.7317, + "step": 960 + }, + { + "epoch": 1.181596587446679, + "grad_norm": 0.24050794541835785, + "learning_rate": 5e-05, + "loss": 0.7275, + "step": 970 + }, + { + "epoch": 1.193784277879342, + "grad_norm": 0.18142102658748627, + "learning_rate": 5e-05, + "loss": 0.7046, + "step": 980 + }, + { + "epoch": 1.205971968312005, + "grad_norm": 0.4781077802181244, + "learning_rate": 5e-05, + "loss": 0.7433, + "step": 990 + }, + { + "epoch": 1.218159658744668, + "grad_norm": 0.20628023147583008, + "learning_rate": 5e-05, + "loss": 0.7403, + "step": 1000 + }, + { + "epoch": 1.230347349177331, + "grad_norm": 0.19117140769958496, + "learning_rate": 5e-05, + "loss": 0.7308, + "step": 1010 + }, + { + "epoch": 1.242535039609994, + "grad_norm": 0.2107745110988617, + "learning_rate": 5e-05, + "loss": 0.7201, + "step": 1020 + }, + { + "epoch": 1.2547227300426569, + "grad_norm": 0.20551154017448425, + "learning_rate": 5e-05, + "loss": 0.7038, + "step": 1030 + }, + { + "epoch": 1.2669104204753199, + "grad_norm": 0.26577022671699524, + "learning_rate": 5e-05, + "loss": 0.7096, + "step": 1040 + }, + { + "epoch": 1.2790981109079829, + "grad_norm": 0.22094209492206573, + "learning_rate": 5e-05, + "loss": 0.718, + "step": 1050 + }, + { + "epoch": 1.291285801340646, + "grad_norm": 0.18224076926708221, + "learning_rate": 5e-05, + "loss": 0.7261, + "step": 1060 + }, + { + "epoch": 1.303473491773309, + "grad_norm": 0.226251482963562, + "learning_rate": 5e-05, + "loss": 0.7372, + "step": 1070 + }, + { + "epoch": 1.315661182205972, + "grad_norm": 0.20186461508274078, + "learning_rate": 5e-05, + "loss": 0.7074, + "step": 1080 + }, + { + "epoch": 1.327848872638635, + "grad_norm": 0.22229762375354767, + "learning_rate": 5e-05, + "loss": 0.7069, + "step": 1090 + }, + { + "epoch": 1.340036563071298, + "grad_norm": 0.1776452213525772, + "learning_rate": 5e-05, + "loss": 0.714, + "step": 1100 + }, + { + "epoch": 1.352224253503961, + "grad_norm": 0.20680229365825653, + "learning_rate": 5e-05, + "loss": 0.7066, + "step": 1110 + }, + { + "epoch": 1.364411943936624, + "grad_norm": 0.21842031180858612, + "learning_rate": 5e-05, + "loss": 0.7281, + "step": 1120 + }, + { + "epoch": 1.376599634369287, + "grad_norm": 0.28037774562835693, + "learning_rate": 5e-05, + "loss": 0.7149, + "step": 1130 + }, + { + "epoch": 1.38878732480195, + "grad_norm": 0.20962920784950256, + "learning_rate": 5e-05, + "loss": 0.6951, + "step": 1140 + }, + { + "epoch": 1.400975015234613, + "grad_norm": 0.18751871585845947, + "learning_rate": 5e-05, + "loss": 0.7101, + "step": 1150 + }, + { + "epoch": 1.413162705667276, + "grad_norm": 0.2722964584827423, + "learning_rate": 5e-05, + "loss": 0.7205, + "step": 1160 + }, + { + "epoch": 1.425350396099939, + "grad_norm": 0.20932041108608246, + "learning_rate": 5e-05, + "loss": 0.7082, + "step": 1170 + }, + { + "epoch": 1.437538086532602, + "grad_norm": 0.17836257815361023, + "learning_rate": 5e-05, + "loss": 0.7232, + "step": 1180 + }, + { + "epoch": 1.449725776965265, + "grad_norm": 0.3180703818798065, + "learning_rate": 5e-05, + "loss": 0.7245, + "step": 1190 + }, + { + "epoch": 1.4619134673979282, + "grad_norm": 0.201424703001976, + "learning_rate": 5e-05, + "loss": 0.7232, + "step": 1200 + }, + { + "epoch": 1.4741011578305911, + "grad_norm": 0.20228999853134155, + "learning_rate": 5e-05, + "loss": 0.7169, + "step": 1210 + }, + { + "epoch": 1.4862888482632541, + "grad_norm": 0.17478394508361816, + "learning_rate": 5e-05, + "loss": 0.704, + "step": 1220 + }, + { + "epoch": 1.4984765386959171, + "grad_norm": 0.20061351358890533, + "learning_rate": 5e-05, + "loss": 0.6792, + "step": 1230 + }, + { + "epoch": 1.5106642291285801, + "grad_norm": 0.1857377588748932, + "learning_rate": 5e-05, + "loss": 0.709, + "step": 1240 + }, + { + "epoch": 1.522851919561243, + "grad_norm": 0.19873379170894623, + "learning_rate": 5e-05, + "loss": 0.7007, + "step": 1250 + }, + { + "epoch": 1.5350396099939063, + "grad_norm": 0.19807074964046478, + "learning_rate": 5e-05, + "loss": 0.7122, + "step": 1260 + }, + { + "epoch": 1.5472273004265693, + "grad_norm": 0.16746796667575836, + "learning_rate": 5e-05, + "loss": 0.7005, + "step": 1270 + }, + { + "epoch": 1.5594149908592323, + "grad_norm": 0.2750326991081238, + "learning_rate": 5e-05, + "loss": 0.7166, + "step": 1280 + }, + { + "epoch": 1.5716026812918953, + "grad_norm": 0.2776738107204437, + "learning_rate": 5e-05, + "loss": 0.7405, + "step": 1290 + }, + { + "epoch": 1.5837903717245583, + "grad_norm": 0.3494517207145691, + "learning_rate": 5e-05, + "loss": 0.7192, + "step": 1300 + }, + { + "epoch": 1.5959780621572213, + "grad_norm": 0.25028494000434875, + "learning_rate": 5e-05, + "loss": 0.741, + "step": 1310 + }, + { + "epoch": 1.6081657525898843, + "grad_norm": 0.23914143443107605, + "learning_rate": 5e-05, + "loss": 0.72, + "step": 1320 + }, + { + "epoch": 1.6203534430225472, + "grad_norm": 0.2052862048149109, + "learning_rate": 5e-05, + "loss": 0.7112, + "step": 1330 + }, + { + "epoch": 1.6325411334552102, + "grad_norm": 0.22285936772823334, + "learning_rate": 5e-05, + "loss": 0.7165, + "step": 1340 + }, + { + "epoch": 1.6447288238878732, + "grad_norm": 0.20810848474502563, + "learning_rate": 5e-05, + "loss": 0.7122, + "step": 1350 + }, + { + "epoch": 1.6569165143205362, + "grad_norm": 0.20422884821891785, + "learning_rate": 5e-05, + "loss": 0.7082, + "step": 1360 + }, + { + "epoch": 1.6691042047531992, + "grad_norm": 0.18164314329624176, + "learning_rate": 5e-05, + "loss": 0.7131, + "step": 1370 + }, + { + "epoch": 1.6812918951858622, + "grad_norm": 0.19893014430999756, + "learning_rate": 5e-05, + "loss": 0.7055, + "step": 1380 + }, + { + "epoch": 1.6934795856185252, + "grad_norm": 0.24865971505641937, + "learning_rate": 5e-05, + "loss": 0.6994, + "step": 1390 + }, + { + "epoch": 1.7056672760511882, + "grad_norm": 0.18078123033046722, + "learning_rate": 5e-05, + "loss": 0.7162, + "step": 1400 + }, + { + "epoch": 1.7178549664838512, + "grad_norm": 0.1973421722650528, + "learning_rate": 5e-05, + "loss": 0.7136, + "step": 1410 + }, + { + "epoch": 1.7300426569165142, + "grad_norm": 0.2057911604642868, + "learning_rate": 5e-05, + "loss": 0.7004, + "step": 1420 + }, + { + "epoch": 1.7422303473491774, + "grad_norm": 0.18640495836734772, + "learning_rate": 5e-05, + "loss": 0.7102, + "step": 1430 + }, + { + "epoch": 1.7544180377818404, + "grad_norm": 0.21038417518138885, + "learning_rate": 5e-05, + "loss": 0.6908, + "step": 1440 + }, + { + "epoch": 1.7666057282145033, + "grad_norm": 0.2123890221118927, + "learning_rate": 5e-05, + "loss": 0.6913, + "step": 1450 + }, + { + "epoch": 1.7787934186471663, + "grad_norm": 0.21619991958141327, + "learning_rate": 5e-05, + "loss": 0.7038, + "step": 1460 + }, + { + "epoch": 1.7909811090798293, + "grad_norm": 0.1975220888853073, + "learning_rate": 5e-05, + "loss": 0.7041, + "step": 1470 + }, + { + "epoch": 1.8031687995124925, + "grad_norm": 0.20595207810401917, + "learning_rate": 5e-05, + "loss": 0.7145, + "step": 1480 + }, + { + "epoch": 1.8153564899451555, + "grad_norm": 0.16633334755897522, + "learning_rate": 5e-05, + "loss": 0.6963, + "step": 1490 + }, + { + "epoch": 1.8275441803778185, + "grad_norm": 0.19140620529651642, + "learning_rate": 5e-05, + "loss": 0.6814, + "step": 1500 + }, + { + "epoch": 1.8397318708104815, + "grad_norm": 0.1833905130624771, + "learning_rate": 5e-05, + "loss": 0.7174, + "step": 1510 + }, + { + "epoch": 1.8519195612431445, + "grad_norm": 0.20587337017059326, + "learning_rate": 5e-05, + "loss": 0.7098, + "step": 1520 + }, + { + "epoch": 1.8641072516758075, + "grad_norm": 0.21952909231185913, + "learning_rate": 5e-05, + "loss": 0.6912, + "step": 1530 + }, + { + "epoch": 1.8762949421084705, + "grad_norm": 0.23233620822429657, + "learning_rate": 5e-05, + "loss": 0.6992, + "step": 1540 + }, + { + "epoch": 1.8884826325411335, + "grad_norm": 0.17159809172153473, + "learning_rate": 5e-05, + "loss": 0.7052, + "step": 1550 + }, + { + "epoch": 1.9006703229737965, + "grad_norm": 0.23522134125232697, + "learning_rate": 5e-05, + "loss": 0.6873, + "step": 1560 + }, + { + "epoch": 1.9128580134064594, + "grad_norm": 0.21959669888019562, + "learning_rate": 5e-05, + "loss": 0.6834, + "step": 1570 + }, + { + "epoch": 1.9250457038391224, + "grad_norm": 0.2110578864812851, + "learning_rate": 5e-05, + "loss": 0.7038, + "step": 1580 + }, + { + "epoch": 1.9372333942717854, + "grad_norm": 0.18674372136592865, + "learning_rate": 5e-05, + "loss": 0.6743, + "step": 1590 + }, + { + "epoch": 1.9494210847044484, + "grad_norm": 0.18220046162605286, + "learning_rate": 5e-05, + "loss": 0.6943, + "step": 1600 + }, + { + "epoch": 1.9616087751371114, + "grad_norm": 0.20756569504737854, + "learning_rate": 5e-05, + "loss": 0.7049, + "step": 1610 + }, + { + "epoch": 1.9737964655697744, + "grad_norm": 0.1925211250782013, + "learning_rate": 5e-05, + "loss": 0.7033, + "step": 1620 + }, + { + "epoch": 1.9859841560024374, + "grad_norm": 0.21374447643756866, + "learning_rate": 5e-05, + "loss": 0.7025, + "step": 1630 + }, + { + "epoch": 1.9981718464351006, + "grad_norm": 0.1952880322933197, + "learning_rate": 5e-05, + "loss": 0.6877, + "step": 1640 + }, + { + "epoch": 2.0097501523461303, + "grad_norm": 0.20457723736763, + "learning_rate": 5e-05, + "loss": 0.6734, + "step": 1650 + }, + { + "epoch": 2.0219378427787933, + "grad_norm": 0.20965932309627533, + "learning_rate": 5e-05, + "loss": 0.6472, + "step": 1660 + }, + { + "epoch": 2.0341255332114563, + "grad_norm": 0.19478890299797058, + "learning_rate": 5e-05, + "loss": 0.6781, + "step": 1670 + }, + { + "epoch": 2.0463132236441193, + "grad_norm": 0.21520769596099854, + "learning_rate": 5e-05, + "loss": 0.6592, + "step": 1680 + }, + { + "epoch": 2.0585009140767823, + "grad_norm": 0.20154155790805817, + "learning_rate": 5e-05, + "loss": 0.6744, + "step": 1690 + }, + { + "epoch": 2.0706886045094453, + "grad_norm": 0.1899706870317459, + "learning_rate": 5e-05, + "loss": 0.6592, + "step": 1700 + }, + { + "epoch": 2.0828762949421087, + "grad_norm": 0.20379123091697693, + "learning_rate": 5e-05, + "loss": 0.6688, + "step": 1710 + }, + { + "epoch": 2.0950639853747717, + "grad_norm": 0.19799865782260895, + "learning_rate": 5e-05, + "loss": 0.6678, + "step": 1720 + }, + { + "epoch": 2.1072516758074347, + "grad_norm": 0.19186025857925415, + "learning_rate": 5e-05, + "loss": 0.6587, + "step": 1730 + }, + { + "epoch": 2.1194393662400977, + "grad_norm": 0.18936173617839813, + "learning_rate": 5e-05, + "loss": 0.668, + "step": 1740 + }, + { + "epoch": 2.1316270566727606, + "grad_norm": 0.19159527122974396, + "learning_rate": 5e-05, + "loss": 0.6643, + "step": 1750 + }, + { + "epoch": 2.1438147471054236, + "grad_norm": 0.20596368610858917, + "learning_rate": 5e-05, + "loss": 0.6667, + "step": 1760 + }, + { + "epoch": 2.1560024375380866, + "grad_norm": 0.18601712584495544, + "learning_rate": 5e-05, + "loss": 0.6598, + "step": 1770 + }, + { + "epoch": 2.1681901279707496, + "grad_norm": 0.2130655199289322, + "learning_rate": 5e-05, + "loss": 0.6844, + "step": 1780 + }, + { + "epoch": 2.1803778184034126, + "grad_norm": 0.20477338135242462, + "learning_rate": 5e-05, + "loss": 0.666, + "step": 1790 + }, + { + "epoch": 2.1925655088360756, + "grad_norm": 0.18484896421432495, + "learning_rate": 5e-05, + "loss": 0.6659, + "step": 1800 + }, + { + "epoch": 2.2047531992687386, + "grad_norm": 0.17023979127407074, + "learning_rate": 5e-05, + "loss": 0.6721, + "step": 1810 + }, + { + "epoch": 2.2169408897014016, + "grad_norm": 0.23462794721126556, + "learning_rate": 5e-05, + "loss": 0.6699, + "step": 1820 + }, + { + "epoch": 2.2291285801340646, + "grad_norm": 0.16521663963794708, + "learning_rate": 5e-05, + "loss": 0.6584, + "step": 1830 + }, + { + "epoch": 2.2413162705667276, + "grad_norm": 0.19637759029865265, + "learning_rate": 5e-05, + "loss": 0.6621, + "step": 1840 + } + ], + "logging_steps": 10, + "max_steps": 2463, + "num_input_tokens_seen": 0, + "num_train_epochs": 3, + "save_steps": 205, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 7.565604904304116e+18, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1845/training_args.bin b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1845/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..23ae0ec8a4226be5457f61c137abab2b2c10a0cf --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1845/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56896ea21940e33d7e2934267dff2a5b4a4a80195a0f165245f6617a0f88b90b +size 7377 diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1845/zero_to_fp32.py b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1845/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..5995d6e6f04e43b989587aa9022a3aef0c66d694 --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-1845/zero_to_fp32.py @@ -0,0 +1,760 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: +# python zero_to_fp32.py . output_dir/ +# or +# python zero_to_fp32.py . output_dir/ --safe_serialization + +import argparse +import torch +import glob +import math +import os +import re +import gc +import json +import numpy as np +from tqdm import tqdm +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device, weights_only=False) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + total_files = len(files) + state_dicts = [] + for f in tqdm(files, desc='Loading checkpoint shards'): + state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + model_files = get_model_state_files(ds_checkpoint_dir) + + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +class GatheredTensor: + """ + A pseudo tensor that collects partitioned weights. + It is more memory efficient when there are multiple groups. + """ + + def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape): + self.flat_groups = flat_groups + self.flat_groups_offset = flat_groups_offset + self.offset = offset + self.partitioned_numel = partitioned_numel + self.shape = shape + self.dtype = self.flat_groups[0][0].dtype + + def contiguous(self): + """ + Merge partitioned weights from flat_groups into a single tensor. + """ + end_idx = self.offset + self.partitioned_numel + world_size = len(self.flat_groups) + pad_flat_param_chunks = [] + + for rank_i in range(world_size): + # for each rank, we need to collect weights from related group/groups + flat_groups_at_rank_i = self.flat_groups[rank_i] + start_group_id = None + end_group_id = None + for group_id in range(len(self.flat_groups_offset)): + if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]: + start_group_id = group_id + if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]: + end_group_id = group_id + break + # collect weights from related group/groups + for group_id in range(start_group_id, end_group_id + 1): + flat_tensor = flat_groups_at_rank_i[group_id] + start_offset = self.offset - self.flat_groups_offset[group_id] + end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id] + pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset]) + + # collect weights from all ranks + pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0) + param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous() + return param + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size + + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]])) + for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'): + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # memory efficient tensor + tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape) + state_dict[name] = tensor + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def to_torch_tensor(state_dict, return_empty_tensor=False): + """ + Convert state_dict of GatheredTensor to torch tensor + """ + torch_state_dict = {} + converted_tensors = {} + for name, tensor in state_dict.items(): + tensor_id = id(tensor) + if tensor_id in converted_tensors: # shared tensors + shared_tensor = torch_state_dict[converted_tensors[tensor_id]] + torch_state_dict[name] = shared_tensor + else: + converted_tensors[tensor_id] = name + if return_empty_tensor: + torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype) + else: + torch_state_dict[name] = tensor.contiguous() + return torch_state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag=None, + exclude_frozen_parameters=False, + lazy_mode=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient. + Convert the pesduo tensor to torch tensor by ``.contiguous()`` + + Returns: + - pytorch ``state_dict`` + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + Note: the above usage may not work if your application doesn't have sufficient free CPU memory. + You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. Or you can load state_dict in lazy mode :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu + for name, lazy_tensor in state_dict.item(): + tensor = lazy_tensor.contiguous() # to cpu + print(name, tensor) + # del tensor to release memory if it no longer in use + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + if lazy_mode: + return state_dict + else: + return to_torch_tensor(state_dict) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, + output_dir, + max_shard_size="5GB", + safe_serialization=False, + tag=None, + exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_dir``: directory to the pytorch fp32 state_dict output files + - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB + - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + + # Dependency pre-check + if safe_serialization: + try: + from safetensors.torch import save_file + except ImportError: + print('If you want to use `safe_serialization`, please `pip install safetensors`') + raise + if max_shard_size is not None: + try: + from huggingface_hub import split_torch_state_dict_into_shards + except ImportError: + print('If you want to use `max_shard_size`, please `pip install huggingface_hub`') + raise + + # Convert zero checkpoint to state_dict + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag, + exclude_frozen_parameters, + lazy_mode=True) + + # Shard the model if it is too big. + weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin" + if max_shard_size is not None: + filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors") + # an memory-efficient approach for sharding + empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True) + state_dict_split = split_torch_state_dict_into_shards(empty_state_dict, + filename_pattern=filename_pattern, + max_shard_size=max_shard_size) + else: + from collections import namedtuple + StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"]) + state_dict_split = StateDictSplit(is_sharded=False, + filename_to_tensors={weights_name: list(state_dict.keys())}) + + # Save the model by shard + os.makedirs(output_dir, exist_ok=True) + filename_to_tensors = state_dict_split.filename_to_tensors.items() + for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"): + shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors} + shard_state_dict = to_torch_tensor(shard_state_dict) + output_path = os.path.join(output_dir, shard_file) + if safe_serialization: + save_file(shard_state_dict, output_path, metadata={"format": "pt"}) + else: + torch.save(shard_state_dict, output_path) + # release the memory of current shard + for tensor_name in list(shard_state_dict.keys()): + del state_dict[tensor_name] + del shard_state_dict[tensor_name] + del shard_state_dict + gc.collect() + + # Save index if sharded + if state_dict_split.is_sharded: + index = { + "metadata": state_dict_split.metadata, + "weight_map": state_dict_split.tensor_to_filename, + } + save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json" + save_index_file = os.path.join(output_dir, save_index_file) + with open(save_index_file, "w", encoding="utf-8") as f: + content = json.dumps(index, indent=2, sort_keys=True) + "\n" + f.write(content) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info("Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info("Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument("output_dir", + type=str, + help="directory to the pytorch fp32 state_dict output files" + "(e.g. path/checkpoint-12-output/)") + parser.add_argument( + "--max_shard_size", + type=str, + default="5GB", + help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size" + "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`" + "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances" + "without CPU OOM issues.") + parser.add_argument( + "--safe_serialization", + default=False, + action='store_true', + help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_dir, + max_shard_size=args.max_shard_size, + safe_serialization=args.safe_serialization, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-205/config.json b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-205/config.json new file mode 100644 index 0000000000000000000000000000000000000000..18bd58d561964a16b60687f1b04aefdfed0b9b20 --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-205/config.json @@ -0,0 +1,39 @@ +{ + "architectures": [ + "LlamaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "dtype": "bfloat16", + "eos_token_id": [ + 128001, + 128008, + 128009 + ], + "head_dim": 64, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 8192, + "max_position_embeddings": 131072, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 32, + "num_hidden_layers": 16, + "num_key_value_heads": 8, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": { + "factor": 32.0, + "high_freq_factor": 4.0, + "low_freq_factor": 1.0, + "original_max_position_embeddings": 8192, + "rope_type": "llama3" + }, + "rope_theta": 500000.0, + "tie_word_embeddings": true, + "transformers_version": "4.57.1", + "use_cache": false, + "vocab_size": 128256 +} diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-205/generation_config.json b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-205/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..125cc74759bc705ab8a0c48d07d7eac083f56011 --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-205/generation_config.json @@ -0,0 +1,12 @@ +{ + "bos_token_id": 128000, + "do_sample": true, + "eos_token_id": [ + 128001, + 128008, + 128009 + ], + "temperature": 0.6, + "top_p": 0.9, + "transformers_version": "4.57.1" +} diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-205/latest b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-205/latest new file mode 100644 index 0000000000000000000000000000000000000000..5ffce141a6424855cf8b1466c130afe80e1e94b8 --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-205/latest @@ -0,0 +1 @@ +global_step205 \ No newline at end of file diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-205/model.safetensors b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-205/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..12e5de66a134b77d8aa2b991c9462502c8ab4dcf --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-205/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fac26a287bd7707ed7900646752bb3ee856f55d81b3d6ef566b39d5fb917d6ca +size 2996982344 diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-205/scheduler.pt b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-205/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..3852902e06dc0a23083a7a44341099cd95f92388 --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-205/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9735b7c3044a848443046134ed732a567646f7954e492149d971037508389cf0 +size 1465 diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-205/trainer_state.json b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-205/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..3b7ed89af672ac60f1eddd555d440ff5d0b2743f --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-205/trainer_state.json @@ -0,0 +1,174 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.2498476538695917, + "eval_steps": 500, + "global_step": 205, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.01218769043266301, + "grad_norm": 0.36486390233039856, + "learning_rate": 5e-05, + "loss": 1.2213, + "step": 10 + }, + { + "epoch": 0.02437538086532602, + "grad_norm": 0.2953564524650574, + "learning_rate": 5e-05, + "loss": 1.0223, + "step": 20 + }, + { + "epoch": 0.03656307129798903, + "grad_norm": 0.2766735851764679, + "learning_rate": 5e-05, + "loss": 0.9589, + "step": 30 + }, + { + "epoch": 0.04875076173065204, + "grad_norm": 0.32770997285842896, + "learning_rate": 5e-05, + "loss": 0.9081, + "step": 40 + }, + { + "epoch": 0.06093845216331505, + "grad_norm": 0.3052276372909546, + "learning_rate": 5e-05, + "loss": 0.8752, + "step": 50 + }, + { + "epoch": 0.07312614259597806, + "grad_norm": 0.240146666765213, + "learning_rate": 5e-05, + "loss": 0.873, + "step": 60 + }, + { + "epoch": 0.08531383302864107, + "grad_norm": 0.23872187733650208, + "learning_rate": 5e-05, + "loss": 0.8587, + "step": 70 + }, + { + "epoch": 0.09750152346130408, + "grad_norm": 0.32638487219810486, + "learning_rate": 5e-05, + "loss": 0.8656, + "step": 80 + }, + { + "epoch": 0.10968921389396709, + "grad_norm": 0.25019028782844543, + "learning_rate": 5e-05, + "loss": 0.8685, + "step": 90 + }, + { + "epoch": 0.1218769043266301, + "grad_norm": 0.2507675886154175, + "learning_rate": 5e-05, + "loss": 0.8662, + "step": 100 + }, + { + "epoch": 0.1340645947592931, + "grad_norm": 0.27966755628585815, + "learning_rate": 5e-05, + "loss": 0.8428, + "step": 110 + }, + { + "epoch": 0.14625228519195613, + "grad_norm": 0.2622673213481903, + "learning_rate": 5e-05, + "loss": 0.8416, + "step": 120 + }, + { + "epoch": 0.15843997562461914, + "grad_norm": 0.3003966510295868, + "learning_rate": 5e-05, + "loss": 0.8528, + "step": 130 + }, + { + "epoch": 0.17062766605728213, + "grad_norm": 0.32308319211006165, + "learning_rate": 5e-05, + "loss": 0.8357, + "step": 140 + }, + { + "epoch": 0.18281535648994515, + "grad_norm": 0.27279096841812134, + "learning_rate": 5e-05, + "loss": 0.8432, + "step": 150 + }, + { + "epoch": 0.19500304692260817, + "grad_norm": 0.27976104617118835, + "learning_rate": 5e-05, + "loss": 0.8119, + "step": 160 + }, + { + "epoch": 0.2071907373552712, + "grad_norm": 0.290866881608963, + "learning_rate": 5e-05, + "loss": 0.8096, + "step": 170 + }, + { + "epoch": 0.21937842778793418, + "grad_norm": 0.29430949687957764, + "learning_rate": 5e-05, + "loss": 0.819, + "step": 180 + }, + { + "epoch": 0.2315661182205972, + "grad_norm": 0.27714136242866516, + "learning_rate": 5e-05, + "loss": 0.8187, + "step": 190 + }, + { + "epoch": 0.2437538086532602, + "grad_norm": 0.2997029423713684, + "learning_rate": 5e-05, + "loss": 0.808, + "step": 200 + } + ], + "logging_steps": 10, + "max_steps": 2463, + "num_input_tokens_seen": 0, + "num_train_epochs": 3, + "save_steps": 205, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 8.38603986725503e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-205/training_args.bin b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-205/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..23ae0ec8a4226be5457f61c137abab2b2c10a0cf --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-205/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56896ea21940e33d7e2934267dff2a5b4a4a80195a0f165245f6617a0f88b90b +size 7377 diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-205/zero_to_fp32.py b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-205/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..5995d6e6f04e43b989587aa9022a3aef0c66d694 --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-205/zero_to_fp32.py @@ -0,0 +1,760 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: +# python zero_to_fp32.py . output_dir/ +# or +# python zero_to_fp32.py . output_dir/ --safe_serialization + +import argparse +import torch +import glob +import math +import os +import re +import gc +import json +import numpy as np +from tqdm import tqdm +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device, weights_only=False) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + total_files = len(files) + state_dicts = [] + for f in tqdm(files, desc='Loading checkpoint shards'): + state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + model_files = get_model_state_files(ds_checkpoint_dir) + + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +class GatheredTensor: + """ + A pseudo tensor that collects partitioned weights. + It is more memory efficient when there are multiple groups. + """ + + def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape): + self.flat_groups = flat_groups + self.flat_groups_offset = flat_groups_offset + self.offset = offset + self.partitioned_numel = partitioned_numel + self.shape = shape + self.dtype = self.flat_groups[0][0].dtype + + def contiguous(self): + """ + Merge partitioned weights from flat_groups into a single tensor. + """ + end_idx = self.offset + self.partitioned_numel + world_size = len(self.flat_groups) + pad_flat_param_chunks = [] + + for rank_i in range(world_size): + # for each rank, we need to collect weights from related group/groups + flat_groups_at_rank_i = self.flat_groups[rank_i] + start_group_id = None + end_group_id = None + for group_id in range(len(self.flat_groups_offset)): + if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]: + start_group_id = group_id + if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]: + end_group_id = group_id + break + # collect weights from related group/groups + for group_id in range(start_group_id, end_group_id + 1): + flat_tensor = flat_groups_at_rank_i[group_id] + start_offset = self.offset - self.flat_groups_offset[group_id] + end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id] + pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset]) + + # collect weights from all ranks + pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0) + param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous() + return param + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size + + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]])) + for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'): + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # memory efficient tensor + tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape) + state_dict[name] = tensor + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def to_torch_tensor(state_dict, return_empty_tensor=False): + """ + Convert state_dict of GatheredTensor to torch tensor + """ + torch_state_dict = {} + converted_tensors = {} + for name, tensor in state_dict.items(): + tensor_id = id(tensor) + if tensor_id in converted_tensors: # shared tensors + shared_tensor = torch_state_dict[converted_tensors[tensor_id]] + torch_state_dict[name] = shared_tensor + else: + converted_tensors[tensor_id] = name + if return_empty_tensor: + torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype) + else: + torch_state_dict[name] = tensor.contiguous() + return torch_state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag=None, + exclude_frozen_parameters=False, + lazy_mode=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient. + Convert the pesduo tensor to torch tensor by ``.contiguous()`` + + Returns: + - pytorch ``state_dict`` + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + Note: the above usage may not work if your application doesn't have sufficient free CPU memory. + You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. Or you can load state_dict in lazy mode :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu + for name, lazy_tensor in state_dict.item(): + tensor = lazy_tensor.contiguous() # to cpu + print(name, tensor) + # del tensor to release memory if it no longer in use + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + if lazy_mode: + return state_dict + else: + return to_torch_tensor(state_dict) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, + output_dir, + max_shard_size="5GB", + safe_serialization=False, + tag=None, + exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_dir``: directory to the pytorch fp32 state_dict output files + - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB + - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + + # Dependency pre-check + if safe_serialization: + try: + from safetensors.torch import save_file + except ImportError: + print('If you want to use `safe_serialization`, please `pip install safetensors`') + raise + if max_shard_size is not None: + try: + from huggingface_hub import split_torch_state_dict_into_shards + except ImportError: + print('If you want to use `max_shard_size`, please `pip install huggingface_hub`') + raise + + # Convert zero checkpoint to state_dict + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag, + exclude_frozen_parameters, + lazy_mode=True) + + # Shard the model if it is too big. + weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin" + if max_shard_size is not None: + filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors") + # an memory-efficient approach for sharding + empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True) + state_dict_split = split_torch_state_dict_into_shards(empty_state_dict, + filename_pattern=filename_pattern, + max_shard_size=max_shard_size) + else: + from collections import namedtuple + StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"]) + state_dict_split = StateDictSplit(is_sharded=False, + filename_to_tensors={weights_name: list(state_dict.keys())}) + + # Save the model by shard + os.makedirs(output_dir, exist_ok=True) + filename_to_tensors = state_dict_split.filename_to_tensors.items() + for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"): + shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors} + shard_state_dict = to_torch_tensor(shard_state_dict) + output_path = os.path.join(output_dir, shard_file) + if safe_serialization: + save_file(shard_state_dict, output_path, metadata={"format": "pt"}) + else: + torch.save(shard_state_dict, output_path) + # release the memory of current shard + for tensor_name in list(shard_state_dict.keys()): + del state_dict[tensor_name] + del shard_state_dict[tensor_name] + del shard_state_dict + gc.collect() + + # Save index if sharded + if state_dict_split.is_sharded: + index = { + "metadata": state_dict_split.metadata, + "weight_map": state_dict_split.tensor_to_filename, + } + save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json" + save_index_file = os.path.join(output_dir, save_index_file) + with open(save_index_file, "w", encoding="utf-8") as f: + content = json.dumps(index, indent=2, sort_keys=True) + "\n" + f.write(content) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info("Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info("Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument("output_dir", + type=str, + help="directory to the pytorch fp32 state_dict output files" + "(e.g. path/checkpoint-12-output/)") + parser.add_argument( + "--max_shard_size", + type=str, + default="5GB", + help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size" + "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`" + "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances" + "without CPU OOM issues.") + parser.add_argument( + "--safe_serialization", + default=False, + action='store_true', + help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_dir, + max_shard_size=args.max_shard_size, + safe_serialization=args.safe_serialization, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2050/config.json b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2050/config.json new file mode 100644 index 0000000000000000000000000000000000000000..18bd58d561964a16b60687f1b04aefdfed0b9b20 --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2050/config.json @@ -0,0 +1,39 @@ +{ + "architectures": [ + "LlamaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "dtype": "bfloat16", + "eos_token_id": [ + 128001, + 128008, + 128009 + ], + "head_dim": 64, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 8192, + "max_position_embeddings": 131072, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 32, + "num_hidden_layers": 16, + "num_key_value_heads": 8, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": { + "factor": 32.0, + "high_freq_factor": 4.0, + "low_freq_factor": 1.0, + "original_max_position_embeddings": 8192, + "rope_type": "llama3" + }, + "rope_theta": 500000.0, + "tie_word_embeddings": true, + "transformers_version": "4.57.1", + "use_cache": false, + "vocab_size": 128256 +} diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2050/generation_config.json b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2050/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..125cc74759bc705ab8a0c48d07d7eac083f56011 --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2050/generation_config.json @@ -0,0 +1,12 @@ +{ + "bos_token_id": 128000, + "do_sample": true, + "eos_token_id": [ + 128001, + 128008, + 128009 + ], + "temperature": 0.6, + "top_p": 0.9, + "transformers_version": "4.57.1" +} diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2050/latest b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2050/latest new file mode 100644 index 0000000000000000000000000000000000000000..526a4b1a97704997de883baa95188aa27c685beb --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2050/latest @@ -0,0 +1 @@ +global_step2050 \ No newline at end of file diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2050/model.safetensors b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2050/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..7021d9b8cfbf8f0ebfd701f60f5eb2b556fbb6c6 --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2050/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a62b2d636f1f0466b1f637be495caa8e026637a95c7945b33a424348924a49cd +size 2996982344 diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2050/scheduler.pt b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2050/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..80e2c590fd9e6bdc59ca350b6859054d382382e3 --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2050/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1415e4ba8802e5ea5b9c92c0eb3b18425b4c01a2d63afa1aa909a2943d66267b +size 1465 diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2050/trainer_state.json b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2050/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..0408ac3388e6a321d3ed08951f59f43c6ba87d94 --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2050/trainer_state.json @@ -0,0 +1,1469 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 2.497257769652651, + "eval_steps": 500, + "global_step": 2050, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.01218769043266301, + "grad_norm": 0.36486390233039856, + "learning_rate": 5e-05, + "loss": 1.2213, + "step": 10 + }, + { + "epoch": 0.02437538086532602, + "grad_norm": 0.2953564524650574, + "learning_rate": 5e-05, + "loss": 1.0223, + "step": 20 + }, + { + "epoch": 0.03656307129798903, + "grad_norm": 0.2766735851764679, + "learning_rate": 5e-05, + "loss": 0.9589, + "step": 30 + }, + { + "epoch": 0.04875076173065204, + "grad_norm": 0.32770997285842896, + "learning_rate": 5e-05, + "loss": 0.9081, + "step": 40 + }, + { + "epoch": 0.06093845216331505, + "grad_norm": 0.3052276372909546, + "learning_rate": 5e-05, + "loss": 0.8752, + "step": 50 + }, + { + "epoch": 0.07312614259597806, + "grad_norm": 0.240146666765213, + "learning_rate": 5e-05, + "loss": 0.873, + "step": 60 + }, + { + "epoch": 0.08531383302864107, + "grad_norm": 0.23872187733650208, + "learning_rate": 5e-05, + "loss": 0.8587, + "step": 70 + }, + { + "epoch": 0.09750152346130408, + "grad_norm": 0.32638487219810486, + "learning_rate": 5e-05, + "loss": 0.8656, + "step": 80 + }, + { + "epoch": 0.10968921389396709, + "grad_norm": 0.25019028782844543, + "learning_rate": 5e-05, + "loss": 0.8685, + "step": 90 + }, + { + "epoch": 0.1218769043266301, + "grad_norm": 0.2507675886154175, + "learning_rate": 5e-05, + "loss": 0.8662, + "step": 100 + }, + { + "epoch": 0.1340645947592931, + "grad_norm": 0.27966755628585815, + "learning_rate": 5e-05, + "loss": 0.8428, + "step": 110 + }, + { + "epoch": 0.14625228519195613, + "grad_norm": 0.2622673213481903, + "learning_rate": 5e-05, + "loss": 0.8416, + "step": 120 + }, + { + "epoch": 0.15843997562461914, + "grad_norm": 0.3003966510295868, + "learning_rate": 5e-05, + "loss": 0.8528, + "step": 130 + }, + { + "epoch": 0.17062766605728213, + "grad_norm": 0.32308319211006165, + "learning_rate": 5e-05, + "loss": 0.8357, + "step": 140 + }, + { + "epoch": 0.18281535648994515, + "grad_norm": 0.27279096841812134, + "learning_rate": 5e-05, + "loss": 0.8432, + "step": 150 + }, + { + "epoch": 0.19500304692260817, + "grad_norm": 0.27976104617118835, + "learning_rate": 5e-05, + "loss": 0.8119, + "step": 160 + }, + { + "epoch": 0.2071907373552712, + "grad_norm": 0.290866881608963, + "learning_rate": 5e-05, + "loss": 0.8096, + "step": 170 + }, + { + "epoch": 0.21937842778793418, + "grad_norm": 0.29430949687957764, + "learning_rate": 5e-05, + "loss": 0.819, + "step": 180 + }, + { + "epoch": 0.2315661182205972, + "grad_norm": 0.27714136242866516, + "learning_rate": 5e-05, + "loss": 0.8187, + "step": 190 + }, + { + "epoch": 0.2437538086532602, + "grad_norm": 0.2997029423713684, + "learning_rate": 5e-05, + "loss": 0.808, + "step": 200 + }, + { + "epoch": 0.25594149908592323, + "grad_norm": 0.2539573609828949, + "learning_rate": 5e-05, + "loss": 0.7943, + "step": 210 + }, + { + "epoch": 0.2681291895185862, + "grad_norm": 0.23294171690940857, + "learning_rate": 5e-05, + "loss": 0.8188, + "step": 220 + }, + { + "epoch": 0.28031687995124926, + "grad_norm": 0.25016292929649353, + "learning_rate": 5e-05, + "loss": 0.8035, + "step": 230 + }, + { + "epoch": 0.29250457038391225, + "grad_norm": 0.37125322222709656, + "learning_rate": 5e-05, + "loss": 0.785, + "step": 240 + }, + { + "epoch": 0.30469226081657524, + "grad_norm": 0.21520452201366425, + "learning_rate": 5e-05, + "loss": 0.8088, + "step": 250 + }, + { + "epoch": 0.3168799512492383, + "grad_norm": 0.20990602672100067, + "learning_rate": 5e-05, + "loss": 0.7813, + "step": 260 + }, + { + "epoch": 0.3290676416819013, + "grad_norm": 0.24193961918354034, + "learning_rate": 5e-05, + "loss": 0.7979, + "step": 270 + }, + { + "epoch": 0.34125533211456427, + "grad_norm": 0.22766050696372986, + "learning_rate": 5e-05, + "loss": 0.785, + "step": 280 + }, + { + "epoch": 0.3534430225472273, + "grad_norm": 0.2662702202796936, + "learning_rate": 5e-05, + "loss": 0.7766, + "step": 290 + }, + { + "epoch": 0.3656307129798903, + "grad_norm": 0.23020495474338531, + "learning_rate": 5e-05, + "loss": 0.7687, + "step": 300 + }, + { + "epoch": 0.3778184034125533, + "grad_norm": 0.21770520508289337, + "learning_rate": 5e-05, + "loss": 0.7815, + "step": 310 + }, + { + "epoch": 0.39000609384521634, + "grad_norm": 0.22677403688430786, + "learning_rate": 5e-05, + "loss": 0.7999, + "step": 320 + }, + { + "epoch": 0.40219378427787933, + "grad_norm": 0.24891860783100128, + "learning_rate": 5e-05, + "loss": 0.7946, + "step": 330 + }, + { + "epoch": 0.4143814747105424, + "grad_norm": 0.2506854832172394, + "learning_rate": 5e-05, + "loss": 0.7889, + "step": 340 + }, + { + "epoch": 0.42656916514320536, + "grad_norm": 0.24388180673122406, + "learning_rate": 5e-05, + "loss": 0.79, + "step": 350 + }, + { + "epoch": 0.43875685557586835, + "grad_norm": 0.22329650819301605, + "learning_rate": 5e-05, + "loss": 0.7632, + "step": 360 + }, + { + "epoch": 0.4509445460085314, + "grad_norm": 0.2240927815437317, + "learning_rate": 5e-05, + "loss": 0.8042, + "step": 370 + }, + { + "epoch": 0.4631322364411944, + "grad_norm": 0.2578203082084656, + "learning_rate": 5e-05, + "loss": 0.7667, + "step": 380 + }, + { + "epoch": 0.4753199268738574, + "grad_norm": 0.25349217653274536, + "learning_rate": 5e-05, + "loss": 0.7911, + "step": 390 + }, + { + "epoch": 0.4875076173065204, + "grad_norm": 0.254446804523468, + "learning_rate": 5e-05, + "loss": 0.7909, + "step": 400 + }, + { + "epoch": 0.4996953077391834, + "grad_norm": 0.24253171682357788, + "learning_rate": 5e-05, + "loss": 0.7896, + "step": 410 + }, + { + "epoch": 0.5118829981718465, + "grad_norm": 0.27901458740234375, + "learning_rate": 5e-05, + "loss": 0.7828, + "step": 420 + }, + { + "epoch": 0.5240706886045094, + "grad_norm": 0.31884434819221497, + "learning_rate": 5e-05, + "loss": 0.7919, + "step": 430 + }, + { + "epoch": 0.5362583790371724, + "grad_norm": 0.22920799255371094, + "learning_rate": 5e-05, + "loss": 0.7777, + "step": 440 + }, + { + "epoch": 0.5484460694698354, + "grad_norm": 0.22606618702411652, + "learning_rate": 5e-05, + "loss": 0.7661, + "step": 450 + }, + { + "epoch": 0.5606337599024985, + "grad_norm": 0.21604052186012268, + "learning_rate": 5e-05, + "loss": 0.769, + "step": 460 + }, + { + "epoch": 0.5728214503351615, + "grad_norm": 0.2296101152896881, + "learning_rate": 5e-05, + "loss": 0.7626, + "step": 470 + }, + { + "epoch": 0.5850091407678245, + "grad_norm": 0.24162150919437408, + "learning_rate": 5e-05, + "loss": 0.7831, + "step": 480 + }, + { + "epoch": 0.5971968312004875, + "grad_norm": 0.22364799678325653, + "learning_rate": 5e-05, + "loss": 0.7737, + "step": 490 + }, + { + "epoch": 0.6093845216331505, + "grad_norm": 0.23953403532505035, + "learning_rate": 5e-05, + "loss": 0.787, + "step": 500 + }, + { + "epoch": 0.6215722120658135, + "grad_norm": 0.22349311411380768, + "learning_rate": 5e-05, + "loss": 0.7763, + "step": 510 + }, + { + "epoch": 0.6337599024984766, + "grad_norm": 0.21996377408504486, + "learning_rate": 5e-05, + "loss": 0.7596, + "step": 520 + }, + { + "epoch": 0.6459475929311396, + "grad_norm": 0.245538130402565, + "learning_rate": 5e-05, + "loss": 0.7785, + "step": 530 + }, + { + "epoch": 0.6581352833638026, + "grad_norm": 0.23442520201206207, + "learning_rate": 5e-05, + "loss": 0.7633, + "step": 540 + }, + { + "epoch": 0.6703229737964655, + "grad_norm": 0.22274106740951538, + "learning_rate": 5e-05, + "loss": 0.7756, + "step": 550 + }, + { + "epoch": 0.6825106642291285, + "grad_norm": 0.21717530488967896, + "learning_rate": 5e-05, + "loss": 0.7726, + "step": 560 + }, + { + "epoch": 0.6946983546617916, + "grad_norm": 0.2465955913066864, + "learning_rate": 5e-05, + "loss": 0.7476, + "step": 570 + }, + { + "epoch": 0.7068860450944546, + "grad_norm": 0.2318575084209442, + "learning_rate": 5e-05, + "loss": 0.7584, + "step": 580 + }, + { + "epoch": 0.7190737355271176, + "grad_norm": 0.25114378333091736, + "learning_rate": 5e-05, + "loss": 0.7625, + "step": 590 + }, + { + "epoch": 0.7312614259597806, + "grad_norm": 0.19145755469799042, + "learning_rate": 5e-05, + "loss": 0.7577, + "step": 600 + }, + { + "epoch": 0.7434491163924436, + "grad_norm": 0.19129066169261932, + "learning_rate": 5e-05, + "loss": 0.7772, + "step": 610 + }, + { + "epoch": 0.7556368068251066, + "grad_norm": 0.2014475166797638, + "learning_rate": 5e-05, + "loss": 0.7601, + "step": 620 + }, + { + "epoch": 0.7678244972577697, + "grad_norm": 0.21480509638786316, + "learning_rate": 5e-05, + "loss": 0.7665, + "step": 630 + }, + { + "epoch": 0.7800121876904327, + "grad_norm": 0.2633228302001953, + "learning_rate": 5e-05, + "loss": 0.7565, + "step": 640 + }, + { + "epoch": 0.7921998781230957, + "grad_norm": 0.24633203446865082, + "learning_rate": 5e-05, + "loss": 0.7602, + "step": 650 + }, + { + "epoch": 0.8043875685557587, + "grad_norm": 0.19770534336566925, + "learning_rate": 5e-05, + "loss": 0.7685, + "step": 660 + }, + { + "epoch": 0.8165752589884216, + "grad_norm": 0.2267930805683136, + "learning_rate": 5e-05, + "loss": 0.7516, + "step": 670 + }, + { + "epoch": 0.8287629494210847, + "grad_norm": 0.238111674785614, + "learning_rate": 5e-05, + "loss": 0.7487, + "step": 680 + }, + { + "epoch": 0.8409506398537477, + "grad_norm": 0.19528904557228088, + "learning_rate": 5e-05, + "loss": 0.7467, + "step": 690 + }, + { + "epoch": 0.8531383302864107, + "grad_norm": 0.21305914223194122, + "learning_rate": 5e-05, + "loss": 0.7527, + "step": 700 + }, + { + "epoch": 0.8653260207190737, + "grad_norm": 0.2205205112695694, + "learning_rate": 5e-05, + "loss": 0.7639, + "step": 710 + }, + { + "epoch": 0.8775137111517367, + "grad_norm": 0.22429127991199493, + "learning_rate": 5e-05, + "loss": 0.7744, + "step": 720 + }, + { + "epoch": 0.8897014015843998, + "grad_norm": 0.23755772411823273, + "learning_rate": 5e-05, + "loss": 0.7446, + "step": 730 + }, + { + "epoch": 0.9018890920170628, + "grad_norm": 0.17592693865299225, + "learning_rate": 5e-05, + "loss": 0.7442, + "step": 740 + }, + { + "epoch": 0.9140767824497258, + "grad_norm": 0.2266898900270462, + "learning_rate": 5e-05, + "loss": 0.7326, + "step": 750 + }, + { + "epoch": 0.9262644728823888, + "grad_norm": 0.23541569709777832, + "learning_rate": 5e-05, + "loss": 0.7338, + "step": 760 + }, + { + "epoch": 0.9384521633150518, + "grad_norm": 0.22233888506889343, + "learning_rate": 5e-05, + "loss": 0.7385, + "step": 770 + }, + { + "epoch": 0.9506398537477148, + "grad_norm": 0.23714767396450043, + "learning_rate": 5e-05, + "loss": 0.7306, + "step": 780 + }, + { + "epoch": 0.9628275441803779, + "grad_norm": 0.21191060543060303, + "learning_rate": 5e-05, + "loss": 0.736, + "step": 790 + }, + { + "epoch": 0.9750152346130408, + "grad_norm": 0.22759106755256653, + "learning_rate": 5e-05, + "loss": 0.7505, + "step": 800 + }, + { + "epoch": 0.9872029250457038, + "grad_norm": 0.19895903766155243, + "learning_rate": 5e-05, + "loss": 0.7536, + "step": 810 + }, + { + "epoch": 0.9993906154783668, + "grad_norm": 0.3061685860157013, + "learning_rate": 5e-05, + "loss": 0.7493, + "step": 820 + }, + { + "epoch": 1.0109689213893966, + "grad_norm": 0.18868015706539154, + "learning_rate": 5e-05, + "loss": 0.7342, + "step": 830 + }, + { + "epoch": 1.0231566118220596, + "grad_norm": 0.1797628402709961, + "learning_rate": 5e-05, + "loss": 0.7192, + "step": 840 + }, + { + "epoch": 1.0353443022547226, + "grad_norm": 0.19511596858501434, + "learning_rate": 5e-05, + "loss": 0.7002, + "step": 850 + }, + { + "epoch": 1.0475319926873858, + "grad_norm": 0.2123185396194458, + "learning_rate": 5e-05, + "loss": 0.7236, + "step": 860 + }, + { + "epoch": 1.0597196831200488, + "grad_norm": 0.2047470211982727, + "learning_rate": 5e-05, + "loss": 0.711, + "step": 870 + }, + { + "epoch": 1.0719073735527118, + "grad_norm": 0.18809868395328522, + "learning_rate": 5e-05, + "loss": 0.7253, + "step": 880 + }, + { + "epoch": 1.0840950639853748, + "grad_norm": 0.2207012176513672, + "learning_rate": 5e-05, + "loss": 0.6971, + "step": 890 + }, + { + "epoch": 1.0962827544180378, + "grad_norm": 0.1912035495042801, + "learning_rate": 5e-05, + "loss": 0.7239, + "step": 900 + }, + { + "epoch": 1.1084704448507008, + "grad_norm": 0.23486360907554626, + "learning_rate": 5e-05, + "loss": 0.7335, + "step": 910 + }, + { + "epoch": 1.1206581352833638, + "grad_norm": 0.22189787030220032, + "learning_rate": 5e-05, + "loss": 0.7149, + "step": 920 + }, + { + "epoch": 1.1328458257160268, + "grad_norm": 0.25818824768066406, + "learning_rate": 5e-05, + "loss": 0.7322, + "step": 930 + }, + { + "epoch": 1.1450335161486898, + "grad_norm": 0.2088775783777237, + "learning_rate": 5e-05, + "loss": 0.7382, + "step": 940 + }, + { + "epoch": 1.1572212065813527, + "grad_norm": 0.19445833563804626, + "learning_rate": 5e-05, + "loss": 0.7223, + "step": 950 + }, + { + "epoch": 1.1694088970140157, + "grad_norm": 0.21489456295967102, + "learning_rate": 5e-05, + "loss": 0.7317, + "step": 960 + }, + { + "epoch": 1.181596587446679, + "grad_norm": 0.24050794541835785, + "learning_rate": 5e-05, + "loss": 0.7275, + "step": 970 + }, + { + "epoch": 1.193784277879342, + "grad_norm": 0.18142102658748627, + "learning_rate": 5e-05, + "loss": 0.7046, + "step": 980 + }, + { + "epoch": 1.205971968312005, + "grad_norm": 0.4781077802181244, + "learning_rate": 5e-05, + "loss": 0.7433, + "step": 990 + }, + { + "epoch": 1.218159658744668, + "grad_norm": 0.20628023147583008, + "learning_rate": 5e-05, + "loss": 0.7403, + "step": 1000 + }, + { + "epoch": 1.230347349177331, + "grad_norm": 0.19117140769958496, + "learning_rate": 5e-05, + "loss": 0.7308, + "step": 1010 + }, + { + "epoch": 1.242535039609994, + "grad_norm": 0.2107745110988617, + "learning_rate": 5e-05, + "loss": 0.7201, + "step": 1020 + }, + { + "epoch": 1.2547227300426569, + "grad_norm": 0.20551154017448425, + "learning_rate": 5e-05, + "loss": 0.7038, + "step": 1030 + }, + { + "epoch": 1.2669104204753199, + "grad_norm": 0.26577022671699524, + "learning_rate": 5e-05, + "loss": 0.7096, + "step": 1040 + }, + { + "epoch": 1.2790981109079829, + "grad_norm": 0.22094209492206573, + "learning_rate": 5e-05, + "loss": 0.718, + "step": 1050 + }, + { + "epoch": 1.291285801340646, + "grad_norm": 0.18224076926708221, + "learning_rate": 5e-05, + "loss": 0.7261, + "step": 1060 + }, + { + "epoch": 1.303473491773309, + "grad_norm": 0.226251482963562, + "learning_rate": 5e-05, + "loss": 0.7372, + "step": 1070 + }, + { + "epoch": 1.315661182205972, + "grad_norm": 0.20186461508274078, + "learning_rate": 5e-05, + "loss": 0.7074, + "step": 1080 + }, + { + "epoch": 1.327848872638635, + "grad_norm": 0.22229762375354767, + "learning_rate": 5e-05, + "loss": 0.7069, + "step": 1090 + }, + { + "epoch": 1.340036563071298, + "grad_norm": 0.1776452213525772, + "learning_rate": 5e-05, + "loss": 0.714, + "step": 1100 + }, + { + "epoch": 1.352224253503961, + "grad_norm": 0.20680229365825653, + "learning_rate": 5e-05, + "loss": 0.7066, + "step": 1110 + }, + { + "epoch": 1.364411943936624, + "grad_norm": 0.21842031180858612, + "learning_rate": 5e-05, + "loss": 0.7281, + "step": 1120 + }, + { + "epoch": 1.376599634369287, + "grad_norm": 0.28037774562835693, + "learning_rate": 5e-05, + "loss": 0.7149, + "step": 1130 + }, + { + "epoch": 1.38878732480195, + "grad_norm": 0.20962920784950256, + "learning_rate": 5e-05, + "loss": 0.6951, + "step": 1140 + }, + { + "epoch": 1.400975015234613, + "grad_norm": 0.18751871585845947, + "learning_rate": 5e-05, + "loss": 0.7101, + "step": 1150 + }, + { + "epoch": 1.413162705667276, + "grad_norm": 0.2722964584827423, + "learning_rate": 5e-05, + "loss": 0.7205, + "step": 1160 + }, + { + "epoch": 1.425350396099939, + "grad_norm": 0.20932041108608246, + "learning_rate": 5e-05, + "loss": 0.7082, + "step": 1170 + }, + { + "epoch": 1.437538086532602, + "grad_norm": 0.17836257815361023, + "learning_rate": 5e-05, + "loss": 0.7232, + "step": 1180 + }, + { + "epoch": 1.449725776965265, + "grad_norm": 0.3180703818798065, + "learning_rate": 5e-05, + "loss": 0.7245, + "step": 1190 + }, + { + "epoch": 1.4619134673979282, + "grad_norm": 0.201424703001976, + "learning_rate": 5e-05, + "loss": 0.7232, + "step": 1200 + }, + { + "epoch": 1.4741011578305911, + "grad_norm": 0.20228999853134155, + "learning_rate": 5e-05, + "loss": 0.7169, + "step": 1210 + }, + { + "epoch": 1.4862888482632541, + "grad_norm": 0.17478394508361816, + "learning_rate": 5e-05, + "loss": 0.704, + "step": 1220 + }, + { + "epoch": 1.4984765386959171, + "grad_norm": 0.20061351358890533, + "learning_rate": 5e-05, + "loss": 0.6792, + "step": 1230 + }, + { + "epoch": 1.5106642291285801, + "grad_norm": 0.1857377588748932, + "learning_rate": 5e-05, + "loss": 0.709, + "step": 1240 + }, + { + "epoch": 1.522851919561243, + "grad_norm": 0.19873379170894623, + "learning_rate": 5e-05, + "loss": 0.7007, + "step": 1250 + }, + { + "epoch": 1.5350396099939063, + "grad_norm": 0.19807074964046478, + "learning_rate": 5e-05, + "loss": 0.7122, + "step": 1260 + }, + { + "epoch": 1.5472273004265693, + "grad_norm": 0.16746796667575836, + "learning_rate": 5e-05, + "loss": 0.7005, + "step": 1270 + }, + { + "epoch": 1.5594149908592323, + "grad_norm": 0.2750326991081238, + "learning_rate": 5e-05, + "loss": 0.7166, + "step": 1280 + }, + { + "epoch": 1.5716026812918953, + "grad_norm": 0.2776738107204437, + "learning_rate": 5e-05, + "loss": 0.7405, + "step": 1290 + }, + { + "epoch": 1.5837903717245583, + "grad_norm": 0.3494517207145691, + "learning_rate": 5e-05, + "loss": 0.7192, + "step": 1300 + }, + { + "epoch": 1.5959780621572213, + "grad_norm": 0.25028494000434875, + "learning_rate": 5e-05, + "loss": 0.741, + "step": 1310 + }, + { + "epoch": 1.6081657525898843, + "grad_norm": 0.23914143443107605, + "learning_rate": 5e-05, + "loss": 0.72, + "step": 1320 + }, + { + "epoch": 1.6203534430225472, + "grad_norm": 0.2052862048149109, + "learning_rate": 5e-05, + "loss": 0.7112, + "step": 1330 + }, + { + "epoch": 1.6325411334552102, + "grad_norm": 0.22285936772823334, + "learning_rate": 5e-05, + "loss": 0.7165, + "step": 1340 + }, + { + "epoch": 1.6447288238878732, + "grad_norm": 0.20810848474502563, + "learning_rate": 5e-05, + "loss": 0.7122, + "step": 1350 + }, + { + "epoch": 1.6569165143205362, + "grad_norm": 0.20422884821891785, + "learning_rate": 5e-05, + "loss": 0.7082, + "step": 1360 + }, + { + "epoch": 1.6691042047531992, + "grad_norm": 0.18164314329624176, + "learning_rate": 5e-05, + "loss": 0.7131, + "step": 1370 + }, + { + "epoch": 1.6812918951858622, + "grad_norm": 0.19893014430999756, + "learning_rate": 5e-05, + "loss": 0.7055, + "step": 1380 + }, + { + "epoch": 1.6934795856185252, + "grad_norm": 0.24865971505641937, + "learning_rate": 5e-05, + "loss": 0.6994, + "step": 1390 + }, + { + "epoch": 1.7056672760511882, + "grad_norm": 0.18078123033046722, + "learning_rate": 5e-05, + "loss": 0.7162, + "step": 1400 + }, + { + "epoch": 1.7178549664838512, + "grad_norm": 0.1973421722650528, + "learning_rate": 5e-05, + "loss": 0.7136, + "step": 1410 + }, + { + "epoch": 1.7300426569165142, + "grad_norm": 0.2057911604642868, + "learning_rate": 5e-05, + "loss": 0.7004, + "step": 1420 + }, + { + "epoch": 1.7422303473491774, + "grad_norm": 0.18640495836734772, + "learning_rate": 5e-05, + "loss": 0.7102, + "step": 1430 + }, + { + "epoch": 1.7544180377818404, + "grad_norm": 0.21038417518138885, + "learning_rate": 5e-05, + "loss": 0.6908, + "step": 1440 + }, + { + "epoch": 1.7666057282145033, + "grad_norm": 0.2123890221118927, + "learning_rate": 5e-05, + "loss": 0.6913, + "step": 1450 + }, + { + "epoch": 1.7787934186471663, + "grad_norm": 0.21619991958141327, + "learning_rate": 5e-05, + "loss": 0.7038, + "step": 1460 + }, + { + "epoch": 1.7909811090798293, + "grad_norm": 0.1975220888853073, + "learning_rate": 5e-05, + "loss": 0.7041, + "step": 1470 + }, + { + "epoch": 1.8031687995124925, + "grad_norm": 0.20595207810401917, + "learning_rate": 5e-05, + "loss": 0.7145, + "step": 1480 + }, + { + "epoch": 1.8153564899451555, + "grad_norm": 0.16633334755897522, + "learning_rate": 5e-05, + "loss": 0.6963, + "step": 1490 + }, + { + "epoch": 1.8275441803778185, + "grad_norm": 0.19140620529651642, + "learning_rate": 5e-05, + "loss": 0.6814, + "step": 1500 + }, + { + "epoch": 1.8397318708104815, + "grad_norm": 0.1833905130624771, + "learning_rate": 5e-05, + "loss": 0.7174, + "step": 1510 + }, + { + "epoch": 1.8519195612431445, + "grad_norm": 0.20587337017059326, + "learning_rate": 5e-05, + "loss": 0.7098, + "step": 1520 + }, + { + "epoch": 1.8641072516758075, + "grad_norm": 0.21952909231185913, + "learning_rate": 5e-05, + "loss": 0.6912, + "step": 1530 + }, + { + "epoch": 1.8762949421084705, + "grad_norm": 0.23233620822429657, + "learning_rate": 5e-05, + "loss": 0.6992, + "step": 1540 + }, + { + "epoch": 1.8884826325411335, + "grad_norm": 0.17159809172153473, + "learning_rate": 5e-05, + "loss": 0.7052, + "step": 1550 + }, + { + "epoch": 1.9006703229737965, + "grad_norm": 0.23522134125232697, + "learning_rate": 5e-05, + "loss": 0.6873, + "step": 1560 + }, + { + "epoch": 1.9128580134064594, + "grad_norm": 0.21959669888019562, + "learning_rate": 5e-05, + "loss": 0.6834, + "step": 1570 + }, + { + "epoch": 1.9250457038391224, + "grad_norm": 0.2110578864812851, + "learning_rate": 5e-05, + "loss": 0.7038, + "step": 1580 + }, + { + "epoch": 1.9372333942717854, + "grad_norm": 0.18674372136592865, + "learning_rate": 5e-05, + "loss": 0.6743, + "step": 1590 + }, + { + "epoch": 1.9494210847044484, + "grad_norm": 0.18220046162605286, + "learning_rate": 5e-05, + "loss": 0.6943, + "step": 1600 + }, + { + "epoch": 1.9616087751371114, + "grad_norm": 0.20756569504737854, + "learning_rate": 5e-05, + "loss": 0.7049, + "step": 1610 + }, + { + "epoch": 1.9737964655697744, + "grad_norm": 0.1925211250782013, + "learning_rate": 5e-05, + "loss": 0.7033, + "step": 1620 + }, + { + "epoch": 1.9859841560024374, + "grad_norm": 0.21374447643756866, + "learning_rate": 5e-05, + "loss": 0.7025, + "step": 1630 + }, + { + "epoch": 1.9981718464351006, + "grad_norm": 0.1952880322933197, + "learning_rate": 5e-05, + "loss": 0.6877, + "step": 1640 + }, + { + "epoch": 2.0097501523461303, + "grad_norm": 0.20457723736763, + "learning_rate": 5e-05, + "loss": 0.6734, + "step": 1650 + }, + { + "epoch": 2.0219378427787933, + "grad_norm": 0.20965932309627533, + "learning_rate": 5e-05, + "loss": 0.6472, + "step": 1660 + }, + { + "epoch": 2.0341255332114563, + "grad_norm": 0.19478890299797058, + "learning_rate": 5e-05, + "loss": 0.6781, + "step": 1670 + }, + { + "epoch": 2.0463132236441193, + "grad_norm": 0.21520769596099854, + "learning_rate": 5e-05, + "loss": 0.6592, + "step": 1680 + }, + { + "epoch": 2.0585009140767823, + "grad_norm": 0.20154155790805817, + "learning_rate": 5e-05, + "loss": 0.6744, + "step": 1690 + }, + { + "epoch": 2.0706886045094453, + "grad_norm": 0.1899706870317459, + "learning_rate": 5e-05, + "loss": 0.6592, + "step": 1700 + }, + { + "epoch": 2.0828762949421087, + "grad_norm": 0.20379123091697693, + "learning_rate": 5e-05, + "loss": 0.6688, + "step": 1710 + }, + { + "epoch": 2.0950639853747717, + "grad_norm": 0.19799865782260895, + "learning_rate": 5e-05, + "loss": 0.6678, + "step": 1720 + }, + { + "epoch": 2.1072516758074347, + "grad_norm": 0.19186025857925415, + "learning_rate": 5e-05, + "loss": 0.6587, + "step": 1730 + }, + { + "epoch": 2.1194393662400977, + "grad_norm": 0.18936173617839813, + "learning_rate": 5e-05, + "loss": 0.668, + "step": 1740 + }, + { + "epoch": 2.1316270566727606, + "grad_norm": 0.19159527122974396, + "learning_rate": 5e-05, + "loss": 0.6643, + "step": 1750 + }, + { + "epoch": 2.1438147471054236, + "grad_norm": 0.20596368610858917, + "learning_rate": 5e-05, + "loss": 0.6667, + "step": 1760 + }, + { + "epoch": 2.1560024375380866, + "grad_norm": 0.18601712584495544, + "learning_rate": 5e-05, + "loss": 0.6598, + "step": 1770 + }, + { + "epoch": 2.1681901279707496, + "grad_norm": 0.2130655199289322, + "learning_rate": 5e-05, + "loss": 0.6844, + "step": 1780 + }, + { + "epoch": 2.1803778184034126, + "grad_norm": 0.20477338135242462, + "learning_rate": 5e-05, + "loss": 0.666, + "step": 1790 + }, + { + "epoch": 2.1925655088360756, + "grad_norm": 0.18484896421432495, + "learning_rate": 5e-05, + "loss": 0.6659, + "step": 1800 + }, + { + "epoch": 2.2047531992687386, + "grad_norm": 0.17023979127407074, + "learning_rate": 5e-05, + "loss": 0.6721, + "step": 1810 + }, + { + "epoch": 2.2169408897014016, + "grad_norm": 0.23462794721126556, + "learning_rate": 5e-05, + "loss": 0.6699, + "step": 1820 + }, + { + "epoch": 2.2291285801340646, + "grad_norm": 0.16521663963794708, + "learning_rate": 5e-05, + "loss": 0.6584, + "step": 1830 + }, + { + "epoch": 2.2413162705667276, + "grad_norm": 0.19637759029865265, + "learning_rate": 5e-05, + "loss": 0.6621, + "step": 1840 + }, + { + "epoch": 2.2535039609993905, + "grad_norm": 0.2099665403366089, + "learning_rate": 5e-05, + "loss": 0.6912, + "step": 1850 + }, + { + "epoch": 2.2656916514320535, + "grad_norm": 0.21308118104934692, + "learning_rate": 5e-05, + "loss": 0.6705, + "step": 1860 + }, + { + "epoch": 2.2778793418647165, + "grad_norm": 0.20638489723205566, + "learning_rate": 5e-05, + "loss": 0.6736, + "step": 1870 + }, + { + "epoch": 2.2900670322973795, + "grad_norm": 0.23812630772590637, + "learning_rate": 5e-05, + "loss": 0.6869, + "step": 1880 + }, + { + "epoch": 2.3022547227300425, + "grad_norm": 0.23223577439785004, + "learning_rate": 5e-05, + "loss": 0.6663, + "step": 1890 + }, + { + "epoch": 2.3144424131627055, + "grad_norm": 0.20844320952892303, + "learning_rate": 5e-05, + "loss": 0.6686, + "step": 1900 + }, + { + "epoch": 2.3266301035953685, + "grad_norm": 0.2517656683921814, + "learning_rate": 5e-05, + "loss": 0.658, + "step": 1910 + }, + { + "epoch": 2.3388177940280315, + "grad_norm": 0.20005013048648834, + "learning_rate": 5e-05, + "loss": 0.6659, + "step": 1920 + }, + { + "epoch": 2.3510054844606945, + "grad_norm": 0.20756477117538452, + "learning_rate": 5e-05, + "loss": 0.6613, + "step": 1930 + }, + { + "epoch": 2.363193174893358, + "grad_norm": 0.19551488757133484, + "learning_rate": 5e-05, + "loss": 0.676, + "step": 1940 + }, + { + "epoch": 2.375380865326021, + "grad_norm": 0.18600064516067505, + "learning_rate": 5e-05, + "loss": 0.6826, + "step": 1950 + }, + { + "epoch": 2.387568555758684, + "grad_norm": 0.2445133775472641, + "learning_rate": 5e-05, + "loss": 0.6623, + "step": 1960 + }, + { + "epoch": 2.399756246191347, + "grad_norm": 0.2174660712480545, + "learning_rate": 5e-05, + "loss": 0.6697, + "step": 1970 + }, + { + "epoch": 2.41194393662401, + "grad_norm": 0.1853487342596054, + "learning_rate": 5e-05, + "loss": 0.6561, + "step": 1980 + }, + { + "epoch": 2.424131627056673, + "grad_norm": 0.18266689777374268, + "learning_rate": 5e-05, + "loss": 0.6697, + "step": 1990 + }, + { + "epoch": 2.436319317489336, + "grad_norm": 0.16344764828681946, + "learning_rate": 5e-05, + "loss": 0.6827, + "step": 2000 + }, + { + "epoch": 2.448507007921999, + "grad_norm": 0.18712133169174194, + "learning_rate": 5e-05, + "loss": 0.6713, + "step": 2010 + }, + { + "epoch": 2.460694698354662, + "grad_norm": 0.2192094475030899, + "learning_rate": 5e-05, + "loss": 0.6761, + "step": 2020 + }, + { + "epoch": 2.472882388787325, + "grad_norm": 0.18980957567691803, + "learning_rate": 5e-05, + "loss": 0.7001, + "step": 2030 + }, + { + "epoch": 2.485070079219988, + "grad_norm": 0.20758011937141418, + "learning_rate": 5e-05, + "loss": 0.6801, + "step": 2040 + }, + { + "epoch": 2.497257769652651, + "grad_norm": 0.166823148727417, + "learning_rate": 5e-05, + "loss": 0.6667, + "step": 2050 + } + ], + "logging_steps": 10, + "max_steps": 2463, + "num_input_tokens_seen": 0, + "num_train_epochs": 3, + "save_steps": 205, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 8.407791830057353e+18, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2050/training_args.bin b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2050/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..23ae0ec8a4226be5457f61c137abab2b2c10a0cf --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2050/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56896ea21940e33d7e2934267dff2a5b4a4a80195a0f165245f6617a0f88b90b +size 7377 diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2050/zero_to_fp32.py b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2050/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..5995d6e6f04e43b989587aa9022a3aef0c66d694 --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2050/zero_to_fp32.py @@ -0,0 +1,760 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: +# python zero_to_fp32.py . output_dir/ +# or +# python zero_to_fp32.py . output_dir/ --safe_serialization + +import argparse +import torch +import glob +import math +import os +import re +import gc +import json +import numpy as np +from tqdm import tqdm +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device, weights_only=False) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + total_files = len(files) + state_dicts = [] + for f in tqdm(files, desc='Loading checkpoint shards'): + state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + model_files = get_model_state_files(ds_checkpoint_dir) + + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +class GatheredTensor: + """ + A pseudo tensor that collects partitioned weights. + It is more memory efficient when there are multiple groups. + """ + + def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape): + self.flat_groups = flat_groups + self.flat_groups_offset = flat_groups_offset + self.offset = offset + self.partitioned_numel = partitioned_numel + self.shape = shape + self.dtype = self.flat_groups[0][0].dtype + + def contiguous(self): + """ + Merge partitioned weights from flat_groups into a single tensor. + """ + end_idx = self.offset + self.partitioned_numel + world_size = len(self.flat_groups) + pad_flat_param_chunks = [] + + for rank_i in range(world_size): + # for each rank, we need to collect weights from related group/groups + flat_groups_at_rank_i = self.flat_groups[rank_i] + start_group_id = None + end_group_id = None + for group_id in range(len(self.flat_groups_offset)): + if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]: + start_group_id = group_id + if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]: + end_group_id = group_id + break + # collect weights from related group/groups + for group_id in range(start_group_id, end_group_id + 1): + flat_tensor = flat_groups_at_rank_i[group_id] + start_offset = self.offset - self.flat_groups_offset[group_id] + end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id] + pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset]) + + # collect weights from all ranks + pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0) + param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous() + return param + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size + + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]])) + for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'): + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # memory efficient tensor + tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape) + state_dict[name] = tensor + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def to_torch_tensor(state_dict, return_empty_tensor=False): + """ + Convert state_dict of GatheredTensor to torch tensor + """ + torch_state_dict = {} + converted_tensors = {} + for name, tensor in state_dict.items(): + tensor_id = id(tensor) + if tensor_id in converted_tensors: # shared tensors + shared_tensor = torch_state_dict[converted_tensors[tensor_id]] + torch_state_dict[name] = shared_tensor + else: + converted_tensors[tensor_id] = name + if return_empty_tensor: + torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype) + else: + torch_state_dict[name] = tensor.contiguous() + return torch_state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag=None, + exclude_frozen_parameters=False, + lazy_mode=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient. + Convert the pesduo tensor to torch tensor by ``.contiguous()`` + + Returns: + - pytorch ``state_dict`` + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + Note: the above usage may not work if your application doesn't have sufficient free CPU memory. + You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. Or you can load state_dict in lazy mode :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu + for name, lazy_tensor in state_dict.item(): + tensor = lazy_tensor.contiguous() # to cpu + print(name, tensor) + # del tensor to release memory if it no longer in use + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + if lazy_mode: + return state_dict + else: + return to_torch_tensor(state_dict) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, + output_dir, + max_shard_size="5GB", + safe_serialization=False, + tag=None, + exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_dir``: directory to the pytorch fp32 state_dict output files + - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB + - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + + # Dependency pre-check + if safe_serialization: + try: + from safetensors.torch import save_file + except ImportError: + print('If you want to use `safe_serialization`, please `pip install safetensors`') + raise + if max_shard_size is not None: + try: + from huggingface_hub import split_torch_state_dict_into_shards + except ImportError: + print('If you want to use `max_shard_size`, please `pip install huggingface_hub`') + raise + + # Convert zero checkpoint to state_dict + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag, + exclude_frozen_parameters, + lazy_mode=True) + + # Shard the model if it is too big. + weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin" + if max_shard_size is not None: + filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors") + # an memory-efficient approach for sharding + empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True) + state_dict_split = split_torch_state_dict_into_shards(empty_state_dict, + filename_pattern=filename_pattern, + max_shard_size=max_shard_size) + else: + from collections import namedtuple + StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"]) + state_dict_split = StateDictSplit(is_sharded=False, + filename_to_tensors={weights_name: list(state_dict.keys())}) + + # Save the model by shard + os.makedirs(output_dir, exist_ok=True) + filename_to_tensors = state_dict_split.filename_to_tensors.items() + for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"): + shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors} + shard_state_dict = to_torch_tensor(shard_state_dict) + output_path = os.path.join(output_dir, shard_file) + if safe_serialization: + save_file(shard_state_dict, output_path, metadata={"format": "pt"}) + else: + torch.save(shard_state_dict, output_path) + # release the memory of current shard + for tensor_name in list(shard_state_dict.keys()): + del state_dict[tensor_name] + del shard_state_dict[tensor_name] + del shard_state_dict + gc.collect() + + # Save index if sharded + if state_dict_split.is_sharded: + index = { + "metadata": state_dict_split.metadata, + "weight_map": state_dict_split.tensor_to_filename, + } + save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json" + save_index_file = os.path.join(output_dir, save_index_file) + with open(save_index_file, "w", encoding="utf-8") as f: + content = json.dumps(index, indent=2, sort_keys=True) + "\n" + f.write(content) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info("Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info("Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument("output_dir", + type=str, + help="directory to the pytorch fp32 state_dict output files" + "(e.g. path/checkpoint-12-output/)") + parser.add_argument( + "--max_shard_size", + type=str, + default="5GB", + help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size" + "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`" + "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances" + "without CPU OOM issues.") + parser.add_argument( + "--safe_serialization", + default=False, + action='store_true', + help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_dir, + max_shard_size=args.max_shard_size, + safe_serialization=args.safe_serialization, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2255/config.json b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2255/config.json new file mode 100644 index 0000000000000000000000000000000000000000..18bd58d561964a16b60687f1b04aefdfed0b9b20 --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2255/config.json @@ -0,0 +1,39 @@ +{ + "architectures": [ + "LlamaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "dtype": "bfloat16", + "eos_token_id": [ + 128001, + 128008, + 128009 + ], + "head_dim": 64, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 8192, + "max_position_embeddings": 131072, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 32, + "num_hidden_layers": 16, + "num_key_value_heads": 8, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": { + "factor": 32.0, + "high_freq_factor": 4.0, + "low_freq_factor": 1.0, + "original_max_position_embeddings": 8192, + "rope_type": "llama3" + }, + "rope_theta": 500000.0, + "tie_word_embeddings": true, + "transformers_version": "4.57.1", + "use_cache": false, + "vocab_size": 128256 +} diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2255/generation_config.json b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2255/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..125cc74759bc705ab8a0c48d07d7eac083f56011 --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2255/generation_config.json @@ -0,0 +1,12 @@ +{ + "bos_token_id": 128000, + "do_sample": true, + "eos_token_id": [ + 128001, + 128008, + 128009 + ], + "temperature": 0.6, + "top_p": 0.9, + "transformers_version": "4.57.1" +} diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2255/latest b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2255/latest new file mode 100644 index 0000000000000000000000000000000000000000..5b1c6e5f28d5f27fd8ab400c959b9753b84891b1 --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2255/latest @@ -0,0 +1 @@ +global_step2255 \ No newline at end of file diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2255/model.safetensors b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2255/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..7d9d1cfb150294eb438c57b8fcde6b47926d9c1d --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2255/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1a18b43d94a3165eed06ac0cf1a799ae92538e7d16457849345e08d028c6df93 +size 2996982344 diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2255/scheduler.pt b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2255/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..34f652cab05d6fc9d0a8de0a33faf5e4951a7a4e --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2255/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0c41992be1395d92e961ff05cc071b8a2f23f6eced3f0009f84211ee9d36ba09 +size 1465 diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2255/trainer_state.json b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2255/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..47c4b50925b35ba6a328a09ea55ec1ecb1330a88 --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2255/trainer_state.json @@ -0,0 +1,1609 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 2.7471054235222425, + "eval_steps": 500, + "global_step": 2255, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.01218769043266301, + "grad_norm": 0.36486390233039856, + "learning_rate": 5e-05, + "loss": 1.2213, + "step": 10 + }, + { + "epoch": 0.02437538086532602, + "grad_norm": 0.2953564524650574, + "learning_rate": 5e-05, + "loss": 1.0223, + "step": 20 + }, + { + "epoch": 0.03656307129798903, + "grad_norm": 0.2766735851764679, + "learning_rate": 5e-05, + "loss": 0.9589, + "step": 30 + }, + { + "epoch": 0.04875076173065204, + "grad_norm": 0.32770997285842896, + "learning_rate": 5e-05, + "loss": 0.9081, + "step": 40 + }, + { + "epoch": 0.06093845216331505, + "grad_norm": 0.3052276372909546, + "learning_rate": 5e-05, + "loss": 0.8752, + "step": 50 + }, + { + "epoch": 0.07312614259597806, + "grad_norm": 0.240146666765213, + "learning_rate": 5e-05, + "loss": 0.873, + "step": 60 + }, + { + "epoch": 0.08531383302864107, + "grad_norm": 0.23872187733650208, + "learning_rate": 5e-05, + "loss": 0.8587, + "step": 70 + }, + { + "epoch": 0.09750152346130408, + "grad_norm": 0.32638487219810486, + "learning_rate": 5e-05, + "loss": 0.8656, + "step": 80 + }, + { + "epoch": 0.10968921389396709, + "grad_norm": 0.25019028782844543, + "learning_rate": 5e-05, + "loss": 0.8685, + "step": 90 + }, + { + "epoch": 0.1218769043266301, + "grad_norm": 0.2507675886154175, + "learning_rate": 5e-05, + "loss": 0.8662, + "step": 100 + }, + { + "epoch": 0.1340645947592931, + "grad_norm": 0.27966755628585815, + "learning_rate": 5e-05, + "loss": 0.8428, + "step": 110 + }, + { + "epoch": 0.14625228519195613, + "grad_norm": 0.2622673213481903, + "learning_rate": 5e-05, + "loss": 0.8416, + "step": 120 + }, + { + "epoch": 0.15843997562461914, + "grad_norm": 0.3003966510295868, + "learning_rate": 5e-05, + "loss": 0.8528, + "step": 130 + }, + { + "epoch": 0.17062766605728213, + "grad_norm": 0.32308319211006165, + "learning_rate": 5e-05, + "loss": 0.8357, + "step": 140 + }, + { + "epoch": 0.18281535648994515, + "grad_norm": 0.27279096841812134, + "learning_rate": 5e-05, + "loss": 0.8432, + "step": 150 + }, + { + "epoch": 0.19500304692260817, + "grad_norm": 0.27976104617118835, + "learning_rate": 5e-05, + "loss": 0.8119, + "step": 160 + }, + { + "epoch": 0.2071907373552712, + "grad_norm": 0.290866881608963, + "learning_rate": 5e-05, + "loss": 0.8096, + "step": 170 + }, + { + "epoch": 0.21937842778793418, + "grad_norm": 0.29430949687957764, + "learning_rate": 5e-05, + "loss": 0.819, + "step": 180 + }, + { + "epoch": 0.2315661182205972, + "grad_norm": 0.27714136242866516, + "learning_rate": 5e-05, + "loss": 0.8187, + "step": 190 + }, + { + "epoch": 0.2437538086532602, + "grad_norm": 0.2997029423713684, + "learning_rate": 5e-05, + "loss": 0.808, + "step": 200 + }, + { + "epoch": 0.25594149908592323, + "grad_norm": 0.2539573609828949, + "learning_rate": 5e-05, + "loss": 0.7943, + "step": 210 + }, + { + "epoch": 0.2681291895185862, + "grad_norm": 0.23294171690940857, + "learning_rate": 5e-05, + "loss": 0.8188, + "step": 220 + }, + { + "epoch": 0.28031687995124926, + "grad_norm": 0.25016292929649353, + "learning_rate": 5e-05, + "loss": 0.8035, + "step": 230 + }, + { + "epoch": 0.29250457038391225, + "grad_norm": 0.37125322222709656, + "learning_rate": 5e-05, + "loss": 0.785, + "step": 240 + }, + { + "epoch": 0.30469226081657524, + "grad_norm": 0.21520452201366425, + "learning_rate": 5e-05, + "loss": 0.8088, + "step": 250 + }, + { + "epoch": 0.3168799512492383, + "grad_norm": 0.20990602672100067, + "learning_rate": 5e-05, + "loss": 0.7813, + "step": 260 + }, + { + "epoch": 0.3290676416819013, + "grad_norm": 0.24193961918354034, + "learning_rate": 5e-05, + "loss": 0.7979, + "step": 270 + }, + { + "epoch": 0.34125533211456427, + "grad_norm": 0.22766050696372986, + "learning_rate": 5e-05, + "loss": 0.785, + "step": 280 + }, + { + "epoch": 0.3534430225472273, + "grad_norm": 0.2662702202796936, + "learning_rate": 5e-05, + "loss": 0.7766, + "step": 290 + }, + { + "epoch": 0.3656307129798903, + "grad_norm": 0.23020495474338531, + "learning_rate": 5e-05, + "loss": 0.7687, + "step": 300 + }, + { + "epoch": 0.3778184034125533, + "grad_norm": 0.21770520508289337, + "learning_rate": 5e-05, + "loss": 0.7815, + "step": 310 + }, + { + "epoch": 0.39000609384521634, + "grad_norm": 0.22677403688430786, + "learning_rate": 5e-05, + "loss": 0.7999, + "step": 320 + }, + { + "epoch": 0.40219378427787933, + "grad_norm": 0.24891860783100128, + "learning_rate": 5e-05, + "loss": 0.7946, + "step": 330 + }, + { + "epoch": 0.4143814747105424, + "grad_norm": 0.2506854832172394, + "learning_rate": 5e-05, + "loss": 0.7889, + "step": 340 + }, + { + "epoch": 0.42656916514320536, + "grad_norm": 0.24388180673122406, + "learning_rate": 5e-05, + "loss": 0.79, + "step": 350 + }, + { + "epoch": 0.43875685557586835, + "grad_norm": 0.22329650819301605, + "learning_rate": 5e-05, + "loss": 0.7632, + "step": 360 + }, + { + "epoch": 0.4509445460085314, + "grad_norm": 0.2240927815437317, + "learning_rate": 5e-05, + "loss": 0.8042, + "step": 370 + }, + { + "epoch": 0.4631322364411944, + "grad_norm": 0.2578203082084656, + "learning_rate": 5e-05, + "loss": 0.7667, + "step": 380 + }, + { + "epoch": 0.4753199268738574, + "grad_norm": 0.25349217653274536, + "learning_rate": 5e-05, + "loss": 0.7911, + "step": 390 + }, + { + "epoch": 0.4875076173065204, + "grad_norm": 0.254446804523468, + "learning_rate": 5e-05, + "loss": 0.7909, + "step": 400 + }, + { + "epoch": 0.4996953077391834, + "grad_norm": 0.24253171682357788, + "learning_rate": 5e-05, + "loss": 0.7896, + "step": 410 + }, + { + "epoch": 0.5118829981718465, + "grad_norm": 0.27901458740234375, + "learning_rate": 5e-05, + "loss": 0.7828, + "step": 420 + }, + { + "epoch": 0.5240706886045094, + "grad_norm": 0.31884434819221497, + "learning_rate": 5e-05, + "loss": 0.7919, + "step": 430 + }, + { + "epoch": 0.5362583790371724, + "grad_norm": 0.22920799255371094, + "learning_rate": 5e-05, + "loss": 0.7777, + "step": 440 + }, + { + "epoch": 0.5484460694698354, + "grad_norm": 0.22606618702411652, + "learning_rate": 5e-05, + "loss": 0.7661, + "step": 450 + }, + { + "epoch": 0.5606337599024985, + "grad_norm": 0.21604052186012268, + "learning_rate": 5e-05, + "loss": 0.769, + "step": 460 + }, + { + "epoch": 0.5728214503351615, + "grad_norm": 0.2296101152896881, + "learning_rate": 5e-05, + "loss": 0.7626, + "step": 470 + }, + { + "epoch": 0.5850091407678245, + "grad_norm": 0.24162150919437408, + "learning_rate": 5e-05, + "loss": 0.7831, + "step": 480 + }, + { + "epoch": 0.5971968312004875, + "grad_norm": 0.22364799678325653, + "learning_rate": 5e-05, + "loss": 0.7737, + "step": 490 + }, + { + "epoch": 0.6093845216331505, + "grad_norm": 0.23953403532505035, + "learning_rate": 5e-05, + "loss": 0.787, + "step": 500 + }, + { + "epoch": 0.6215722120658135, + "grad_norm": 0.22349311411380768, + "learning_rate": 5e-05, + "loss": 0.7763, + "step": 510 + }, + { + "epoch": 0.6337599024984766, + "grad_norm": 0.21996377408504486, + "learning_rate": 5e-05, + "loss": 0.7596, + "step": 520 + }, + { + "epoch": 0.6459475929311396, + "grad_norm": 0.245538130402565, + "learning_rate": 5e-05, + "loss": 0.7785, + "step": 530 + }, + { + "epoch": 0.6581352833638026, + "grad_norm": 0.23442520201206207, + "learning_rate": 5e-05, + "loss": 0.7633, + "step": 540 + }, + { + "epoch": 0.6703229737964655, + "grad_norm": 0.22274106740951538, + "learning_rate": 5e-05, + "loss": 0.7756, + "step": 550 + }, + { + "epoch": 0.6825106642291285, + "grad_norm": 0.21717530488967896, + "learning_rate": 5e-05, + "loss": 0.7726, + "step": 560 + }, + { + "epoch": 0.6946983546617916, + "grad_norm": 0.2465955913066864, + "learning_rate": 5e-05, + "loss": 0.7476, + "step": 570 + }, + { + "epoch": 0.7068860450944546, + "grad_norm": 0.2318575084209442, + "learning_rate": 5e-05, + "loss": 0.7584, + "step": 580 + }, + { + "epoch": 0.7190737355271176, + "grad_norm": 0.25114378333091736, + "learning_rate": 5e-05, + "loss": 0.7625, + "step": 590 + }, + { + "epoch": 0.7312614259597806, + "grad_norm": 0.19145755469799042, + "learning_rate": 5e-05, + "loss": 0.7577, + "step": 600 + }, + { + "epoch": 0.7434491163924436, + "grad_norm": 0.19129066169261932, + "learning_rate": 5e-05, + "loss": 0.7772, + "step": 610 + }, + { + "epoch": 0.7556368068251066, + "grad_norm": 0.2014475166797638, + "learning_rate": 5e-05, + "loss": 0.7601, + "step": 620 + }, + { + "epoch": 0.7678244972577697, + "grad_norm": 0.21480509638786316, + "learning_rate": 5e-05, + "loss": 0.7665, + "step": 630 + }, + { + "epoch": 0.7800121876904327, + "grad_norm": 0.2633228302001953, + "learning_rate": 5e-05, + "loss": 0.7565, + "step": 640 + }, + { + "epoch": 0.7921998781230957, + "grad_norm": 0.24633203446865082, + "learning_rate": 5e-05, + "loss": 0.7602, + "step": 650 + }, + { + "epoch": 0.8043875685557587, + "grad_norm": 0.19770534336566925, + "learning_rate": 5e-05, + "loss": 0.7685, + "step": 660 + }, + { + "epoch": 0.8165752589884216, + "grad_norm": 0.2267930805683136, + "learning_rate": 5e-05, + "loss": 0.7516, + "step": 670 + }, + { + "epoch": 0.8287629494210847, + "grad_norm": 0.238111674785614, + "learning_rate": 5e-05, + "loss": 0.7487, + "step": 680 + }, + { + "epoch": 0.8409506398537477, + "grad_norm": 0.19528904557228088, + "learning_rate": 5e-05, + "loss": 0.7467, + "step": 690 + }, + { + "epoch": 0.8531383302864107, + "grad_norm": 0.21305914223194122, + "learning_rate": 5e-05, + "loss": 0.7527, + "step": 700 + }, + { + "epoch": 0.8653260207190737, + "grad_norm": 0.2205205112695694, + "learning_rate": 5e-05, + "loss": 0.7639, + "step": 710 + }, + { + "epoch": 0.8775137111517367, + "grad_norm": 0.22429127991199493, + "learning_rate": 5e-05, + "loss": 0.7744, + "step": 720 + }, + { + "epoch": 0.8897014015843998, + "grad_norm": 0.23755772411823273, + "learning_rate": 5e-05, + "loss": 0.7446, + "step": 730 + }, + { + "epoch": 0.9018890920170628, + "grad_norm": 0.17592693865299225, + "learning_rate": 5e-05, + "loss": 0.7442, + "step": 740 + }, + { + "epoch": 0.9140767824497258, + "grad_norm": 0.2266898900270462, + "learning_rate": 5e-05, + "loss": 0.7326, + "step": 750 + }, + { + "epoch": 0.9262644728823888, + "grad_norm": 0.23541569709777832, + "learning_rate": 5e-05, + "loss": 0.7338, + "step": 760 + }, + { + "epoch": 0.9384521633150518, + "grad_norm": 0.22233888506889343, + "learning_rate": 5e-05, + "loss": 0.7385, + "step": 770 + }, + { + "epoch": 0.9506398537477148, + "grad_norm": 0.23714767396450043, + "learning_rate": 5e-05, + "loss": 0.7306, + "step": 780 + }, + { + "epoch": 0.9628275441803779, + "grad_norm": 0.21191060543060303, + "learning_rate": 5e-05, + "loss": 0.736, + "step": 790 + }, + { + "epoch": 0.9750152346130408, + "grad_norm": 0.22759106755256653, + "learning_rate": 5e-05, + "loss": 0.7505, + "step": 800 + }, + { + "epoch": 0.9872029250457038, + "grad_norm": 0.19895903766155243, + "learning_rate": 5e-05, + "loss": 0.7536, + "step": 810 + }, + { + "epoch": 0.9993906154783668, + "grad_norm": 0.3061685860157013, + "learning_rate": 5e-05, + "loss": 0.7493, + "step": 820 + }, + { + "epoch": 1.0109689213893966, + "grad_norm": 0.18868015706539154, + "learning_rate": 5e-05, + "loss": 0.7342, + "step": 830 + }, + { + "epoch": 1.0231566118220596, + "grad_norm": 0.1797628402709961, + "learning_rate": 5e-05, + "loss": 0.7192, + "step": 840 + }, + { + "epoch": 1.0353443022547226, + "grad_norm": 0.19511596858501434, + "learning_rate": 5e-05, + "loss": 0.7002, + "step": 850 + }, + { + "epoch": 1.0475319926873858, + "grad_norm": 0.2123185396194458, + "learning_rate": 5e-05, + "loss": 0.7236, + "step": 860 + }, + { + "epoch": 1.0597196831200488, + "grad_norm": 0.2047470211982727, + "learning_rate": 5e-05, + "loss": 0.711, + "step": 870 + }, + { + "epoch": 1.0719073735527118, + "grad_norm": 0.18809868395328522, + "learning_rate": 5e-05, + "loss": 0.7253, + "step": 880 + }, + { + "epoch": 1.0840950639853748, + "grad_norm": 0.2207012176513672, + "learning_rate": 5e-05, + "loss": 0.6971, + "step": 890 + }, + { + "epoch": 1.0962827544180378, + "grad_norm": 0.1912035495042801, + "learning_rate": 5e-05, + "loss": 0.7239, + "step": 900 + }, + { + "epoch": 1.1084704448507008, + "grad_norm": 0.23486360907554626, + "learning_rate": 5e-05, + "loss": 0.7335, + "step": 910 + }, + { + "epoch": 1.1206581352833638, + "grad_norm": 0.22189787030220032, + "learning_rate": 5e-05, + "loss": 0.7149, + "step": 920 + }, + { + "epoch": 1.1328458257160268, + "grad_norm": 0.25818824768066406, + "learning_rate": 5e-05, + "loss": 0.7322, + "step": 930 + }, + { + "epoch": 1.1450335161486898, + "grad_norm": 0.2088775783777237, + "learning_rate": 5e-05, + "loss": 0.7382, + "step": 940 + }, + { + "epoch": 1.1572212065813527, + "grad_norm": 0.19445833563804626, + "learning_rate": 5e-05, + "loss": 0.7223, + "step": 950 + }, + { + "epoch": 1.1694088970140157, + "grad_norm": 0.21489456295967102, + "learning_rate": 5e-05, + "loss": 0.7317, + "step": 960 + }, + { + "epoch": 1.181596587446679, + "grad_norm": 0.24050794541835785, + "learning_rate": 5e-05, + "loss": 0.7275, + "step": 970 + }, + { + "epoch": 1.193784277879342, + "grad_norm": 0.18142102658748627, + "learning_rate": 5e-05, + "loss": 0.7046, + "step": 980 + }, + { + "epoch": 1.205971968312005, + "grad_norm": 0.4781077802181244, + "learning_rate": 5e-05, + "loss": 0.7433, + "step": 990 + }, + { + "epoch": 1.218159658744668, + "grad_norm": 0.20628023147583008, + "learning_rate": 5e-05, + "loss": 0.7403, + "step": 1000 + }, + { + "epoch": 1.230347349177331, + "grad_norm": 0.19117140769958496, + "learning_rate": 5e-05, + "loss": 0.7308, + "step": 1010 + }, + { + "epoch": 1.242535039609994, + "grad_norm": 0.2107745110988617, + "learning_rate": 5e-05, + "loss": 0.7201, + "step": 1020 + }, + { + "epoch": 1.2547227300426569, + "grad_norm": 0.20551154017448425, + "learning_rate": 5e-05, + "loss": 0.7038, + "step": 1030 + }, + { + "epoch": 1.2669104204753199, + "grad_norm": 0.26577022671699524, + "learning_rate": 5e-05, + "loss": 0.7096, + "step": 1040 + }, + { + "epoch": 1.2790981109079829, + "grad_norm": 0.22094209492206573, + "learning_rate": 5e-05, + "loss": 0.718, + "step": 1050 + }, + { + "epoch": 1.291285801340646, + "grad_norm": 0.18224076926708221, + "learning_rate": 5e-05, + "loss": 0.7261, + "step": 1060 + }, + { + "epoch": 1.303473491773309, + "grad_norm": 0.226251482963562, + "learning_rate": 5e-05, + "loss": 0.7372, + "step": 1070 + }, + { + "epoch": 1.315661182205972, + "grad_norm": 0.20186461508274078, + "learning_rate": 5e-05, + "loss": 0.7074, + "step": 1080 + }, + { + "epoch": 1.327848872638635, + "grad_norm": 0.22229762375354767, + "learning_rate": 5e-05, + "loss": 0.7069, + "step": 1090 + }, + { + "epoch": 1.340036563071298, + "grad_norm": 0.1776452213525772, + "learning_rate": 5e-05, + "loss": 0.714, + "step": 1100 + }, + { + "epoch": 1.352224253503961, + "grad_norm": 0.20680229365825653, + "learning_rate": 5e-05, + "loss": 0.7066, + "step": 1110 + }, + { + "epoch": 1.364411943936624, + "grad_norm": 0.21842031180858612, + "learning_rate": 5e-05, + "loss": 0.7281, + "step": 1120 + }, + { + "epoch": 1.376599634369287, + "grad_norm": 0.28037774562835693, + "learning_rate": 5e-05, + "loss": 0.7149, + "step": 1130 + }, + { + "epoch": 1.38878732480195, + "grad_norm": 0.20962920784950256, + "learning_rate": 5e-05, + "loss": 0.6951, + "step": 1140 + }, + { + "epoch": 1.400975015234613, + "grad_norm": 0.18751871585845947, + "learning_rate": 5e-05, + "loss": 0.7101, + "step": 1150 + }, + { + "epoch": 1.413162705667276, + "grad_norm": 0.2722964584827423, + "learning_rate": 5e-05, + "loss": 0.7205, + "step": 1160 + }, + { + "epoch": 1.425350396099939, + "grad_norm": 0.20932041108608246, + "learning_rate": 5e-05, + "loss": 0.7082, + "step": 1170 + }, + { + "epoch": 1.437538086532602, + "grad_norm": 0.17836257815361023, + "learning_rate": 5e-05, + "loss": 0.7232, + "step": 1180 + }, + { + "epoch": 1.449725776965265, + "grad_norm": 0.3180703818798065, + "learning_rate": 5e-05, + "loss": 0.7245, + "step": 1190 + }, + { + "epoch": 1.4619134673979282, + "grad_norm": 0.201424703001976, + "learning_rate": 5e-05, + "loss": 0.7232, + "step": 1200 + }, + { + "epoch": 1.4741011578305911, + "grad_norm": 0.20228999853134155, + "learning_rate": 5e-05, + "loss": 0.7169, + "step": 1210 + }, + { + "epoch": 1.4862888482632541, + "grad_norm": 0.17478394508361816, + "learning_rate": 5e-05, + "loss": 0.704, + "step": 1220 + }, + { + "epoch": 1.4984765386959171, + "grad_norm": 0.20061351358890533, + "learning_rate": 5e-05, + "loss": 0.6792, + "step": 1230 + }, + { + "epoch": 1.5106642291285801, + "grad_norm": 0.1857377588748932, + "learning_rate": 5e-05, + "loss": 0.709, + "step": 1240 + }, + { + "epoch": 1.522851919561243, + "grad_norm": 0.19873379170894623, + "learning_rate": 5e-05, + "loss": 0.7007, + "step": 1250 + }, + { + "epoch": 1.5350396099939063, + "grad_norm": 0.19807074964046478, + "learning_rate": 5e-05, + "loss": 0.7122, + "step": 1260 + }, + { + "epoch": 1.5472273004265693, + "grad_norm": 0.16746796667575836, + "learning_rate": 5e-05, + "loss": 0.7005, + "step": 1270 + }, + { + "epoch": 1.5594149908592323, + "grad_norm": 0.2750326991081238, + "learning_rate": 5e-05, + "loss": 0.7166, + "step": 1280 + }, + { + "epoch": 1.5716026812918953, + "grad_norm": 0.2776738107204437, + "learning_rate": 5e-05, + "loss": 0.7405, + "step": 1290 + }, + { + "epoch": 1.5837903717245583, + "grad_norm": 0.3494517207145691, + "learning_rate": 5e-05, + "loss": 0.7192, + "step": 1300 + }, + { + "epoch": 1.5959780621572213, + "grad_norm": 0.25028494000434875, + "learning_rate": 5e-05, + "loss": 0.741, + "step": 1310 + }, + { + "epoch": 1.6081657525898843, + "grad_norm": 0.23914143443107605, + "learning_rate": 5e-05, + "loss": 0.72, + "step": 1320 + }, + { + "epoch": 1.6203534430225472, + "grad_norm": 0.2052862048149109, + "learning_rate": 5e-05, + "loss": 0.7112, + "step": 1330 + }, + { + "epoch": 1.6325411334552102, + "grad_norm": 0.22285936772823334, + "learning_rate": 5e-05, + "loss": 0.7165, + "step": 1340 + }, + { + "epoch": 1.6447288238878732, + "grad_norm": 0.20810848474502563, + "learning_rate": 5e-05, + "loss": 0.7122, + "step": 1350 + }, + { + "epoch": 1.6569165143205362, + "grad_norm": 0.20422884821891785, + "learning_rate": 5e-05, + "loss": 0.7082, + "step": 1360 + }, + { + "epoch": 1.6691042047531992, + "grad_norm": 0.18164314329624176, + "learning_rate": 5e-05, + "loss": 0.7131, + "step": 1370 + }, + { + "epoch": 1.6812918951858622, + "grad_norm": 0.19893014430999756, + "learning_rate": 5e-05, + "loss": 0.7055, + "step": 1380 + }, + { + "epoch": 1.6934795856185252, + "grad_norm": 0.24865971505641937, + "learning_rate": 5e-05, + "loss": 0.6994, + "step": 1390 + }, + { + "epoch": 1.7056672760511882, + "grad_norm": 0.18078123033046722, + "learning_rate": 5e-05, + "loss": 0.7162, + "step": 1400 + }, + { + "epoch": 1.7178549664838512, + "grad_norm": 0.1973421722650528, + "learning_rate": 5e-05, + "loss": 0.7136, + "step": 1410 + }, + { + "epoch": 1.7300426569165142, + "grad_norm": 0.2057911604642868, + "learning_rate": 5e-05, + "loss": 0.7004, + "step": 1420 + }, + { + "epoch": 1.7422303473491774, + "grad_norm": 0.18640495836734772, + "learning_rate": 5e-05, + "loss": 0.7102, + "step": 1430 + }, + { + "epoch": 1.7544180377818404, + "grad_norm": 0.21038417518138885, + "learning_rate": 5e-05, + "loss": 0.6908, + "step": 1440 + }, + { + "epoch": 1.7666057282145033, + "grad_norm": 0.2123890221118927, + "learning_rate": 5e-05, + "loss": 0.6913, + "step": 1450 + }, + { + "epoch": 1.7787934186471663, + "grad_norm": 0.21619991958141327, + "learning_rate": 5e-05, + "loss": 0.7038, + "step": 1460 + }, + { + "epoch": 1.7909811090798293, + "grad_norm": 0.1975220888853073, + "learning_rate": 5e-05, + "loss": 0.7041, + "step": 1470 + }, + { + "epoch": 1.8031687995124925, + "grad_norm": 0.20595207810401917, + "learning_rate": 5e-05, + "loss": 0.7145, + "step": 1480 + }, + { + "epoch": 1.8153564899451555, + "grad_norm": 0.16633334755897522, + "learning_rate": 5e-05, + "loss": 0.6963, + "step": 1490 + }, + { + "epoch": 1.8275441803778185, + "grad_norm": 0.19140620529651642, + "learning_rate": 5e-05, + "loss": 0.6814, + "step": 1500 + }, + { + "epoch": 1.8397318708104815, + "grad_norm": 0.1833905130624771, + "learning_rate": 5e-05, + "loss": 0.7174, + "step": 1510 + }, + { + "epoch": 1.8519195612431445, + "grad_norm": 0.20587337017059326, + "learning_rate": 5e-05, + "loss": 0.7098, + "step": 1520 + }, + { + "epoch": 1.8641072516758075, + "grad_norm": 0.21952909231185913, + "learning_rate": 5e-05, + "loss": 0.6912, + "step": 1530 + }, + { + "epoch": 1.8762949421084705, + "grad_norm": 0.23233620822429657, + "learning_rate": 5e-05, + "loss": 0.6992, + "step": 1540 + }, + { + "epoch": 1.8884826325411335, + "grad_norm": 0.17159809172153473, + "learning_rate": 5e-05, + "loss": 0.7052, + "step": 1550 + }, + { + "epoch": 1.9006703229737965, + "grad_norm": 0.23522134125232697, + "learning_rate": 5e-05, + "loss": 0.6873, + "step": 1560 + }, + { + "epoch": 1.9128580134064594, + "grad_norm": 0.21959669888019562, + "learning_rate": 5e-05, + "loss": 0.6834, + "step": 1570 + }, + { + "epoch": 1.9250457038391224, + "grad_norm": 0.2110578864812851, + "learning_rate": 5e-05, + "loss": 0.7038, + "step": 1580 + }, + { + "epoch": 1.9372333942717854, + "grad_norm": 0.18674372136592865, + "learning_rate": 5e-05, + "loss": 0.6743, + "step": 1590 + }, + { + "epoch": 1.9494210847044484, + "grad_norm": 0.18220046162605286, + "learning_rate": 5e-05, + "loss": 0.6943, + "step": 1600 + }, + { + "epoch": 1.9616087751371114, + "grad_norm": 0.20756569504737854, + "learning_rate": 5e-05, + "loss": 0.7049, + "step": 1610 + }, + { + "epoch": 1.9737964655697744, + "grad_norm": 0.1925211250782013, + "learning_rate": 5e-05, + "loss": 0.7033, + "step": 1620 + }, + { + "epoch": 1.9859841560024374, + "grad_norm": 0.21374447643756866, + "learning_rate": 5e-05, + "loss": 0.7025, + "step": 1630 + }, + { + "epoch": 1.9981718464351006, + "grad_norm": 0.1952880322933197, + "learning_rate": 5e-05, + "loss": 0.6877, + "step": 1640 + }, + { + "epoch": 2.0097501523461303, + "grad_norm": 0.20457723736763, + "learning_rate": 5e-05, + "loss": 0.6734, + "step": 1650 + }, + { + "epoch": 2.0219378427787933, + "grad_norm": 0.20965932309627533, + "learning_rate": 5e-05, + "loss": 0.6472, + "step": 1660 + }, + { + "epoch": 2.0341255332114563, + "grad_norm": 0.19478890299797058, + "learning_rate": 5e-05, + "loss": 0.6781, + "step": 1670 + }, + { + "epoch": 2.0463132236441193, + "grad_norm": 0.21520769596099854, + "learning_rate": 5e-05, + "loss": 0.6592, + "step": 1680 + }, + { + "epoch": 2.0585009140767823, + "grad_norm": 0.20154155790805817, + "learning_rate": 5e-05, + "loss": 0.6744, + "step": 1690 + }, + { + "epoch": 2.0706886045094453, + "grad_norm": 0.1899706870317459, + "learning_rate": 5e-05, + "loss": 0.6592, + "step": 1700 + }, + { + "epoch": 2.0828762949421087, + "grad_norm": 0.20379123091697693, + "learning_rate": 5e-05, + "loss": 0.6688, + "step": 1710 + }, + { + "epoch": 2.0950639853747717, + "grad_norm": 0.19799865782260895, + "learning_rate": 5e-05, + "loss": 0.6678, + "step": 1720 + }, + { + "epoch": 2.1072516758074347, + "grad_norm": 0.19186025857925415, + "learning_rate": 5e-05, + "loss": 0.6587, + "step": 1730 + }, + { + "epoch": 2.1194393662400977, + "grad_norm": 0.18936173617839813, + "learning_rate": 5e-05, + "loss": 0.668, + "step": 1740 + }, + { + "epoch": 2.1316270566727606, + "grad_norm": 0.19159527122974396, + "learning_rate": 5e-05, + "loss": 0.6643, + "step": 1750 + }, + { + "epoch": 2.1438147471054236, + "grad_norm": 0.20596368610858917, + "learning_rate": 5e-05, + "loss": 0.6667, + "step": 1760 + }, + { + "epoch": 2.1560024375380866, + "grad_norm": 0.18601712584495544, + "learning_rate": 5e-05, + "loss": 0.6598, + "step": 1770 + }, + { + "epoch": 2.1681901279707496, + "grad_norm": 0.2130655199289322, + "learning_rate": 5e-05, + "loss": 0.6844, + "step": 1780 + }, + { + "epoch": 2.1803778184034126, + "grad_norm": 0.20477338135242462, + "learning_rate": 5e-05, + "loss": 0.666, + "step": 1790 + }, + { + "epoch": 2.1925655088360756, + "grad_norm": 0.18484896421432495, + "learning_rate": 5e-05, + "loss": 0.6659, + "step": 1800 + }, + { + "epoch": 2.2047531992687386, + "grad_norm": 0.17023979127407074, + "learning_rate": 5e-05, + "loss": 0.6721, + "step": 1810 + }, + { + "epoch": 2.2169408897014016, + "grad_norm": 0.23462794721126556, + "learning_rate": 5e-05, + "loss": 0.6699, + "step": 1820 + }, + { + "epoch": 2.2291285801340646, + "grad_norm": 0.16521663963794708, + "learning_rate": 5e-05, + "loss": 0.6584, + "step": 1830 + }, + { + "epoch": 2.2413162705667276, + "grad_norm": 0.19637759029865265, + "learning_rate": 5e-05, + "loss": 0.6621, + "step": 1840 + }, + { + "epoch": 2.2535039609993905, + "grad_norm": 0.2099665403366089, + "learning_rate": 5e-05, + "loss": 0.6912, + "step": 1850 + }, + { + "epoch": 2.2656916514320535, + "grad_norm": 0.21308118104934692, + "learning_rate": 5e-05, + "loss": 0.6705, + "step": 1860 + }, + { + "epoch": 2.2778793418647165, + "grad_norm": 0.20638489723205566, + "learning_rate": 5e-05, + "loss": 0.6736, + "step": 1870 + }, + { + "epoch": 2.2900670322973795, + "grad_norm": 0.23812630772590637, + "learning_rate": 5e-05, + "loss": 0.6869, + "step": 1880 + }, + { + "epoch": 2.3022547227300425, + "grad_norm": 0.23223577439785004, + "learning_rate": 5e-05, + "loss": 0.6663, + "step": 1890 + }, + { + "epoch": 2.3144424131627055, + "grad_norm": 0.20844320952892303, + "learning_rate": 5e-05, + "loss": 0.6686, + "step": 1900 + }, + { + "epoch": 2.3266301035953685, + "grad_norm": 0.2517656683921814, + "learning_rate": 5e-05, + "loss": 0.658, + "step": 1910 + }, + { + "epoch": 2.3388177940280315, + "grad_norm": 0.20005013048648834, + "learning_rate": 5e-05, + "loss": 0.6659, + "step": 1920 + }, + { + "epoch": 2.3510054844606945, + "grad_norm": 0.20756477117538452, + "learning_rate": 5e-05, + "loss": 0.6613, + "step": 1930 + }, + { + "epoch": 2.363193174893358, + "grad_norm": 0.19551488757133484, + "learning_rate": 5e-05, + "loss": 0.676, + "step": 1940 + }, + { + "epoch": 2.375380865326021, + "grad_norm": 0.18600064516067505, + "learning_rate": 5e-05, + "loss": 0.6826, + "step": 1950 + }, + { + "epoch": 2.387568555758684, + "grad_norm": 0.2445133775472641, + "learning_rate": 5e-05, + "loss": 0.6623, + "step": 1960 + }, + { + "epoch": 2.399756246191347, + "grad_norm": 0.2174660712480545, + "learning_rate": 5e-05, + "loss": 0.6697, + "step": 1970 + }, + { + "epoch": 2.41194393662401, + "grad_norm": 0.1853487342596054, + "learning_rate": 5e-05, + "loss": 0.6561, + "step": 1980 + }, + { + "epoch": 2.424131627056673, + "grad_norm": 0.18266689777374268, + "learning_rate": 5e-05, + "loss": 0.6697, + "step": 1990 + }, + { + "epoch": 2.436319317489336, + "grad_norm": 0.16344764828681946, + "learning_rate": 5e-05, + "loss": 0.6827, + "step": 2000 + }, + { + "epoch": 2.448507007921999, + "grad_norm": 0.18712133169174194, + "learning_rate": 5e-05, + "loss": 0.6713, + "step": 2010 + }, + { + "epoch": 2.460694698354662, + "grad_norm": 0.2192094475030899, + "learning_rate": 5e-05, + "loss": 0.6761, + "step": 2020 + }, + { + "epoch": 2.472882388787325, + "grad_norm": 0.18980957567691803, + "learning_rate": 5e-05, + "loss": 0.7001, + "step": 2030 + }, + { + "epoch": 2.485070079219988, + "grad_norm": 0.20758011937141418, + "learning_rate": 5e-05, + "loss": 0.6801, + "step": 2040 + }, + { + "epoch": 2.497257769652651, + "grad_norm": 0.166823148727417, + "learning_rate": 5e-05, + "loss": 0.6667, + "step": 2050 + }, + { + "epoch": 2.5094454600853138, + "grad_norm": 0.17631390690803528, + "learning_rate": 5e-05, + "loss": 0.66, + "step": 2060 + }, + { + "epoch": 2.5216331505179768, + "grad_norm": 0.19680799543857574, + "learning_rate": 5e-05, + "loss": 0.6513, + "step": 2070 + }, + { + "epoch": 2.5338208409506398, + "grad_norm": 0.202071875333786, + "learning_rate": 5e-05, + "loss": 0.6655, + "step": 2080 + }, + { + "epoch": 2.5460085313833027, + "grad_norm": 0.1721998006105423, + "learning_rate": 5e-05, + "loss": 0.6564, + "step": 2090 + }, + { + "epoch": 2.5581962218159657, + "grad_norm": 0.17605794966220856, + "learning_rate": 5e-05, + "loss": 0.6715, + "step": 2100 + }, + { + "epoch": 2.5703839122486287, + "grad_norm": 0.2009730488061905, + "learning_rate": 5e-05, + "loss": 0.6764, + "step": 2110 + }, + { + "epoch": 2.582571602681292, + "grad_norm": 0.21355752646923065, + "learning_rate": 5e-05, + "loss": 0.6696, + "step": 2120 + }, + { + "epoch": 2.594759293113955, + "grad_norm": 0.18136924505233765, + "learning_rate": 5e-05, + "loss": 0.6459, + "step": 2130 + }, + { + "epoch": 2.606946983546618, + "grad_norm": 0.1764170080423355, + "learning_rate": 5e-05, + "loss": 0.6565, + "step": 2140 + }, + { + "epoch": 2.619134673979281, + "grad_norm": 0.1911628693342209, + "learning_rate": 5e-05, + "loss": 0.6534, + "step": 2150 + }, + { + "epoch": 2.631322364411944, + "grad_norm": 0.18562854826450348, + "learning_rate": 5e-05, + "loss": 0.6505, + "step": 2160 + }, + { + "epoch": 2.643510054844607, + "grad_norm": 0.17010731995105743, + "learning_rate": 5e-05, + "loss": 0.6676, + "step": 2170 + }, + { + "epoch": 2.65569774527727, + "grad_norm": 0.1590568870306015, + "learning_rate": 5e-05, + "loss": 0.638, + "step": 2180 + }, + { + "epoch": 2.667885435709933, + "grad_norm": 0.17398956418037415, + "learning_rate": 5e-05, + "loss": 0.6684, + "step": 2190 + }, + { + "epoch": 2.680073126142596, + "grad_norm": 0.1765497624874115, + "learning_rate": 5e-05, + "loss": 0.6633, + "step": 2200 + }, + { + "epoch": 2.692260816575259, + "grad_norm": 0.19518159329891205, + "learning_rate": 5e-05, + "loss": 0.6742, + "step": 2210 + }, + { + "epoch": 2.704448507007922, + "grad_norm": 0.22384262084960938, + "learning_rate": 5e-05, + "loss": 0.6544, + "step": 2220 + }, + { + "epoch": 2.716636197440585, + "grad_norm": 0.1654004603624344, + "learning_rate": 5e-05, + "loss": 0.6572, + "step": 2230 + }, + { + "epoch": 2.728823887873248, + "grad_norm": 0.18328379094600677, + "learning_rate": 5e-05, + "loss": 0.6714, + "step": 2240 + }, + { + "epoch": 2.741011578305911, + "grad_norm": 0.26779523491859436, + "learning_rate": 5e-05, + "loss": 0.6624, + "step": 2250 + } + ], + "logging_steps": 10, + "max_steps": 2463, + "num_input_tokens_seen": 0, + "num_train_epochs": 3, + "save_steps": 205, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 9.246895589914837e+18, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2255/training_args.bin b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2255/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..23ae0ec8a4226be5457f61c137abab2b2c10a0cf --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2255/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56896ea21940e33d7e2934267dff2a5b4a4a80195a0f165245f6617a0f88b90b +size 7377 diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2255/zero_to_fp32.py b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2255/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..5995d6e6f04e43b989587aa9022a3aef0c66d694 --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2255/zero_to_fp32.py @@ -0,0 +1,760 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: +# python zero_to_fp32.py . output_dir/ +# or +# python zero_to_fp32.py . output_dir/ --safe_serialization + +import argparse +import torch +import glob +import math +import os +import re +import gc +import json +import numpy as np +from tqdm import tqdm +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device, weights_only=False) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + total_files = len(files) + state_dicts = [] + for f in tqdm(files, desc='Loading checkpoint shards'): + state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + model_files = get_model_state_files(ds_checkpoint_dir) + + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +class GatheredTensor: + """ + A pseudo tensor that collects partitioned weights. + It is more memory efficient when there are multiple groups. + """ + + def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape): + self.flat_groups = flat_groups + self.flat_groups_offset = flat_groups_offset + self.offset = offset + self.partitioned_numel = partitioned_numel + self.shape = shape + self.dtype = self.flat_groups[0][0].dtype + + def contiguous(self): + """ + Merge partitioned weights from flat_groups into a single tensor. + """ + end_idx = self.offset + self.partitioned_numel + world_size = len(self.flat_groups) + pad_flat_param_chunks = [] + + for rank_i in range(world_size): + # for each rank, we need to collect weights from related group/groups + flat_groups_at_rank_i = self.flat_groups[rank_i] + start_group_id = None + end_group_id = None + for group_id in range(len(self.flat_groups_offset)): + if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]: + start_group_id = group_id + if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]: + end_group_id = group_id + break + # collect weights from related group/groups + for group_id in range(start_group_id, end_group_id + 1): + flat_tensor = flat_groups_at_rank_i[group_id] + start_offset = self.offset - self.flat_groups_offset[group_id] + end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id] + pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset]) + + # collect weights from all ranks + pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0) + param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous() + return param + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size + + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]])) + for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'): + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # memory efficient tensor + tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape) + state_dict[name] = tensor + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def to_torch_tensor(state_dict, return_empty_tensor=False): + """ + Convert state_dict of GatheredTensor to torch tensor + """ + torch_state_dict = {} + converted_tensors = {} + for name, tensor in state_dict.items(): + tensor_id = id(tensor) + if tensor_id in converted_tensors: # shared tensors + shared_tensor = torch_state_dict[converted_tensors[tensor_id]] + torch_state_dict[name] = shared_tensor + else: + converted_tensors[tensor_id] = name + if return_empty_tensor: + torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype) + else: + torch_state_dict[name] = tensor.contiguous() + return torch_state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag=None, + exclude_frozen_parameters=False, + lazy_mode=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient. + Convert the pesduo tensor to torch tensor by ``.contiguous()`` + + Returns: + - pytorch ``state_dict`` + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + Note: the above usage may not work if your application doesn't have sufficient free CPU memory. + You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. Or you can load state_dict in lazy mode :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu + for name, lazy_tensor in state_dict.item(): + tensor = lazy_tensor.contiguous() # to cpu + print(name, tensor) + # del tensor to release memory if it no longer in use + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + if lazy_mode: + return state_dict + else: + return to_torch_tensor(state_dict) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, + output_dir, + max_shard_size="5GB", + safe_serialization=False, + tag=None, + exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_dir``: directory to the pytorch fp32 state_dict output files + - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB + - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + + # Dependency pre-check + if safe_serialization: + try: + from safetensors.torch import save_file + except ImportError: + print('If you want to use `safe_serialization`, please `pip install safetensors`') + raise + if max_shard_size is not None: + try: + from huggingface_hub import split_torch_state_dict_into_shards + except ImportError: + print('If you want to use `max_shard_size`, please `pip install huggingface_hub`') + raise + + # Convert zero checkpoint to state_dict + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag, + exclude_frozen_parameters, + lazy_mode=True) + + # Shard the model if it is too big. + weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin" + if max_shard_size is not None: + filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors") + # an memory-efficient approach for sharding + empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True) + state_dict_split = split_torch_state_dict_into_shards(empty_state_dict, + filename_pattern=filename_pattern, + max_shard_size=max_shard_size) + else: + from collections import namedtuple + StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"]) + state_dict_split = StateDictSplit(is_sharded=False, + filename_to_tensors={weights_name: list(state_dict.keys())}) + + # Save the model by shard + os.makedirs(output_dir, exist_ok=True) + filename_to_tensors = state_dict_split.filename_to_tensors.items() + for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"): + shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors} + shard_state_dict = to_torch_tensor(shard_state_dict) + output_path = os.path.join(output_dir, shard_file) + if safe_serialization: + save_file(shard_state_dict, output_path, metadata={"format": "pt"}) + else: + torch.save(shard_state_dict, output_path) + # release the memory of current shard + for tensor_name in list(shard_state_dict.keys()): + del state_dict[tensor_name] + del shard_state_dict[tensor_name] + del shard_state_dict + gc.collect() + + # Save index if sharded + if state_dict_split.is_sharded: + index = { + "metadata": state_dict_split.metadata, + "weight_map": state_dict_split.tensor_to_filename, + } + save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json" + save_index_file = os.path.join(output_dir, save_index_file) + with open(save_index_file, "w", encoding="utf-8") as f: + content = json.dumps(index, indent=2, sort_keys=True) + "\n" + f.write(content) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info("Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info("Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument("output_dir", + type=str, + help="directory to the pytorch fp32 state_dict output files" + "(e.g. path/checkpoint-12-output/)") + parser.add_argument( + "--max_shard_size", + type=str, + default="5GB", + help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size" + "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`" + "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances" + "without CPU OOM issues.") + parser.add_argument( + "--safe_serialization", + default=False, + action='store_true', + help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_dir, + max_shard_size=args.max_shard_size, + safe_serialization=args.safe_serialization, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2460/config.json b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2460/config.json new file mode 100644 index 0000000000000000000000000000000000000000..18bd58d561964a16b60687f1b04aefdfed0b9b20 --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2460/config.json @@ -0,0 +1,39 @@ +{ + "architectures": [ + "LlamaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "dtype": "bfloat16", + "eos_token_id": [ + 128001, + 128008, + 128009 + ], + "head_dim": 64, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 8192, + "max_position_embeddings": 131072, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 32, + "num_hidden_layers": 16, + "num_key_value_heads": 8, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": { + "factor": 32.0, + "high_freq_factor": 4.0, + "low_freq_factor": 1.0, + "original_max_position_embeddings": 8192, + "rope_type": "llama3" + }, + "rope_theta": 500000.0, + "tie_word_embeddings": true, + "transformers_version": "4.57.1", + "use_cache": false, + "vocab_size": 128256 +} diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2460/generation_config.json b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2460/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..125cc74759bc705ab8a0c48d07d7eac083f56011 --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2460/generation_config.json @@ -0,0 +1,12 @@ +{ + "bos_token_id": 128000, + "do_sample": true, + "eos_token_id": [ + 128001, + 128008, + 128009 + ], + "temperature": 0.6, + "top_p": 0.9, + "transformers_version": "4.57.1" +} diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2460/latest b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2460/latest new file mode 100644 index 0000000000000000000000000000000000000000..a3f30ad831df46fc035d55dc59a2f8c49978b853 --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2460/latest @@ -0,0 +1 @@ +global_step2460 \ No newline at end of file diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2460/model.safetensors b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2460/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..41daf238bd9765d419613c4fdb3fd79c60ae1136 --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2460/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c3d6acd640009c99ce7a992fb61d38bd490f5ad256780b5ccdaecbb3080ccb05 +size 2996982344 diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2460/scheduler.pt b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2460/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..fb290b749cc4f3c4f73f633f4f7755c4cbeb953b --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2460/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fed0e655965f858d137ea2fee905c1a311c243ad3a6a0a61d1c852e0233dd8c4 +size 1465 diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2460/trainer_state.json b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2460/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..c022f00db87845584da47f364985b566b058295f --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2460/trainer_state.json @@ -0,0 +1,1756 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 2.9969530773918343, + "eval_steps": 500, + "global_step": 2460, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.01218769043266301, + "grad_norm": 0.36486390233039856, + "learning_rate": 5e-05, + "loss": 1.2213, + "step": 10 + }, + { + "epoch": 0.02437538086532602, + "grad_norm": 0.2953564524650574, + "learning_rate": 5e-05, + "loss": 1.0223, + "step": 20 + }, + { + "epoch": 0.03656307129798903, + "grad_norm": 0.2766735851764679, + "learning_rate": 5e-05, + "loss": 0.9589, + "step": 30 + }, + { + "epoch": 0.04875076173065204, + "grad_norm": 0.32770997285842896, + "learning_rate": 5e-05, + "loss": 0.9081, + "step": 40 + }, + { + "epoch": 0.06093845216331505, + "grad_norm": 0.3052276372909546, + "learning_rate": 5e-05, + "loss": 0.8752, + "step": 50 + }, + { + "epoch": 0.07312614259597806, + "grad_norm": 0.240146666765213, + "learning_rate": 5e-05, + "loss": 0.873, + "step": 60 + }, + { + "epoch": 0.08531383302864107, + "grad_norm": 0.23872187733650208, + "learning_rate": 5e-05, + "loss": 0.8587, + "step": 70 + }, + { + "epoch": 0.09750152346130408, + "grad_norm": 0.32638487219810486, + "learning_rate": 5e-05, + "loss": 0.8656, + "step": 80 + }, + { + "epoch": 0.10968921389396709, + "grad_norm": 0.25019028782844543, + "learning_rate": 5e-05, + "loss": 0.8685, + "step": 90 + }, + { + "epoch": 0.1218769043266301, + "grad_norm": 0.2507675886154175, + "learning_rate": 5e-05, + "loss": 0.8662, + "step": 100 + }, + { + "epoch": 0.1340645947592931, + "grad_norm": 0.27966755628585815, + "learning_rate": 5e-05, + "loss": 0.8428, + "step": 110 + }, + { + "epoch": 0.14625228519195613, + "grad_norm": 0.2622673213481903, + "learning_rate": 5e-05, + "loss": 0.8416, + "step": 120 + }, + { + "epoch": 0.15843997562461914, + "grad_norm": 0.3003966510295868, + "learning_rate": 5e-05, + "loss": 0.8528, + "step": 130 + }, + { + "epoch": 0.17062766605728213, + "grad_norm": 0.32308319211006165, + "learning_rate": 5e-05, + "loss": 0.8357, + "step": 140 + }, + { + "epoch": 0.18281535648994515, + "grad_norm": 0.27279096841812134, + "learning_rate": 5e-05, + "loss": 0.8432, + "step": 150 + }, + { + "epoch": 0.19500304692260817, + "grad_norm": 0.27976104617118835, + "learning_rate": 5e-05, + "loss": 0.8119, + "step": 160 + }, + { + "epoch": 0.2071907373552712, + "grad_norm": 0.290866881608963, + "learning_rate": 5e-05, + "loss": 0.8096, + "step": 170 + }, + { + "epoch": 0.21937842778793418, + "grad_norm": 0.29430949687957764, + "learning_rate": 5e-05, + "loss": 0.819, + "step": 180 + }, + { + "epoch": 0.2315661182205972, + "grad_norm": 0.27714136242866516, + "learning_rate": 5e-05, + "loss": 0.8187, + "step": 190 + }, + { + "epoch": 0.2437538086532602, + "grad_norm": 0.2997029423713684, + "learning_rate": 5e-05, + "loss": 0.808, + "step": 200 + }, + { + "epoch": 0.25594149908592323, + "grad_norm": 0.2539573609828949, + "learning_rate": 5e-05, + "loss": 0.7943, + "step": 210 + }, + { + "epoch": 0.2681291895185862, + "grad_norm": 0.23294171690940857, + "learning_rate": 5e-05, + "loss": 0.8188, + "step": 220 + }, + { + "epoch": 0.28031687995124926, + "grad_norm": 0.25016292929649353, + "learning_rate": 5e-05, + "loss": 0.8035, + "step": 230 + }, + { + "epoch": 0.29250457038391225, + "grad_norm": 0.37125322222709656, + "learning_rate": 5e-05, + "loss": 0.785, + "step": 240 + }, + { + "epoch": 0.30469226081657524, + "grad_norm": 0.21520452201366425, + "learning_rate": 5e-05, + "loss": 0.8088, + "step": 250 + }, + { + "epoch": 0.3168799512492383, + "grad_norm": 0.20990602672100067, + "learning_rate": 5e-05, + "loss": 0.7813, + "step": 260 + }, + { + "epoch": 0.3290676416819013, + "grad_norm": 0.24193961918354034, + "learning_rate": 5e-05, + "loss": 0.7979, + "step": 270 + }, + { + "epoch": 0.34125533211456427, + "grad_norm": 0.22766050696372986, + "learning_rate": 5e-05, + "loss": 0.785, + "step": 280 + }, + { + "epoch": 0.3534430225472273, + "grad_norm": 0.2662702202796936, + "learning_rate": 5e-05, + "loss": 0.7766, + "step": 290 + }, + { + "epoch": 0.3656307129798903, + "grad_norm": 0.23020495474338531, + "learning_rate": 5e-05, + "loss": 0.7687, + "step": 300 + }, + { + "epoch": 0.3778184034125533, + "grad_norm": 0.21770520508289337, + "learning_rate": 5e-05, + "loss": 0.7815, + "step": 310 + }, + { + "epoch": 0.39000609384521634, + "grad_norm": 0.22677403688430786, + "learning_rate": 5e-05, + "loss": 0.7999, + "step": 320 + }, + { + "epoch": 0.40219378427787933, + "grad_norm": 0.24891860783100128, + "learning_rate": 5e-05, + "loss": 0.7946, + "step": 330 + }, + { + "epoch": 0.4143814747105424, + "grad_norm": 0.2506854832172394, + "learning_rate": 5e-05, + "loss": 0.7889, + "step": 340 + }, + { + "epoch": 0.42656916514320536, + "grad_norm": 0.24388180673122406, + "learning_rate": 5e-05, + "loss": 0.79, + "step": 350 + }, + { + "epoch": 0.43875685557586835, + "grad_norm": 0.22329650819301605, + "learning_rate": 5e-05, + "loss": 0.7632, + "step": 360 + }, + { + "epoch": 0.4509445460085314, + "grad_norm": 0.2240927815437317, + "learning_rate": 5e-05, + "loss": 0.8042, + "step": 370 + }, + { + "epoch": 0.4631322364411944, + "grad_norm": 0.2578203082084656, + "learning_rate": 5e-05, + "loss": 0.7667, + "step": 380 + }, + { + "epoch": 0.4753199268738574, + "grad_norm": 0.25349217653274536, + "learning_rate": 5e-05, + "loss": 0.7911, + "step": 390 + }, + { + "epoch": 0.4875076173065204, + "grad_norm": 0.254446804523468, + "learning_rate": 5e-05, + "loss": 0.7909, + "step": 400 + }, + { + "epoch": 0.4996953077391834, + "grad_norm": 0.24253171682357788, + "learning_rate": 5e-05, + "loss": 0.7896, + "step": 410 + }, + { + "epoch": 0.5118829981718465, + "grad_norm": 0.27901458740234375, + "learning_rate": 5e-05, + "loss": 0.7828, + "step": 420 + }, + { + "epoch": 0.5240706886045094, + "grad_norm": 0.31884434819221497, + "learning_rate": 5e-05, + "loss": 0.7919, + "step": 430 + }, + { + "epoch": 0.5362583790371724, + "grad_norm": 0.22920799255371094, + "learning_rate": 5e-05, + "loss": 0.7777, + "step": 440 + }, + { + "epoch": 0.5484460694698354, + "grad_norm": 0.22606618702411652, + "learning_rate": 5e-05, + "loss": 0.7661, + "step": 450 + }, + { + "epoch": 0.5606337599024985, + "grad_norm": 0.21604052186012268, + "learning_rate": 5e-05, + "loss": 0.769, + "step": 460 + }, + { + "epoch": 0.5728214503351615, + "grad_norm": 0.2296101152896881, + "learning_rate": 5e-05, + "loss": 0.7626, + "step": 470 + }, + { + "epoch": 0.5850091407678245, + "grad_norm": 0.24162150919437408, + "learning_rate": 5e-05, + "loss": 0.7831, + "step": 480 + }, + { + "epoch": 0.5971968312004875, + "grad_norm": 0.22364799678325653, + "learning_rate": 5e-05, + "loss": 0.7737, + "step": 490 + }, + { + "epoch": 0.6093845216331505, + "grad_norm": 0.23953403532505035, + "learning_rate": 5e-05, + "loss": 0.787, + "step": 500 + }, + { + "epoch": 0.6215722120658135, + "grad_norm": 0.22349311411380768, + "learning_rate": 5e-05, + "loss": 0.7763, + "step": 510 + }, + { + "epoch": 0.6337599024984766, + "grad_norm": 0.21996377408504486, + "learning_rate": 5e-05, + "loss": 0.7596, + "step": 520 + }, + { + "epoch": 0.6459475929311396, + "grad_norm": 0.245538130402565, + "learning_rate": 5e-05, + "loss": 0.7785, + "step": 530 + }, + { + "epoch": 0.6581352833638026, + "grad_norm": 0.23442520201206207, + "learning_rate": 5e-05, + "loss": 0.7633, + "step": 540 + }, + { + "epoch": 0.6703229737964655, + "grad_norm": 0.22274106740951538, + "learning_rate": 5e-05, + "loss": 0.7756, + "step": 550 + }, + { + "epoch": 0.6825106642291285, + "grad_norm": 0.21717530488967896, + "learning_rate": 5e-05, + "loss": 0.7726, + "step": 560 + }, + { + "epoch": 0.6946983546617916, + "grad_norm": 0.2465955913066864, + "learning_rate": 5e-05, + "loss": 0.7476, + "step": 570 + }, + { + "epoch": 0.7068860450944546, + "grad_norm": 0.2318575084209442, + "learning_rate": 5e-05, + "loss": 0.7584, + "step": 580 + }, + { + "epoch": 0.7190737355271176, + "grad_norm": 0.25114378333091736, + "learning_rate": 5e-05, + "loss": 0.7625, + "step": 590 + }, + { + "epoch": 0.7312614259597806, + "grad_norm": 0.19145755469799042, + "learning_rate": 5e-05, + "loss": 0.7577, + "step": 600 + }, + { + "epoch": 0.7434491163924436, + "grad_norm": 0.19129066169261932, + "learning_rate": 5e-05, + "loss": 0.7772, + "step": 610 + }, + { + "epoch": 0.7556368068251066, + "grad_norm": 0.2014475166797638, + "learning_rate": 5e-05, + "loss": 0.7601, + "step": 620 + }, + { + "epoch": 0.7678244972577697, + "grad_norm": 0.21480509638786316, + "learning_rate": 5e-05, + "loss": 0.7665, + "step": 630 + }, + { + "epoch": 0.7800121876904327, + "grad_norm": 0.2633228302001953, + "learning_rate": 5e-05, + "loss": 0.7565, + "step": 640 + }, + { + "epoch": 0.7921998781230957, + "grad_norm": 0.24633203446865082, + "learning_rate": 5e-05, + "loss": 0.7602, + "step": 650 + }, + { + "epoch": 0.8043875685557587, + "grad_norm": 0.19770534336566925, + "learning_rate": 5e-05, + "loss": 0.7685, + "step": 660 + }, + { + "epoch": 0.8165752589884216, + "grad_norm": 0.2267930805683136, + "learning_rate": 5e-05, + "loss": 0.7516, + "step": 670 + }, + { + "epoch": 0.8287629494210847, + "grad_norm": 0.238111674785614, + "learning_rate": 5e-05, + "loss": 0.7487, + "step": 680 + }, + { + "epoch": 0.8409506398537477, + "grad_norm": 0.19528904557228088, + "learning_rate": 5e-05, + "loss": 0.7467, + "step": 690 + }, + { + "epoch": 0.8531383302864107, + "grad_norm": 0.21305914223194122, + "learning_rate": 5e-05, + "loss": 0.7527, + "step": 700 + }, + { + "epoch": 0.8653260207190737, + "grad_norm": 0.2205205112695694, + "learning_rate": 5e-05, + "loss": 0.7639, + "step": 710 + }, + { + "epoch": 0.8775137111517367, + "grad_norm": 0.22429127991199493, + "learning_rate": 5e-05, + "loss": 0.7744, + "step": 720 + }, + { + "epoch": 0.8897014015843998, + "grad_norm": 0.23755772411823273, + "learning_rate": 5e-05, + "loss": 0.7446, + "step": 730 + }, + { + "epoch": 0.9018890920170628, + "grad_norm": 0.17592693865299225, + "learning_rate": 5e-05, + "loss": 0.7442, + "step": 740 + }, + { + "epoch": 0.9140767824497258, + "grad_norm": 0.2266898900270462, + "learning_rate": 5e-05, + "loss": 0.7326, + "step": 750 + }, + { + "epoch": 0.9262644728823888, + "grad_norm": 0.23541569709777832, + "learning_rate": 5e-05, + "loss": 0.7338, + "step": 760 + }, + { + "epoch": 0.9384521633150518, + "grad_norm": 0.22233888506889343, + "learning_rate": 5e-05, + "loss": 0.7385, + "step": 770 + }, + { + "epoch": 0.9506398537477148, + "grad_norm": 0.23714767396450043, + "learning_rate": 5e-05, + "loss": 0.7306, + "step": 780 + }, + { + "epoch": 0.9628275441803779, + "grad_norm": 0.21191060543060303, + "learning_rate": 5e-05, + "loss": 0.736, + "step": 790 + }, + { + "epoch": 0.9750152346130408, + "grad_norm": 0.22759106755256653, + "learning_rate": 5e-05, + "loss": 0.7505, + "step": 800 + }, + { + "epoch": 0.9872029250457038, + "grad_norm": 0.19895903766155243, + "learning_rate": 5e-05, + "loss": 0.7536, + "step": 810 + }, + { + "epoch": 0.9993906154783668, + "grad_norm": 0.3061685860157013, + "learning_rate": 5e-05, + "loss": 0.7493, + "step": 820 + }, + { + "epoch": 1.0109689213893966, + "grad_norm": 0.18868015706539154, + "learning_rate": 5e-05, + "loss": 0.7342, + "step": 830 + }, + { + "epoch": 1.0231566118220596, + "grad_norm": 0.1797628402709961, + "learning_rate": 5e-05, + "loss": 0.7192, + "step": 840 + }, + { + "epoch": 1.0353443022547226, + "grad_norm": 0.19511596858501434, + "learning_rate": 5e-05, + "loss": 0.7002, + "step": 850 + }, + { + "epoch": 1.0475319926873858, + "grad_norm": 0.2123185396194458, + "learning_rate": 5e-05, + "loss": 0.7236, + "step": 860 + }, + { + "epoch": 1.0597196831200488, + "grad_norm": 0.2047470211982727, + "learning_rate": 5e-05, + "loss": 0.711, + "step": 870 + }, + { + "epoch": 1.0719073735527118, + "grad_norm": 0.18809868395328522, + "learning_rate": 5e-05, + "loss": 0.7253, + "step": 880 + }, + { + "epoch": 1.0840950639853748, + "grad_norm": 0.2207012176513672, + "learning_rate": 5e-05, + "loss": 0.6971, + "step": 890 + }, + { + "epoch": 1.0962827544180378, + "grad_norm": 0.1912035495042801, + "learning_rate": 5e-05, + "loss": 0.7239, + "step": 900 + }, + { + "epoch": 1.1084704448507008, + "grad_norm": 0.23486360907554626, + "learning_rate": 5e-05, + "loss": 0.7335, + "step": 910 + }, + { + "epoch": 1.1206581352833638, + "grad_norm": 0.22189787030220032, + "learning_rate": 5e-05, + "loss": 0.7149, + "step": 920 + }, + { + "epoch": 1.1328458257160268, + "grad_norm": 0.25818824768066406, + "learning_rate": 5e-05, + "loss": 0.7322, + "step": 930 + }, + { + "epoch": 1.1450335161486898, + "grad_norm": 0.2088775783777237, + "learning_rate": 5e-05, + "loss": 0.7382, + "step": 940 + }, + { + "epoch": 1.1572212065813527, + "grad_norm": 0.19445833563804626, + "learning_rate": 5e-05, + "loss": 0.7223, + "step": 950 + }, + { + "epoch": 1.1694088970140157, + "grad_norm": 0.21489456295967102, + "learning_rate": 5e-05, + "loss": 0.7317, + "step": 960 + }, + { + "epoch": 1.181596587446679, + "grad_norm": 0.24050794541835785, + "learning_rate": 5e-05, + "loss": 0.7275, + "step": 970 + }, + { + "epoch": 1.193784277879342, + "grad_norm": 0.18142102658748627, + "learning_rate": 5e-05, + "loss": 0.7046, + "step": 980 + }, + { + "epoch": 1.205971968312005, + "grad_norm": 0.4781077802181244, + "learning_rate": 5e-05, + "loss": 0.7433, + "step": 990 + }, + { + "epoch": 1.218159658744668, + "grad_norm": 0.20628023147583008, + "learning_rate": 5e-05, + "loss": 0.7403, + "step": 1000 + }, + { + "epoch": 1.230347349177331, + "grad_norm": 0.19117140769958496, + "learning_rate": 5e-05, + "loss": 0.7308, + "step": 1010 + }, + { + "epoch": 1.242535039609994, + "grad_norm": 0.2107745110988617, + "learning_rate": 5e-05, + "loss": 0.7201, + "step": 1020 + }, + { + "epoch": 1.2547227300426569, + "grad_norm": 0.20551154017448425, + "learning_rate": 5e-05, + "loss": 0.7038, + "step": 1030 + }, + { + "epoch": 1.2669104204753199, + "grad_norm": 0.26577022671699524, + "learning_rate": 5e-05, + "loss": 0.7096, + "step": 1040 + }, + { + "epoch": 1.2790981109079829, + "grad_norm": 0.22094209492206573, + "learning_rate": 5e-05, + "loss": 0.718, + "step": 1050 + }, + { + "epoch": 1.291285801340646, + "grad_norm": 0.18224076926708221, + "learning_rate": 5e-05, + "loss": 0.7261, + "step": 1060 + }, + { + "epoch": 1.303473491773309, + "grad_norm": 0.226251482963562, + "learning_rate": 5e-05, + "loss": 0.7372, + "step": 1070 + }, + { + "epoch": 1.315661182205972, + "grad_norm": 0.20186461508274078, + "learning_rate": 5e-05, + "loss": 0.7074, + "step": 1080 + }, + { + "epoch": 1.327848872638635, + "grad_norm": 0.22229762375354767, + "learning_rate": 5e-05, + "loss": 0.7069, + "step": 1090 + }, + { + "epoch": 1.340036563071298, + "grad_norm": 0.1776452213525772, + "learning_rate": 5e-05, + "loss": 0.714, + "step": 1100 + }, + { + "epoch": 1.352224253503961, + "grad_norm": 0.20680229365825653, + "learning_rate": 5e-05, + "loss": 0.7066, + "step": 1110 + }, + { + "epoch": 1.364411943936624, + "grad_norm": 0.21842031180858612, + "learning_rate": 5e-05, + "loss": 0.7281, + "step": 1120 + }, + { + "epoch": 1.376599634369287, + "grad_norm": 0.28037774562835693, + "learning_rate": 5e-05, + "loss": 0.7149, + "step": 1130 + }, + { + "epoch": 1.38878732480195, + "grad_norm": 0.20962920784950256, + "learning_rate": 5e-05, + "loss": 0.6951, + "step": 1140 + }, + { + "epoch": 1.400975015234613, + "grad_norm": 0.18751871585845947, + "learning_rate": 5e-05, + "loss": 0.7101, + "step": 1150 + }, + { + "epoch": 1.413162705667276, + "grad_norm": 0.2722964584827423, + "learning_rate": 5e-05, + "loss": 0.7205, + "step": 1160 + }, + { + "epoch": 1.425350396099939, + "grad_norm": 0.20932041108608246, + "learning_rate": 5e-05, + "loss": 0.7082, + "step": 1170 + }, + { + "epoch": 1.437538086532602, + "grad_norm": 0.17836257815361023, + "learning_rate": 5e-05, + "loss": 0.7232, + "step": 1180 + }, + { + "epoch": 1.449725776965265, + "grad_norm": 0.3180703818798065, + "learning_rate": 5e-05, + "loss": 0.7245, + "step": 1190 + }, + { + "epoch": 1.4619134673979282, + "grad_norm": 0.201424703001976, + "learning_rate": 5e-05, + "loss": 0.7232, + "step": 1200 + }, + { + "epoch": 1.4741011578305911, + "grad_norm": 0.20228999853134155, + "learning_rate": 5e-05, + "loss": 0.7169, + "step": 1210 + }, + { + "epoch": 1.4862888482632541, + "grad_norm": 0.17478394508361816, + "learning_rate": 5e-05, + "loss": 0.704, + "step": 1220 + }, + { + "epoch": 1.4984765386959171, + "grad_norm": 0.20061351358890533, + "learning_rate": 5e-05, + "loss": 0.6792, + "step": 1230 + }, + { + "epoch": 1.5106642291285801, + "grad_norm": 0.1857377588748932, + "learning_rate": 5e-05, + "loss": 0.709, + "step": 1240 + }, + { + "epoch": 1.522851919561243, + "grad_norm": 0.19873379170894623, + "learning_rate": 5e-05, + "loss": 0.7007, + "step": 1250 + }, + { + "epoch": 1.5350396099939063, + "grad_norm": 0.19807074964046478, + "learning_rate": 5e-05, + "loss": 0.7122, + "step": 1260 + }, + { + "epoch": 1.5472273004265693, + "grad_norm": 0.16746796667575836, + "learning_rate": 5e-05, + "loss": 0.7005, + "step": 1270 + }, + { + "epoch": 1.5594149908592323, + "grad_norm": 0.2750326991081238, + "learning_rate": 5e-05, + "loss": 0.7166, + "step": 1280 + }, + { + "epoch": 1.5716026812918953, + "grad_norm": 0.2776738107204437, + "learning_rate": 5e-05, + "loss": 0.7405, + "step": 1290 + }, + { + "epoch": 1.5837903717245583, + "grad_norm": 0.3494517207145691, + "learning_rate": 5e-05, + "loss": 0.7192, + "step": 1300 + }, + { + "epoch": 1.5959780621572213, + "grad_norm": 0.25028494000434875, + "learning_rate": 5e-05, + "loss": 0.741, + "step": 1310 + }, + { + "epoch": 1.6081657525898843, + "grad_norm": 0.23914143443107605, + "learning_rate": 5e-05, + "loss": 0.72, + "step": 1320 + }, + { + "epoch": 1.6203534430225472, + "grad_norm": 0.2052862048149109, + "learning_rate": 5e-05, + "loss": 0.7112, + "step": 1330 + }, + { + "epoch": 1.6325411334552102, + "grad_norm": 0.22285936772823334, + "learning_rate": 5e-05, + "loss": 0.7165, + "step": 1340 + }, + { + "epoch": 1.6447288238878732, + "grad_norm": 0.20810848474502563, + "learning_rate": 5e-05, + "loss": 0.7122, + "step": 1350 + }, + { + "epoch": 1.6569165143205362, + "grad_norm": 0.20422884821891785, + "learning_rate": 5e-05, + "loss": 0.7082, + "step": 1360 + }, + { + "epoch": 1.6691042047531992, + "grad_norm": 0.18164314329624176, + "learning_rate": 5e-05, + "loss": 0.7131, + "step": 1370 + }, + { + "epoch": 1.6812918951858622, + "grad_norm": 0.19893014430999756, + "learning_rate": 5e-05, + "loss": 0.7055, + "step": 1380 + }, + { + "epoch": 1.6934795856185252, + "grad_norm": 0.24865971505641937, + "learning_rate": 5e-05, + "loss": 0.6994, + "step": 1390 + }, + { + "epoch": 1.7056672760511882, + "grad_norm": 0.18078123033046722, + "learning_rate": 5e-05, + "loss": 0.7162, + "step": 1400 + }, + { + "epoch": 1.7178549664838512, + "grad_norm": 0.1973421722650528, + "learning_rate": 5e-05, + "loss": 0.7136, + "step": 1410 + }, + { + "epoch": 1.7300426569165142, + "grad_norm": 0.2057911604642868, + "learning_rate": 5e-05, + "loss": 0.7004, + "step": 1420 + }, + { + "epoch": 1.7422303473491774, + "grad_norm": 0.18640495836734772, + "learning_rate": 5e-05, + "loss": 0.7102, + "step": 1430 + }, + { + "epoch": 1.7544180377818404, + "grad_norm": 0.21038417518138885, + "learning_rate": 5e-05, + "loss": 0.6908, + "step": 1440 + }, + { + "epoch": 1.7666057282145033, + "grad_norm": 0.2123890221118927, + "learning_rate": 5e-05, + "loss": 0.6913, + "step": 1450 + }, + { + "epoch": 1.7787934186471663, + "grad_norm": 0.21619991958141327, + "learning_rate": 5e-05, + "loss": 0.7038, + "step": 1460 + }, + { + "epoch": 1.7909811090798293, + "grad_norm": 0.1975220888853073, + "learning_rate": 5e-05, + "loss": 0.7041, + "step": 1470 + }, + { + "epoch": 1.8031687995124925, + "grad_norm": 0.20595207810401917, + "learning_rate": 5e-05, + "loss": 0.7145, + "step": 1480 + }, + { + "epoch": 1.8153564899451555, + "grad_norm": 0.16633334755897522, + "learning_rate": 5e-05, + "loss": 0.6963, + "step": 1490 + }, + { + "epoch": 1.8275441803778185, + "grad_norm": 0.19140620529651642, + "learning_rate": 5e-05, + "loss": 0.6814, + "step": 1500 + }, + { + "epoch": 1.8397318708104815, + "grad_norm": 0.1833905130624771, + "learning_rate": 5e-05, + "loss": 0.7174, + "step": 1510 + }, + { + "epoch": 1.8519195612431445, + "grad_norm": 0.20587337017059326, + "learning_rate": 5e-05, + "loss": 0.7098, + "step": 1520 + }, + { + "epoch": 1.8641072516758075, + "grad_norm": 0.21952909231185913, + "learning_rate": 5e-05, + "loss": 0.6912, + "step": 1530 + }, + { + "epoch": 1.8762949421084705, + "grad_norm": 0.23233620822429657, + "learning_rate": 5e-05, + "loss": 0.6992, + "step": 1540 + }, + { + "epoch": 1.8884826325411335, + "grad_norm": 0.17159809172153473, + "learning_rate": 5e-05, + "loss": 0.7052, + "step": 1550 + }, + { + "epoch": 1.9006703229737965, + "grad_norm": 0.23522134125232697, + "learning_rate": 5e-05, + "loss": 0.6873, + "step": 1560 + }, + { + "epoch": 1.9128580134064594, + "grad_norm": 0.21959669888019562, + "learning_rate": 5e-05, + "loss": 0.6834, + "step": 1570 + }, + { + "epoch": 1.9250457038391224, + "grad_norm": 0.2110578864812851, + "learning_rate": 5e-05, + "loss": 0.7038, + "step": 1580 + }, + { + "epoch": 1.9372333942717854, + "grad_norm": 0.18674372136592865, + "learning_rate": 5e-05, + "loss": 0.6743, + "step": 1590 + }, + { + "epoch": 1.9494210847044484, + "grad_norm": 0.18220046162605286, + "learning_rate": 5e-05, + "loss": 0.6943, + "step": 1600 + }, + { + "epoch": 1.9616087751371114, + "grad_norm": 0.20756569504737854, + "learning_rate": 5e-05, + "loss": 0.7049, + "step": 1610 + }, + { + "epoch": 1.9737964655697744, + "grad_norm": 0.1925211250782013, + "learning_rate": 5e-05, + "loss": 0.7033, + "step": 1620 + }, + { + "epoch": 1.9859841560024374, + "grad_norm": 0.21374447643756866, + "learning_rate": 5e-05, + "loss": 0.7025, + "step": 1630 + }, + { + "epoch": 1.9981718464351006, + "grad_norm": 0.1952880322933197, + "learning_rate": 5e-05, + "loss": 0.6877, + "step": 1640 + }, + { + "epoch": 2.0097501523461303, + "grad_norm": 0.20457723736763, + "learning_rate": 5e-05, + "loss": 0.6734, + "step": 1650 + }, + { + "epoch": 2.0219378427787933, + "grad_norm": 0.20965932309627533, + "learning_rate": 5e-05, + "loss": 0.6472, + "step": 1660 + }, + { + "epoch": 2.0341255332114563, + "grad_norm": 0.19478890299797058, + "learning_rate": 5e-05, + "loss": 0.6781, + "step": 1670 + }, + { + "epoch": 2.0463132236441193, + "grad_norm": 0.21520769596099854, + "learning_rate": 5e-05, + "loss": 0.6592, + "step": 1680 + }, + { + "epoch": 2.0585009140767823, + "grad_norm": 0.20154155790805817, + "learning_rate": 5e-05, + "loss": 0.6744, + "step": 1690 + }, + { + "epoch": 2.0706886045094453, + "grad_norm": 0.1899706870317459, + "learning_rate": 5e-05, + "loss": 0.6592, + "step": 1700 + }, + { + "epoch": 2.0828762949421087, + "grad_norm": 0.20379123091697693, + "learning_rate": 5e-05, + "loss": 0.6688, + "step": 1710 + }, + { + "epoch": 2.0950639853747717, + "grad_norm": 0.19799865782260895, + "learning_rate": 5e-05, + "loss": 0.6678, + "step": 1720 + }, + { + "epoch": 2.1072516758074347, + "grad_norm": 0.19186025857925415, + "learning_rate": 5e-05, + "loss": 0.6587, + "step": 1730 + }, + { + "epoch": 2.1194393662400977, + "grad_norm": 0.18936173617839813, + "learning_rate": 5e-05, + "loss": 0.668, + "step": 1740 + }, + { + "epoch": 2.1316270566727606, + "grad_norm": 0.19159527122974396, + "learning_rate": 5e-05, + "loss": 0.6643, + "step": 1750 + }, + { + "epoch": 2.1438147471054236, + "grad_norm": 0.20596368610858917, + "learning_rate": 5e-05, + "loss": 0.6667, + "step": 1760 + }, + { + "epoch": 2.1560024375380866, + "grad_norm": 0.18601712584495544, + "learning_rate": 5e-05, + "loss": 0.6598, + "step": 1770 + }, + { + "epoch": 2.1681901279707496, + "grad_norm": 0.2130655199289322, + "learning_rate": 5e-05, + "loss": 0.6844, + "step": 1780 + }, + { + "epoch": 2.1803778184034126, + "grad_norm": 0.20477338135242462, + "learning_rate": 5e-05, + "loss": 0.666, + "step": 1790 + }, + { + "epoch": 2.1925655088360756, + "grad_norm": 0.18484896421432495, + "learning_rate": 5e-05, + "loss": 0.6659, + "step": 1800 + }, + { + "epoch": 2.2047531992687386, + "grad_norm": 0.17023979127407074, + "learning_rate": 5e-05, + "loss": 0.6721, + "step": 1810 + }, + { + "epoch": 2.2169408897014016, + "grad_norm": 0.23462794721126556, + "learning_rate": 5e-05, + "loss": 0.6699, + "step": 1820 + }, + { + "epoch": 2.2291285801340646, + "grad_norm": 0.16521663963794708, + "learning_rate": 5e-05, + "loss": 0.6584, + "step": 1830 + }, + { + "epoch": 2.2413162705667276, + "grad_norm": 0.19637759029865265, + "learning_rate": 5e-05, + "loss": 0.6621, + "step": 1840 + }, + { + "epoch": 2.2535039609993905, + "grad_norm": 0.2099665403366089, + "learning_rate": 5e-05, + "loss": 0.6912, + "step": 1850 + }, + { + "epoch": 2.2656916514320535, + "grad_norm": 0.21308118104934692, + "learning_rate": 5e-05, + "loss": 0.6705, + "step": 1860 + }, + { + "epoch": 2.2778793418647165, + "grad_norm": 0.20638489723205566, + "learning_rate": 5e-05, + "loss": 0.6736, + "step": 1870 + }, + { + "epoch": 2.2900670322973795, + "grad_norm": 0.23812630772590637, + "learning_rate": 5e-05, + "loss": 0.6869, + "step": 1880 + }, + { + "epoch": 2.3022547227300425, + "grad_norm": 0.23223577439785004, + "learning_rate": 5e-05, + "loss": 0.6663, + "step": 1890 + }, + { + "epoch": 2.3144424131627055, + "grad_norm": 0.20844320952892303, + "learning_rate": 5e-05, + "loss": 0.6686, + "step": 1900 + }, + { + "epoch": 2.3266301035953685, + "grad_norm": 0.2517656683921814, + "learning_rate": 5e-05, + "loss": 0.658, + "step": 1910 + }, + { + "epoch": 2.3388177940280315, + "grad_norm": 0.20005013048648834, + "learning_rate": 5e-05, + "loss": 0.6659, + "step": 1920 + }, + { + "epoch": 2.3510054844606945, + "grad_norm": 0.20756477117538452, + "learning_rate": 5e-05, + "loss": 0.6613, + "step": 1930 + }, + { + "epoch": 2.363193174893358, + "grad_norm": 0.19551488757133484, + "learning_rate": 5e-05, + "loss": 0.676, + "step": 1940 + }, + { + "epoch": 2.375380865326021, + "grad_norm": 0.18600064516067505, + "learning_rate": 5e-05, + "loss": 0.6826, + "step": 1950 + }, + { + "epoch": 2.387568555758684, + "grad_norm": 0.2445133775472641, + "learning_rate": 5e-05, + "loss": 0.6623, + "step": 1960 + }, + { + "epoch": 2.399756246191347, + "grad_norm": 0.2174660712480545, + "learning_rate": 5e-05, + "loss": 0.6697, + "step": 1970 + }, + { + "epoch": 2.41194393662401, + "grad_norm": 0.1853487342596054, + "learning_rate": 5e-05, + "loss": 0.6561, + "step": 1980 + }, + { + "epoch": 2.424131627056673, + "grad_norm": 0.18266689777374268, + "learning_rate": 5e-05, + "loss": 0.6697, + "step": 1990 + }, + { + "epoch": 2.436319317489336, + "grad_norm": 0.16344764828681946, + "learning_rate": 5e-05, + "loss": 0.6827, + "step": 2000 + }, + { + "epoch": 2.448507007921999, + "grad_norm": 0.18712133169174194, + "learning_rate": 5e-05, + "loss": 0.6713, + "step": 2010 + }, + { + "epoch": 2.460694698354662, + "grad_norm": 0.2192094475030899, + "learning_rate": 5e-05, + "loss": 0.6761, + "step": 2020 + }, + { + "epoch": 2.472882388787325, + "grad_norm": 0.18980957567691803, + "learning_rate": 5e-05, + "loss": 0.7001, + "step": 2030 + }, + { + "epoch": 2.485070079219988, + "grad_norm": 0.20758011937141418, + "learning_rate": 5e-05, + "loss": 0.6801, + "step": 2040 + }, + { + "epoch": 2.497257769652651, + "grad_norm": 0.166823148727417, + "learning_rate": 5e-05, + "loss": 0.6667, + "step": 2050 + }, + { + "epoch": 2.5094454600853138, + "grad_norm": 0.17631390690803528, + "learning_rate": 5e-05, + "loss": 0.66, + "step": 2060 + }, + { + "epoch": 2.5216331505179768, + "grad_norm": 0.19680799543857574, + "learning_rate": 5e-05, + "loss": 0.6513, + "step": 2070 + }, + { + "epoch": 2.5338208409506398, + "grad_norm": 0.202071875333786, + "learning_rate": 5e-05, + "loss": 0.6655, + "step": 2080 + }, + { + "epoch": 2.5460085313833027, + "grad_norm": 0.1721998006105423, + "learning_rate": 5e-05, + "loss": 0.6564, + "step": 2090 + }, + { + "epoch": 2.5581962218159657, + "grad_norm": 0.17605794966220856, + "learning_rate": 5e-05, + "loss": 0.6715, + "step": 2100 + }, + { + "epoch": 2.5703839122486287, + "grad_norm": 0.2009730488061905, + "learning_rate": 5e-05, + "loss": 0.6764, + "step": 2110 + }, + { + "epoch": 2.582571602681292, + "grad_norm": 0.21355752646923065, + "learning_rate": 5e-05, + "loss": 0.6696, + "step": 2120 + }, + { + "epoch": 2.594759293113955, + "grad_norm": 0.18136924505233765, + "learning_rate": 5e-05, + "loss": 0.6459, + "step": 2130 + }, + { + "epoch": 2.606946983546618, + "grad_norm": 0.1764170080423355, + "learning_rate": 5e-05, + "loss": 0.6565, + "step": 2140 + }, + { + "epoch": 2.619134673979281, + "grad_norm": 0.1911628693342209, + "learning_rate": 5e-05, + "loss": 0.6534, + "step": 2150 + }, + { + "epoch": 2.631322364411944, + "grad_norm": 0.18562854826450348, + "learning_rate": 5e-05, + "loss": 0.6505, + "step": 2160 + }, + { + "epoch": 2.643510054844607, + "grad_norm": 0.17010731995105743, + "learning_rate": 5e-05, + "loss": 0.6676, + "step": 2170 + }, + { + "epoch": 2.65569774527727, + "grad_norm": 0.1590568870306015, + "learning_rate": 5e-05, + "loss": 0.638, + "step": 2180 + }, + { + "epoch": 2.667885435709933, + "grad_norm": 0.17398956418037415, + "learning_rate": 5e-05, + "loss": 0.6684, + "step": 2190 + }, + { + "epoch": 2.680073126142596, + "grad_norm": 0.1765497624874115, + "learning_rate": 5e-05, + "loss": 0.6633, + "step": 2200 + }, + { + "epoch": 2.692260816575259, + "grad_norm": 0.19518159329891205, + "learning_rate": 5e-05, + "loss": 0.6742, + "step": 2210 + }, + { + "epoch": 2.704448507007922, + "grad_norm": 0.22384262084960938, + "learning_rate": 5e-05, + "loss": 0.6544, + "step": 2220 + }, + { + "epoch": 2.716636197440585, + "grad_norm": 0.1654004603624344, + "learning_rate": 5e-05, + "loss": 0.6572, + "step": 2230 + }, + { + "epoch": 2.728823887873248, + "grad_norm": 0.18328379094600677, + "learning_rate": 5e-05, + "loss": 0.6714, + "step": 2240 + }, + { + "epoch": 2.741011578305911, + "grad_norm": 0.26779523491859436, + "learning_rate": 5e-05, + "loss": 0.6624, + "step": 2250 + }, + { + "epoch": 2.753199268738574, + "grad_norm": 0.1870274394750595, + "learning_rate": 5e-05, + "loss": 0.6818, + "step": 2260 + }, + { + "epoch": 2.765386959171237, + "grad_norm": 0.21015630662441254, + "learning_rate": 5e-05, + "loss": 0.6758, + "step": 2270 + }, + { + "epoch": 2.7775746496039, + "grad_norm": 0.18667075037956238, + "learning_rate": 5e-05, + "loss": 0.6822, + "step": 2280 + }, + { + "epoch": 2.789762340036563, + "grad_norm": 0.22627151012420654, + "learning_rate": 5e-05, + "loss": 0.6776, + "step": 2290 + }, + { + "epoch": 2.801950030469226, + "grad_norm": 0.20017723739147186, + "learning_rate": 5e-05, + "loss": 0.6685, + "step": 2300 + }, + { + "epoch": 2.814137720901889, + "grad_norm": 0.1894821971654892, + "learning_rate": 5e-05, + "loss": 0.6515, + "step": 2310 + }, + { + "epoch": 2.826325411334552, + "grad_norm": 0.17785170674324036, + "learning_rate": 5e-05, + "loss": 0.676, + "step": 2320 + }, + { + "epoch": 2.838513101767215, + "grad_norm": 0.7713242769241333, + "learning_rate": 5e-05, + "loss": 0.6646, + "step": 2330 + }, + { + "epoch": 2.850700792199878, + "grad_norm": 0.19354282319545746, + "learning_rate": 5e-05, + "loss": 0.6781, + "step": 2340 + }, + { + "epoch": 2.862888482632541, + "grad_norm": 0.2110157161951065, + "learning_rate": 5e-05, + "loss": 0.6765, + "step": 2350 + }, + { + "epoch": 2.875076173065204, + "grad_norm": 0.2045946568250656, + "learning_rate": 5e-05, + "loss": 0.6855, + "step": 2360 + }, + { + "epoch": 2.887263863497867, + "grad_norm": 0.15799324214458466, + "learning_rate": 5e-05, + "loss": 0.6747, + "step": 2370 + }, + { + "epoch": 2.89945155393053, + "grad_norm": 0.17158573865890503, + "learning_rate": 5e-05, + "loss": 0.6735, + "step": 2380 + }, + { + "epoch": 2.9116392443631933, + "grad_norm": 0.16838699579238892, + "learning_rate": 5e-05, + "loss": 0.6717, + "step": 2390 + }, + { + "epoch": 2.9238269347958563, + "grad_norm": 0.23229676485061646, + "learning_rate": 5e-05, + "loss": 0.6609, + "step": 2400 + }, + { + "epoch": 2.9360146252285193, + "grad_norm": 0.1893588900566101, + "learning_rate": 5e-05, + "loss": 0.6676, + "step": 2410 + }, + { + "epoch": 2.9482023156611823, + "grad_norm": 0.17633271217346191, + "learning_rate": 5e-05, + "loss": 0.6556, + "step": 2420 + }, + { + "epoch": 2.9603900060938453, + "grad_norm": 0.18573276698589325, + "learning_rate": 5e-05, + "loss": 0.6624, + "step": 2430 + }, + { + "epoch": 2.9725776965265083, + "grad_norm": 0.16133904457092285, + "learning_rate": 5e-05, + "loss": 0.662, + "step": 2440 + }, + { + "epoch": 2.9847653869591713, + "grad_norm": 0.1573924720287323, + "learning_rate": 5e-05, + "loss": 0.6613, + "step": 2450 + }, + { + "epoch": 2.9969530773918343, + "grad_norm": 0.17543254792690277, + "learning_rate": 5e-05, + "loss": 0.6895, + "step": 2460 + } + ], + "logging_steps": 10, + "max_steps": 2463, + "num_input_tokens_seen": 0, + "num_train_epochs": 3, + "save_steps": 205, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.0090427970008121e+19, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2460/training_args.bin b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2460/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..23ae0ec8a4226be5457f61c137abab2b2c10a0cf --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2460/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56896ea21940e33d7e2934267dff2a5b4a4a80195a0f165245f6617a0f88b90b +size 7377 diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2460/zero_to_fp32.py b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2460/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..5995d6e6f04e43b989587aa9022a3aef0c66d694 --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2460/zero_to_fp32.py @@ -0,0 +1,760 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: +# python zero_to_fp32.py . output_dir/ +# or +# python zero_to_fp32.py . output_dir/ --safe_serialization + +import argparse +import torch +import glob +import math +import os +import re +import gc +import json +import numpy as np +from tqdm import tqdm +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device, weights_only=False) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + total_files = len(files) + state_dicts = [] + for f in tqdm(files, desc='Loading checkpoint shards'): + state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + model_files = get_model_state_files(ds_checkpoint_dir) + + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +class GatheredTensor: + """ + A pseudo tensor that collects partitioned weights. + It is more memory efficient when there are multiple groups. + """ + + def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape): + self.flat_groups = flat_groups + self.flat_groups_offset = flat_groups_offset + self.offset = offset + self.partitioned_numel = partitioned_numel + self.shape = shape + self.dtype = self.flat_groups[0][0].dtype + + def contiguous(self): + """ + Merge partitioned weights from flat_groups into a single tensor. + """ + end_idx = self.offset + self.partitioned_numel + world_size = len(self.flat_groups) + pad_flat_param_chunks = [] + + for rank_i in range(world_size): + # for each rank, we need to collect weights from related group/groups + flat_groups_at_rank_i = self.flat_groups[rank_i] + start_group_id = None + end_group_id = None + for group_id in range(len(self.flat_groups_offset)): + if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]: + start_group_id = group_id + if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]: + end_group_id = group_id + break + # collect weights from related group/groups + for group_id in range(start_group_id, end_group_id + 1): + flat_tensor = flat_groups_at_rank_i[group_id] + start_offset = self.offset - self.flat_groups_offset[group_id] + end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id] + pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset]) + + # collect weights from all ranks + pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0) + param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous() + return param + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size + + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]])) + for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'): + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # memory efficient tensor + tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape) + state_dict[name] = tensor + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def to_torch_tensor(state_dict, return_empty_tensor=False): + """ + Convert state_dict of GatheredTensor to torch tensor + """ + torch_state_dict = {} + converted_tensors = {} + for name, tensor in state_dict.items(): + tensor_id = id(tensor) + if tensor_id in converted_tensors: # shared tensors + shared_tensor = torch_state_dict[converted_tensors[tensor_id]] + torch_state_dict[name] = shared_tensor + else: + converted_tensors[tensor_id] = name + if return_empty_tensor: + torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype) + else: + torch_state_dict[name] = tensor.contiguous() + return torch_state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag=None, + exclude_frozen_parameters=False, + lazy_mode=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient. + Convert the pesduo tensor to torch tensor by ``.contiguous()`` + + Returns: + - pytorch ``state_dict`` + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + Note: the above usage may not work if your application doesn't have sufficient free CPU memory. + You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. Or you can load state_dict in lazy mode :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu + for name, lazy_tensor in state_dict.item(): + tensor = lazy_tensor.contiguous() # to cpu + print(name, tensor) + # del tensor to release memory if it no longer in use + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + if lazy_mode: + return state_dict + else: + return to_torch_tensor(state_dict) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, + output_dir, + max_shard_size="5GB", + safe_serialization=False, + tag=None, + exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_dir``: directory to the pytorch fp32 state_dict output files + - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB + - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + + # Dependency pre-check + if safe_serialization: + try: + from safetensors.torch import save_file + except ImportError: + print('If you want to use `safe_serialization`, please `pip install safetensors`') + raise + if max_shard_size is not None: + try: + from huggingface_hub import split_torch_state_dict_into_shards + except ImportError: + print('If you want to use `max_shard_size`, please `pip install huggingface_hub`') + raise + + # Convert zero checkpoint to state_dict + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag, + exclude_frozen_parameters, + lazy_mode=True) + + # Shard the model if it is too big. + weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin" + if max_shard_size is not None: + filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors") + # an memory-efficient approach for sharding + empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True) + state_dict_split = split_torch_state_dict_into_shards(empty_state_dict, + filename_pattern=filename_pattern, + max_shard_size=max_shard_size) + else: + from collections import namedtuple + StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"]) + state_dict_split = StateDictSplit(is_sharded=False, + filename_to_tensors={weights_name: list(state_dict.keys())}) + + # Save the model by shard + os.makedirs(output_dir, exist_ok=True) + filename_to_tensors = state_dict_split.filename_to_tensors.items() + for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"): + shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors} + shard_state_dict = to_torch_tensor(shard_state_dict) + output_path = os.path.join(output_dir, shard_file) + if safe_serialization: + save_file(shard_state_dict, output_path, metadata={"format": "pt"}) + else: + torch.save(shard_state_dict, output_path) + # release the memory of current shard + for tensor_name in list(shard_state_dict.keys()): + del state_dict[tensor_name] + del shard_state_dict[tensor_name] + del shard_state_dict + gc.collect() + + # Save index if sharded + if state_dict_split.is_sharded: + index = { + "metadata": state_dict_split.metadata, + "weight_map": state_dict_split.tensor_to_filename, + } + save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json" + save_index_file = os.path.join(output_dir, save_index_file) + with open(save_index_file, "w", encoding="utf-8") as f: + content = json.dumps(index, indent=2, sort_keys=True) + "\n" + f.write(content) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info("Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info("Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument("output_dir", + type=str, + help="directory to the pytorch fp32 state_dict output files" + "(e.g. path/checkpoint-12-output/)") + parser.add_argument( + "--max_shard_size", + type=str, + default="5GB", + help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size" + "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`" + "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances" + "without CPU OOM issues.") + parser.add_argument( + "--safe_serialization", + default=False, + action='store_true', + help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_dir, + max_shard_size=args.max_shard_size, + safe_serialization=args.safe_serialization, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2463/config.json b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2463/config.json new file mode 100644 index 0000000000000000000000000000000000000000..18bd58d561964a16b60687f1b04aefdfed0b9b20 --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2463/config.json @@ -0,0 +1,39 @@ +{ + "architectures": [ + "LlamaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "dtype": "bfloat16", + "eos_token_id": [ + 128001, + 128008, + 128009 + ], + "head_dim": 64, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 8192, + "max_position_embeddings": 131072, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 32, + "num_hidden_layers": 16, + "num_key_value_heads": 8, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": { + "factor": 32.0, + "high_freq_factor": 4.0, + "low_freq_factor": 1.0, + "original_max_position_embeddings": 8192, + "rope_type": "llama3" + }, + "rope_theta": 500000.0, + "tie_word_embeddings": true, + "transformers_version": "4.57.1", + "use_cache": false, + "vocab_size": 128256 +} diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2463/generation_config.json b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2463/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..125cc74759bc705ab8a0c48d07d7eac083f56011 --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2463/generation_config.json @@ -0,0 +1,12 @@ +{ + "bos_token_id": 128000, + "do_sample": true, + "eos_token_id": [ + 128001, + 128008, + 128009 + ], + "temperature": 0.6, + "top_p": 0.9, + "transformers_version": "4.57.1" +} diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2463/latest b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2463/latest new file mode 100644 index 0000000000000000000000000000000000000000..06f2bdb58bd6724af311ff8d34cc64f17876ebf8 --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2463/latest @@ -0,0 +1 @@ +global_step2463 \ No newline at end of file diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2463/model.safetensors b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2463/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..4e6227afb9e2a2e24c3d577689195c53b45025ac --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2463/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1a85202f3b9d7637d5ddf92afd78cc046781eec3941783c6b3ad6a63c4668714 +size 2996982344 diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2463/scheduler.pt b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2463/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..63daa5ee16d09d77c346c9970f8269e4f72d5b1e --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2463/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d9df5ed1f4bd891666d3868477e93a2af719b19da98ec36697788ac7276a2a6 +size 1465 diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2463/trainer_state.json b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2463/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..89b6e9bdf27dd22dc2aae0dfa71f01ebabb5a262 --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2463/trainer_state.json @@ -0,0 +1,1756 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 3.0, + "eval_steps": 500, + "global_step": 2463, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.01218769043266301, + "grad_norm": 0.36486390233039856, + "learning_rate": 5e-05, + "loss": 1.2213, + "step": 10 + }, + { + "epoch": 0.02437538086532602, + "grad_norm": 0.2953564524650574, + "learning_rate": 5e-05, + "loss": 1.0223, + "step": 20 + }, + { + "epoch": 0.03656307129798903, + "grad_norm": 0.2766735851764679, + "learning_rate": 5e-05, + "loss": 0.9589, + "step": 30 + }, + { + "epoch": 0.04875076173065204, + "grad_norm": 0.32770997285842896, + "learning_rate": 5e-05, + "loss": 0.9081, + "step": 40 + }, + { + "epoch": 0.06093845216331505, + "grad_norm": 0.3052276372909546, + "learning_rate": 5e-05, + "loss": 0.8752, + "step": 50 + }, + { + "epoch": 0.07312614259597806, + "grad_norm": 0.240146666765213, + "learning_rate": 5e-05, + "loss": 0.873, + "step": 60 + }, + { + "epoch": 0.08531383302864107, + "grad_norm": 0.23872187733650208, + "learning_rate": 5e-05, + "loss": 0.8587, + "step": 70 + }, + { + "epoch": 0.09750152346130408, + "grad_norm": 0.32638487219810486, + "learning_rate": 5e-05, + "loss": 0.8656, + "step": 80 + }, + { + "epoch": 0.10968921389396709, + "grad_norm": 0.25019028782844543, + "learning_rate": 5e-05, + "loss": 0.8685, + "step": 90 + }, + { + "epoch": 0.1218769043266301, + "grad_norm": 0.2507675886154175, + "learning_rate": 5e-05, + "loss": 0.8662, + "step": 100 + }, + { + "epoch": 0.1340645947592931, + "grad_norm": 0.27966755628585815, + "learning_rate": 5e-05, + "loss": 0.8428, + "step": 110 + }, + { + "epoch": 0.14625228519195613, + "grad_norm": 0.2622673213481903, + "learning_rate": 5e-05, + "loss": 0.8416, + "step": 120 + }, + { + "epoch": 0.15843997562461914, + "grad_norm": 0.3003966510295868, + "learning_rate": 5e-05, + "loss": 0.8528, + "step": 130 + }, + { + "epoch": 0.17062766605728213, + "grad_norm": 0.32308319211006165, + "learning_rate": 5e-05, + "loss": 0.8357, + "step": 140 + }, + { + "epoch": 0.18281535648994515, + "grad_norm": 0.27279096841812134, + "learning_rate": 5e-05, + "loss": 0.8432, + "step": 150 + }, + { + "epoch": 0.19500304692260817, + "grad_norm": 0.27976104617118835, + "learning_rate": 5e-05, + "loss": 0.8119, + "step": 160 + }, + { + "epoch": 0.2071907373552712, + "grad_norm": 0.290866881608963, + "learning_rate": 5e-05, + "loss": 0.8096, + "step": 170 + }, + { + "epoch": 0.21937842778793418, + "grad_norm": 0.29430949687957764, + "learning_rate": 5e-05, + "loss": 0.819, + "step": 180 + }, + { + "epoch": 0.2315661182205972, + "grad_norm": 0.27714136242866516, + "learning_rate": 5e-05, + "loss": 0.8187, + "step": 190 + }, + { + "epoch": 0.2437538086532602, + "grad_norm": 0.2997029423713684, + "learning_rate": 5e-05, + "loss": 0.808, + "step": 200 + }, + { + "epoch": 0.25594149908592323, + "grad_norm": 0.2539573609828949, + "learning_rate": 5e-05, + "loss": 0.7943, + "step": 210 + }, + { + "epoch": 0.2681291895185862, + "grad_norm": 0.23294171690940857, + "learning_rate": 5e-05, + "loss": 0.8188, + "step": 220 + }, + { + "epoch": 0.28031687995124926, + "grad_norm": 0.25016292929649353, + "learning_rate": 5e-05, + "loss": 0.8035, + "step": 230 + }, + { + "epoch": 0.29250457038391225, + "grad_norm": 0.37125322222709656, + "learning_rate": 5e-05, + "loss": 0.785, + "step": 240 + }, + { + "epoch": 0.30469226081657524, + "grad_norm": 0.21520452201366425, + "learning_rate": 5e-05, + "loss": 0.8088, + "step": 250 + }, + { + "epoch": 0.3168799512492383, + "grad_norm": 0.20990602672100067, + "learning_rate": 5e-05, + "loss": 0.7813, + "step": 260 + }, + { + "epoch": 0.3290676416819013, + "grad_norm": 0.24193961918354034, + "learning_rate": 5e-05, + "loss": 0.7979, + "step": 270 + }, + { + "epoch": 0.34125533211456427, + "grad_norm": 0.22766050696372986, + "learning_rate": 5e-05, + "loss": 0.785, + "step": 280 + }, + { + "epoch": 0.3534430225472273, + "grad_norm": 0.2662702202796936, + "learning_rate": 5e-05, + "loss": 0.7766, + "step": 290 + }, + { + "epoch": 0.3656307129798903, + "grad_norm": 0.23020495474338531, + "learning_rate": 5e-05, + "loss": 0.7687, + "step": 300 + }, + { + "epoch": 0.3778184034125533, + "grad_norm": 0.21770520508289337, + "learning_rate": 5e-05, + "loss": 0.7815, + "step": 310 + }, + { + "epoch": 0.39000609384521634, + "grad_norm": 0.22677403688430786, + "learning_rate": 5e-05, + "loss": 0.7999, + "step": 320 + }, + { + "epoch": 0.40219378427787933, + "grad_norm": 0.24891860783100128, + "learning_rate": 5e-05, + "loss": 0.7946, + "step": 330 + }, + { + "epoch": 0.4143814747105424, + "grad_norm": 0.2506854832172394, + "learning_rate": 5e-05, + "loss": 0.7889, + "step": 340 + }, + { + "epoch": 0.42656916514320536, + "grad_norm": 0.24388180673122406, + "learning_rate": 5e-05, + "loss": 0.79, + "step": 350 + }, + { + "epoch": 0.43875685557586835, + "grad_norm": 0.22329650819301605, + "learning_rate": 5e-05, + "loss": 0.7632, + "step": 360 + }, + { + "epoch": 0.4509445460085314, + "grad_norm": 0.2240927815437317, + "learning_rate": 5e-05, + "loss": 0.8042, + "step": 370 + }, + { + "epoch": 0.4631322364411944, + "grad_norm": 0.2578203082084656, + "learning_rate": 5e-05, + "loss": 0.7667, + "step": 380 + }, + { + "epoch": 0.4753199268738574, + "grad_norm": 0.25349217653274536, + "learning_rate": 5e-05, + "loss": 0.7911, + "step": 390 + }, + { + "epoch": 0.4875076173065204, + "grad_norm": 0.254446804523468, + "learning_rate": 5e-05, + "loss": 0.7909, + "step": 400 + }, + { + "epoch": 0.4996953077391834, + "grad_norm": 0.24253171682357788, + "learning_rate": 5e-05, + "loss": 0.7896, + "step": 410 + }, + { + "epoch": 0.5118829981718465, + "grad_norm": 0.27901458740234375, + "learning_rate": 5e-05, + "loss": 0.7828, + "step": 420 + }, + { + "epoch": 0.5240706886045094, + "grad_norm": 0.31884434819221497, + "learning_rate": 5e-05, + "loss": 0.7919, + "step": 430 + }, + { + "epoch": 0.5362583790371724, + "grad_norm": 0.22920799255371094, + "learning_rate": 5e-05, + "loss": 0.7777, + "step": 440 + }, + { + "epoch": 0.5484460694698354, + "grad_norm": 0.22606618702411652, + "learning_rate": 5e-05, + "loss": 0.7661, + "step": 450 + }, + { + "epoch": 0.5606337599024985, + "grad_norm": 0.21604052186012268, + "learning_rate": 5e-05, + "loss": 0.769, + "step": 460 + }, + { + "epoch": 0.5728214503351615, + "grad_norm": 0.2296101152896881, + "learning_rate": 5e-05, + "loss": 0.7626, + "step": 470 + }, + { + "epoch": 0.5850091407678245, + "grad_norm": 0.24162150919437408, + "learning_rate": 5e-05, + "loss": 0.7831, + "step": 480 + }, + { + "epoch": 0.5971968312004875, + "grad_norm": 0.22364799678325653, + "learning_rate": 5e-05, + "loss": 0.7737, + "step": 490 + }, + { + "epoch": 0.6093845216331505, + "grad_norm": 0.23953403532505035, + "learning_rate": 5e-05, + "loss": 0.787, + "step": 500 + }, + { + "epoch": 0.6215722120658135, + "grad_norm": 0.22349311411380768, + "learning_rate": 5e-05, + "loss": 0.7763, + "step": 510 + }, + { + "epoch": 0.6337599024984766, + "grad_norm": 0.21996377408504486, + "learning_rate": 5e-05, + "loss": 0.7596, + "step": 520 + }, + { + "epoch": 0.6459475929311396, + "grad_norm": 0.245538130402565, + "learning_rate": 5e-05, + "loss": 0.7785, + "step": 530 + }, + { + "epoch": 0.6581352833638026, + "grad_norm": 0.23442520201206207, + "learning_rate": 5e-05, + "loss": 0.7633, + "step": 540 + }, + { + "epoch": 0.6703229737964655, + "grad_norm": 0.22274106740951538, + "learning_rate": 5e-05, + "loss": 0.7756, + "step": 550 + }, + { + "epoch": 0.6825106642291285, + "grad_norm": 0.21717530488967896, + "learning_rate": 5e-05, + "loss": 0.7726, + "step": 560 + }, + { + "epoch": 0.6946983546617916, + "grad_norm": 0.2465955913066864, + "learning_rate": 5e-05, + "loss": 0.7476, + "step": 570 + }, + { + "epoch": 0.7068860450944546, + "grad_norm": 0.2318575084209442, + "learning_rate": 5e-05, + "loss": 0.7584, + "step": 580 + }, + { + "epoch": 0.7190737355271176, + "grad_norm": 0.25114378333091736, + "learning_rate": 5e-05, + "loss": 0.7625, + "step": 590 + }, + { + "epoch": 0.7312614259597806, + "grad_norm": 0.19145755469799042, + "learning_rate": 5e-05, + "loss": 0.7577, + "step": 600 + }, + { + "epoch": 0.7434491163924436, + "grad_norm": 0.19129066169261932, + "learning_rate": 5e-05, + "loss": 0.7772, + "step": 610 + }, + { + "epoch": 0.7556368068251066, + "grad_norm": 0.2014475166797638, + "learning_rate": 5e-05, + "loss": 0.7601, + "step": 620 + }, + { + "epoch": 0.7678244972577697, + "grad_norm": 0.21480509638786316, + "learning_rate": 5e-05, + "loss": 0.7665, + "step": 630 + }, + { + "epoch": 0.7800121876904327, + "grad_norm": 0.2633228302001953, + "learning_rate": 5e-05, + "loss": 0.7565, + "step": 640 + }, + { + "epoch": 0.7921998781230957, + "grad_norm": 0.24633203446865082, + "learning_rate": 5e-05, + "loss": 0.7602, + "step": 650 + }, + { + "epoch": 0.8043875685557587, + "grad_norm": 0.19770534336566925, + "learning_rate": 5e-05, + "loss": 0.7685, + "step": 660 + }, + { + "epoch": 0.8165752589884216, + "grad_norm": 0.2267930805683136, + "learning_rate": 5e-05, + "loss": 0.7516, + "step": 670 + }, + { + "epoch": 0.8287629494210847, + "grad_norm": 0.238111674785614, + "learning_rate": 5e-05, + "loss": 0.7487, + "step": 680 + }, + { + "epoch": 0.8409506398537477, + "grad_norm": 0.19528904557228088, + "learning_rate": 5e-05, + "loss": 0.7467, + "step": 690 + }, + { + "epoch": 0.8531383302864107, + "grad_norm": 0.21305914223194122, + "learning_rate": 5e-05, + "loss": 0.7527, + "step": 700 + }, + { + "epoch": 0.8653260207190737, + "grad_norm": 0.2205205112695694, + "learning_rate": 5e-05, + "loss": 0.7639, + "step": 710 + }, + { + "epoch": 0.8775137111517367, + "grad_norm": 0.22429127991199493, + "learning_rate": 5e-05, + "loss": 0.7744, + "step": 720 + }, + { + "epoch": 0.8897014015843998, + "grad_norm": 0.23755772411823273, + "learning_rate": 5e-05, + "loss": 0.7446, + "step": 730 + }, + { + "epoch": 0.9018890920170628, + "grad_norm": 0.17592693865299225, + "learning_rate": 5e-05, + "loss": 0.7442, + "step": 740 + }, + { + "epoch": 0.9140767824497258, + "grad_norm": 0.2266898900270462, + "learning_rate": 5e-05, + "loss": 0.7326, + "step": 750 + }, + { + "epoch": 0.9262644728823888, + "grad_norm": 0.23541569709777832, + "learning_rate": 5e-05, + "loss": 0.7338, + "step": 760 + }, + { + "epoch": 0.9384521633150518, + "grad_norm": 0.22233888506889343, + "learning_rate": 5e-05, + "loss": 0.7385, + "step": 770 + }, + { + "epoch": 0.9506398537477148, + "grad_norm": 0.23714767396450043, + "learning_rate": 5e-05, + "loss": 0.7306, + "step": 780 + }, + { + "epoch": 0.9628275441803779, + "grad_norm": 0.21191060543060303, + "learning_rate": 5e-05, + "loss": 0.736, + "step": 790 + }, + { + "epoch": 0.9750152346130408, + "grad_norm": 0.22759106755256653, + "learning_rate": 5e-05, + "loss": 0.7505, + "step": 800 + }, + { + "epoch": 0.9872029250457038, + "grad_norm": 0.19895903766155243, + "learning_rate": 5e-05, + "loss": 0.7536, + "step": 810 + }, + { + "epoch": 0.9993906154783668, + "grad_norm": 0.3061685860157013, + "learning_rate": 5e-05, + "loss": 0.7493, + "step": 820 + }, + { + "epoch": 1.0109689213893966, + "grad_norm": 0.18868015706539154, + "learning_rate": 5e-05, + "loss": 0.7342, + "step": 830 + }, + { + "epoch": 1.0231566118220596, + "grad_norm": 0.1797628402709961, + "learning_rate": 5e-05, + "loss": 0.7192, + "step": 840 + }, + { + "epoch": 1.0353443022547226, + "grad_norm": 0.19511596858501434, + "learning_rate": 5e-05, + "loss": 0.7002, + "step": 850 + }, + { + "epoch": 1.0475319926873858, + "grad_norm": 0.2123185396194458, + "learning_rate": 5e-05, + "loss": 0.7236, + "step": 860 + }, + { + "epoch": 1.0597196831200488, + "grad_norm": 0.2047470211982727, + "learning_rate": 5e-05, + "loss": 0.711, + "step": 870 + }, + { + "epoch": 1.0719073735527118, + "grad_norm": 0.18809868395328522, + "learning_rate": 5e-05, + "loss": 0.7253, + "step": 880 + }, + { + "epoch": 1.0840950639853748, + "grad_norm": 0.2207012176513672, + "learning_rate": 5e-05, + "loss": 0.6971, + "step": 890 + }, + { + "epoch": 1.0962827544180378, + "grad_norm": 0.1912035495042801, + "learning_rate": 5e-05, + "loss": 0.7239, + "step": 900 + }, + { + "epoch": 1.1084704448507008, + "grad_norm": 0.23486360907554626, + "learning_rate": 5e-05, + "loss": 0.7335, + "step": 910 + }, + { + "epoch": 1.1206581352833638, + "grad_norm": 0.22189787030220032, + "learning_rate": 5e-05, + "loss": 0.7149, + "step": 920 + }, + { + "epoch": 1.1328458257160268, + "grad_norm": 0.25818824768066406, + "learning_rate": 5e-05, + "loss": 0.7322, + "step": 930 + }, + { + "epoch": 1.1450335161486898, + "grad_norm": 0.2088775783777237, + "learning_rate": 5e-05, + "loss": 0.7382, + "step": 940 + }, + { + "epoch": 1.1572212065813527, + "grad_norm": 0.19445833563804626, + "learning_rate": 5e-05, + "loss": 0.7223, + "step": 950 + }, + { + "epoch": 1.1694088970140157, + "grad_norm": 0.21489456295967102, + "learning_rate": 5e-05, + "loss": 0.7317, + "step": 960 + }, + { + "epoch": 1.181596587446679, + "grad_norm": 0.24050794541835785, + "learning_rate": 5e-05, + "loss": 0.7275, + "step": 970 + }, + { + "epoch": 1.193784277879342, + "grad_norm": 0.18142102658748627, + "learning_rate": 5e-05, + "loss": 0.7046, + "step": 980 + }, + { + "epoch": 1.205971968312005, + "grad_norm": 0.4781077802181244, + "learning_rate": 5e-05, + "loss": 0.7433, + "step": 990 + }, + { + "epoch": 1.218159658744668, + "grad_norm": 0.20628023147583008, + "learning_rate": 5e-05, + "loss": 0.7403, + "step": 1000 + }, + { + "epoch": 1.230347349177331, + "grad_norm": 0.19117140769958496, + "learning_rate": 5e-05, + "loss": 0.7308, + "step": 1010 + }, + { + "epoch": 1.242535039609994, + "grad_norm": 0.2107745110988617, + "learning_rate": 5e-05, + "loss": 0.7201, + "step": 1020 + }, + { + "epoch": 1.2547227300426569, + "grad_norm": 0.20551154017448425, + "learning_rate": 5e-05, + "loss": 0.7038, + "step": 1030 + }, + { + "epoch": 1.2669104204753199, + "grad_norm": 0.26577022671699524, + "learning_rate": 5e-05, + "loss": 0.7096, + "step": 1040 + }, + { + "epoch": 1.2790981109079829, + "grad_norm": 0.22094209492206573, + "learning_rate": 5e-05, + "loss": 0.718, + "step": 1050 + }, + { + "epoch": 1.291285801340646, + "grad_norm": 0.18224076926708221, + "learning_rate": 5e-05, + "loss": 0.7261, + "step": 1060 + }, + { + "epoch": 1.303473491773309, + "grad_norm": 0.226251482963562, + "learning_rate": 5e-05, + "loss": 0.7372, + "step": 1070 + }, + { + "epoch": 1.315661182205972, + "grad_norm": 0.20186461508274078, + "learning_rate": 5e-05, + "loss": 0.7074, + "step": 1080 + }, + { + "epoch": 1.327848872638635, + "grad_norm": 0.22229762375354767, + "learning_rate": 5e-05, + "loss": 0.7069, + "step": 1090 + }, + { + "epoch": 1.340036563071298, + "grad_norm": 0.1776452213525772, + "learning_rate": 5e-05, + "loss": 0.714, + "step": 1100 + }, + { + "epoch": 1.352224253503961, + "grad_norm": 0.20680229365825653, + "learning_rate": 5e-05, + "loss": 0.7066, + "step": 1110 + }, + { + "epoch": 1.364411943936624, + "grad_norm": 0.21842031180858612, + "learning_rate": 5e-05, + "loss": 0.7281, + "step": 1120 + }, + { + "epoch": 1.376599634369287, + "grad_norm": 0.28037774562835693, + "learning_rate": 5e-05, + "loss": 0.7149, + "step": 1130 + }, + { + "epoch": 1.38878732480195, + "grad_norm": 0.20962920784950256, + "learning_rate": 5e-05, + "loss": 0.6951, + "step": 1140 + }, + { + "epoch": 1.400975015234613, + "grad_norm": 0.18751871585845947, + "learning_rate": 5e-05, + "loss": 0.7101, + "step": 1150 + }, + { + "epoch": 1.413162705667276, + "grad_norm": 0.2722964584827423, + "learning_rate": 5e-05, + "loss": 0.7205, + "step": 1160 + }, + { + "epoch": 1.425350396099939, + "grad_norm": 0.20932041108608246, + "learning_rate": 5e-05, + "loss": 0.7082, + "step": 1170 + }, + { + "epoch": 1.437538086532602, + "grad_norm": 0.17836257815361023, + "learning_rate": 5e-05, + "loss": 0.7232, + "step": 1180 + }, + { + "epoch": 1.449725776965265, + "grad_norm": 0.3180703818798065, + "learning_rate": 5e-05, + "loss": 0.7245, + "step": 1190 + }, + { + "epoch": 1.4619134673979282, + "grad_norm": 0.201424703001976, + "learning_rate": 5e-05, + "loss": 0.7232, + "step": 1200 + }, + { + "epoch": 1.4741011578305911, + "grad_norm": 0.20228999853134155, + "learning_rate": 5e-05, + "loss": 0.7169, + "step": 1210 + }, + { + "epoch": 1.4862888482632541, + "grad_norm": 0.17478394508361816, + "learning_rate": 5e-05, + "loss": 0.704, + "step": 1220 + }, + { + "epoch": 1.4984765386959171, + "grad_norm": 0.20061351358890533, + "learning_rate": 5e-05, + "loss": 0.6792, + "step": 1230 + }, + { + "epoch": 1.5106642291285801, + "grad_norm": 0.1857377588748932, + "learning_rate": 5e-05, + "loss": 0.709, + "step": 1240 + }, + { + "epoch": 1.522851919561243, + "grad_norm": 0.19873379170894623, + "learning_rate": 5e-05, + "loss": 0.7007, + "step": 1250 + }, + { + "epoch": 1.5350396099939063, + "grad_norm": 0.19807074964046478, + "learning_rate": 5e-05, + "loss": 0.7122, + "step": 1260 + }, + { + "epoch": 1.5472273004265693, + "grad_norm": 0.16746796667575836, + "learning_rate": 5e-05, + "loss": 0.7005, + "step": 1270 + }, + { + "epoch": 1.5594149908592323, + "grad_norm": 0.2750326991081238, + "learning_rate": 5e-05, + "loss": 0.7166, + "step": 1280 + }, + { + "epoch": 1.5716026812918953, + "grad_norm": 0.2776738107204437, + "learning_rate": 5e-05, + "loss": 0.7405, + "step": 1290 + }, + { + "epoch": 1.5837903717245583, + "grad_norm": 0.3494517207145691, + "learning_rate": 5e-05, + "loss": 0.7192, + "step": 1300 + }, + { + "epoch": 1.5959780621572213, + "grad_norm": 0.25028494000434875, + "learning_rate": 5e-05, + "loss": 0.741, + "step": 1310 + }, + { + "epoch": 1.6081657525898843, + "grad_norm": 0.23914143443107605, + "learning_rate": 5e-05, + "loss": 0.72, + "step": 1320 + }, + { + "epoch": 1.6203534430225472, + "grad_norm": 0.2052862048149109, + "learning_rate": 5e-05, + "loss": 0.7112, + "step": 1330 + }, + { + "epoch": 1.6325411334552102, + "grad_norm": 0.22285936772823334, + "learning_rate": 5e-05, + "loss": 0.7165, + "step": 1340 + }, + { + "epoch": 1.6447288238878732, + "grad_norm": 0.20810848474502563, + "learning_rate": 5e-05, + "loss": 0.7122, + "step": 1350 + }, + { + "epoch": 1.6569165143205362, + "grad_norm": 0.20422884821891785, + "learning_rate": 5e-05, + "loss": 0.7082, + "step": 1360 + }, + { + "epoch": 1.6691042047531992, + "grad_norm": 0.18164314329624176, + "learning_rate": 5e-05, + "loss": 0.7131, + "step": 1370 + }, + { + "epoch": 1.6812918951858622, + "grad_norm": 0.19893014430999756, + "learning_rate": 5e-05, + "loss": 0.7055, + "step": 1380 + }, + { + "epoch": 1.6934795856185252, + "grad_norm": 0.24865971505641937, + "learning_rate": 5e-05, + "loss": 0.6994, + "step": 1390 + }, + { + "epoch": 1.7056672760511882, + "grad_norm": 0.18078123033046722, + "learning_rate": 5e-05, + "loss": 0.7162, + "step": 1400 + }, + { + "epoch": 1.7178549664838512, + "grad_norm": 0.1973421722650528, + "learning_rate": 5e-05, + "loss": 0.7136, + "step": 1410 + }, + { + "epoch": 1.7300426569165142, + "grad_norm": 0.2057911604642868, + "learning_rate": 5e-05, + "loss": 0.7004, + "step": 1420 + }, + { + "epoch": 1.7422303473491774, + "grad_norm": 0.18640495836734772, + "learning_rate": 5e-05, + "loss": 0.7102, + "step": 1430 + }, + { + "epoch": 1.7544180377818404, + "grad_norm": 0.21038417518138885, + "learning_rate": 5e-05, + "loss": 0.6908, + "step": 1440 + }, + { + "epoch": 1.7666057282145033, + "grad_norm": 0.2123890221118927, + "learning_rate": 5e-05, + "loss": 0.6913, + "step": 1450 + }, + { + "epoch": 1.7787934186471663, + "grad_norm": 0.21619991958141327, + "learning_rate": 5e-05, + "loss": 0.7038, + "step": 1460 + }, + { + "epoch": 1.7909811090798293, + "grad_norm": 0.1975220888853073, + "learning_rate": 5e-05, + "loss": 0.7041, + "step": 1470 + }, + { + "epoch": 1.8031687995124925, + "grad_norm": 0.20595207810401917, + "learning_rate": 5e-05, + "loss": 0.7145, + "step": 1480 + }, + { + "epoch": 1.8153564899451555, + "grad_norm": 0.16633334755897522, + "learning_rate": 5e-05, + "loss": 0.6963, + "step": 1490 + }, + { + "epoch": 1.8275441803778185, + "grad_norm": 0.19140620529651642, + "learning_rate": 5e-05, + "loss": 0.6814, + "step": 1500 + }, + { + "epoch": 1.8397318708104815, + "grad_norm": 0.1833905130624771, + "learning_rate": 5e-05, + "loss": 0.7174, + "step": 1510 + }, + { + "epoch": 1.8519195612431445, + "grad_norm": 0.20587337017059326, + "learning_rate": 5e-05, + "loss": 0.7098, + "step": 1520 + }, + { + "epoch": 1.8641072516758075, + "grad_norm": 0.21952909231185913, + "learning_rate": 5e-05, + "loss": 0.6912, + "step": 1530 + }, + { + "epoch": 1.8762949421084705, + "grad_norm": 0.23233620822429657, + "learning_rate": 5e-05, + "loss": 0.6992, + "step": 1540 + }, + { + "epoch": 1.8884826325411335, + "grad_norm": 0.17159809172153473, + "learning_rate": 5e-05, + "loss": 0.7052, + "step": 1550 + }, + { + "epoch": 1.9006703229737965, + "grad_norm": 0.23522134125232697, + "learning_rate": 5e-05, + "loss": 0.6873, + "step": 1560 + }, + { + "epoch": 1.9128580134064594, + "grad_norm": 0.21959669888019562, + "learning_rate": 5e-05, + "loss": 0.6834, + "step": 1570 + }, + { + "epoch": 1.9250457038391224, + "grad_norm": 0.2110578864812851, + "learning_rate": 5e-05, + "loss": 0.7038, + "step": 1580 + }, + { + "epoch": 1.9372333942717854, + "grad_norm": 0.18674372136592865, + "learning_rate": 5e-05, + "loss": 0.6743, + "step": 1590 + }, + { + "epoch": 1.9494210847044484, + "grad_norm": 0.18220046162605286, + "learning_rate": 5e-05, + "loss": 0.6943, + "step": 1600 + }, + { + "epoch": 1.9616087751371114, + "grad_norm": 0.20756569504737854, + "learning_rate": 5e-05, + "loss": 0.7049, + "step": 1610 + }, + { + "epoch": 1.9737964655697744, + "grad_norm": 0.1925211250782013, + "learning_rate": 5e-05, + "loss": 0.7033, + "step": 1620 + }, + { + "epoch": 1.9859841560024374, + "grad_norm": 0.21374447643756866, + "learning_rate": 5e-05, + "loss": 0.7025, + "step": 1630 + }, + { + "epoch": 1.9981718464351006, + "grad_norm": 0.1952880322933197, + "learning_rate": 5e-05, + "loss": 0.6877, + "step": 1640 + }, + { + "epoch": 2.0097501523461303, + "grad_norm": 0.20457723736763, + "learning_rate": 5e-05, + "loss": 0.6734, + "step": 1650 + }, + { + "epoch": 2.0219378427787933, + "grad_norm": 0.20965932309627533, + "learning_rate": 5e-05, + "loss": 0.6472, + "step": 1660 + }, + { + "epoch": 2.0341255332114563, + "grad_norm": 0.19478890299797058, + "learning_rate": 5e-05, + "loss": 0.6781, + "step": 1670 + }, + { + "epoch": 2.0463132236441193, + "grad_norm": 0.21520769596099854, + "learning_rate": 5e-05, + "loss": 0.6592, + "step": 1680 + }, + { + "epoch": 2.0585009140767823, + "grad_norm": 0.20154155790805817, + "learning_rate": 5e-05, + "loss": 0.6744, + "step": 1690 + }, + { + "epoch": 2.0706886045094453, + "grad_norm": 0.1899706870317459, + "learning_rate": 5e-05, + "loss": 0.6592, + "step": 1700 + }, + { + "epoch": 2.0828762949421087, + "grad_norm": 0.20379123091697693, + "learning_rate": 5e-05, + "loss": 0.6688, + "step": 1710 + }, + { + "epoch": 2.0950639853747717, + "grad_norm": 0.19799865782260895, + "learning_rate": 5e-05, + "loss": 0.6678, + "step": 1720 + }, + { + "epoch": 2.1072516758074347, + "grad_norm": 0.19186025857925415, + "learning_rate": 5e-05, + "loss": 0.6587, + "step": 1730 + }, + { + "epoch": 2.1194393662400977, + "grad_norm": 0.18936173617839813, + "learning_rate": 5e-05, + "loss": 0.668, + "step": 1740 + }, + { + "epoch": 2.1316270566727606, + "grad_norm": 0.19159527122974396, + "learning_rate": 5e-05, + "loss": 0.6643, + "step": 1750 + }, + { + "epoch": 2.1438147471054236, + "grad_norm": 0.20596368610858917, + "learning_rate": 5e-05, + "loss": 0.6667, + "step": 1760 + }, + { + "epoch": 2.1560024375380866, + "grad_norm": 0.18601712584495544, + "learning_rate": 5e-05, + "loss": 0.6598, + "step": 1770 + }, + { + "epoch": 2.1681901279707496, + "grad_norm": 0.2130655199289322, + "learning_rate": 5e-05, + "loss": 0.6844, + "step": 1780 + }, + { + "epoch": 2.1803778184034126, + "grad_norm": 0.20477338135242462, + "learning_rate": 5e-05, + "loss": 0.666, + "step": 1790 + }, + { + "epoch": 2.1925655088360756, + "grad_norm": 0.18484896421432495, + "learning_rate": 5e-05, + "loss": 0.6659, + "step": 1800 + }, + { + "epoch": 2.2047531992687386, + "grad_norm": 0.17023979127407074, + "learning_rate": 5e-05, + "loss": 0.6721, + "step": 1810 + }, + { + "epoch": 2.2169408897014016, + "grad_norm": 0.23462794721126556, + "learning_rate": 5e-05, + "loss": 0.6699, + "step": 1820 + }, + { + "epoch": 2.2291285801340646, + "grad_norm": 0.16521663963794708, + "learning_rate": 5e-05, + "loss": 0.6584, + "step": 1830 + }, + { + "epoch": 2.2413162705667276, + "grad_norm": 0.19637759029865265, + "learning_rate": 5e-05, + "loss": 0.6621, + "step": 1840 + }, + { + "epoch": 2.2535039609993905, + "grad_norm": 0.2099665403366089, + "learning_rate": 5e-05, + "loss": 0.6912, + "step": 1850 + }, + { + "epoch": 2.2656916514320535, + "grad_norm": 0.21308118104934692, + "learning_rate": 5e-05, + "loss": 0.6705, + "step": 1860 + }, + { + "epoch": 2.2778793418647165, + "grad_norm": 0.20638489723205566, + "learning_rate": 5e-05, + "loss": 0.6736, + "step": 1870 + }, + { + "epoch": 2.2900670322973795, + "grad_norm": 0.23812630772590637, + "learning_rate": 5e-05, + "loss": 0.6869, + "step": 1880 + }, + { + "epoch": 2.3022547227300425, + "grad_norm": 0.23223577439785004, + "learning_rate": 5e-05, + "loss": 0.6663, + "step": 1890 + }, + { + "epoch": 2.3144424131627055, + "grad_norm": 0.20844320952892303, + "learning_rate": 5e-05, + "loss": 0.6686, + "step": 1900 + }, + { + "epoch": 2.3266301035953685, + "grad_norm": 0.2517656683921814, + "learning_rate": 5e-05, + "loss": 0.658, + "step": 1910 + }, + { + "epoch": 2.3388177940280315, + "grad_norm": 0.20005013048648834, + "learning_rate": 5e-05, + "loss": 0.6659, + "step": 1920 + }, + { + "epoch": 2.3510054844606945, + "grad_norm": 0.20756477117538452, + "learning_rate": 5e-05, + "loss": 0.6613, + "step": 1930 + }, + { + "epoch": 2.363193174893358, + "grad_norm": 0.19551488757133484, + "learning_rate": 5e-05, + "loss": 0.676, + "step": 1940 + }, + { + "epoch": 2.375380865326021, + "grad_norm": 0.18600064516067505, + "learning_rate": 5e-05, + "loss": 0.6826, + "step": 1950 + }, + { + "epoch": 2.387568555758684, + "grad_norm": 0.2445133775472641, + "learning_rate": 5e-05, + "loss": 0.6623, + "step": 1960 + }, + { + "epoch": 2.399756246191347, + "grad_norm": 0.2174660712480545, + "learning_rate": 5e-05, + "loss": 0.6697, + "step": 1970 + }, + { + "epoch": 2.41194393662401, + "grad_norm": 0.1853487342596054, + "learning_rate": 5e-05, + "loss": 0.6561, + "step": 1980 + }, + { + "epoch": 2.424131627056673, + "grad_norm": 0.18266689777374268, + "learning_rate": 5e-05, + "loss": 0.6697, + "step": 1990 + }, + { + "epoch": 2.436319317489336, + "grad_norm": 0.16344764828681946, + "learning_rate": 5e-05, + "loss": 0.6827, + "step": 2000 + }, + { + "epoch": 2.448507007921999, + "grad_norm": 0.18712133169174194, + "learning_rate": 5e-05, + "loss": 0.6713, + "step": 2010 + }, + { + "epoch": 2.460694698354662, + "grad_norm": 0.2192094475030899, + "learning_rate": 5e-05, + "loss": 0.6761, + "step": 2020 + }, + { + "epoch": 2.472882388787325, + "grad_norm": 0.18980957567691803, + "learning_rate": 5e-05, + "loss": 0.7001, + "step": 2030 + }, + { + "epoch": 2.485070079219988, + "grad_norm": 0.20758011937141418, + "learning_rate": 5e-05, + "loss": 0.6801, + "step": 2040 + }, + { + "epoch": 2.497257769652651, + "grad_norm": 0.166823148727417, + "learning_rate": 5e-05, + "loss": 0.6667, + "step": 2050 + }, + { + "epoch": 2.5094454600853138, + "grad_norm": 0.17631390690803528, + "learning_rate": 5e-05, + "loss": 0.66, + "step": 2060 + }, + { + "epoch": 2.5216331505179768, + "grad_norm": 0.19680799543857574, + "learning_rate": 5e-05, + "loss": 0.6513, + "step": 2070 + }, + { + "epoch": 2.5338208409506398, + "grad_norm": 0.202071875333786, + "learning_rate": 5e-05, + "loss": 0.6655, + "step": 2080 + }, + { + "epoch": 2.5460085313833027, + "grad_norm": 0.1721998006105423, + "learning_rate": 5e-05, + "loss": 0.6564, + "step": 2090 + }, + { + "epoch": 2.5581962218159657, + "grad_norm": 0.17605794966220856, + "learning_rate": 5e-05, + "loss": 0.6715, + "step": 2100 + }, + { + "epoch": 2.5703839122486287, + "grad_norm": 0.2009730488061905, + "learning_rate": 5e-05, + "loss": 0.6764, + "step": 2110 + }, + { + "epoch": 2.582571602681292, + "grad_norm": 0.21355752646923065, + "learning_rate": 5e-05, + "loss": 0.6696, + "step": 2120 + }, + { + "epoch": 2.594759293113955, + "grad_norm": 0.18136924505233765, + "learning_rate": 5e-05, + "loss": 0.6459, + "step": 2130 + }, + { + "epoch": 2.606946983546618, + "grad_norm": 0.1764170080423355, + "learning_rate": 5e-05, + "loss": 0.6565, + "step": 2140 + }, + { + "epoch": 2.619134673979281, + "grad_norm": 0.1911628693342209, + "learning_rate": 5e-05, + "loss": 0.6534, + "step": 2150 + }, + { + "epoch": 2.631322364411944, + "grad_norm": 0.18562854826450348, + "learning_rate": 5e-05, + "loss": 0.6505, + "step": 2160 + }, + { + "epoch": 2.643510054844607, + "grad_norm": 0.17010731995105743, + "learning_rate": 5e-05, + "loss": 0.6676, + "step": 2170 + }, + { + "epoch": 2.65569774527727, + "grad_norm": 0.1590568870306015, + "learning_rate": 5e-05, + "loss": 0.638, + "step": 2180 + }, + { + "epoch": 2.667885435709933, + "grad_norm": 0.17398956418037415, + "learning_rate": 5e-05, + "loss": 0.6684, + "step": 2190 + }, + { + "epoch": 2.680073126142596, + "grad_norm": 0.1765497624874115, + "learning_rate": 5e-05, + "loss": 0.6633, + "step": 2200 + }, + { + "epoch": 2.692260816575259, + "grad_norm": 0.19518159329891205, + "learning_rate": 5e-05, + "loss": 0.6742, + "step": 2210 + }, + { + "epoch": 2.704448507007922, + "grad_norm": 0.22384262084960938, + "learning_rate": 5e-05, + "loss": 0.6544, + "step": 2220 + }, + { + "epoch": 2.716636197440585, + "grad_norm": 0.1654004603624344, + "learning_rate": 5e-05, + "loss": 0.6572, + "step": 2230 + }, + { + "epoch": 2.728823887873248, + "grad_norm": 0.18328379094600677, + "learning_rate": 5e-05, + "loss": 0.6714, + "step": 2240 + }, + { + "epoch": 2.741011578305911, + "grad_norm": 0.26779523491859436, + "learning_rate": 5e-05, + "loss": 0.6624, + "step": 2250 + }, + { + "epoch": 2.753199268738574, + "grad_norm": 0.1870274394750595, + "learning_rate": 5e-05, + "loss": 0.6818, + "step": 2260 + }, + { + "epoch": 2.765386959171237, + "grad_norm": 0.21015630662441254, + "learning_rate": 5e-05, + "loss": 0.6758, + "step": 2270 + }, + { + "epoch": 2.7775746496039, + "grad_norm": 0.18667075037956238, + "learning_rate": 5e-05, + "loss": 0.6822, + "step": 2280 + }, + { + "epoch": 2.789762340036563, + "grad_norm": 0.22627151012420654, + "learning_rate": 5e-05, + "loss": 0.6776, + "step": 2290 + }, + { + "epoch": 2.801950030469226, + "grad_norm": 0.20017723739147186, + "learning_rate": 5e-05, + "loss": 0.6685, + "step": 2300 + }, + { + "epoch": 2.814137720901889, + "grad_norm": 0.1894821971654892, + "learning_rate": 5e-05, + "loss": 0.6515, + "step": 2310 + }, + { + "epoch": 2.826325411334552, + "grad_norm": 0.17785170674324036, + "learning_rate": 5e-05, + "loss": 0.676, + "step": 2320 + }, + { + "epoch": 2.838513101767215, + "grad_norm": 0.7713242769241333, + "learning_rate": 5e-05, + "loss": 0.6646, + "step": 2330 + }, + { + "epoch": 2.850700792199878, + "grad_norm": 0.19354282319545746, + "learning_rate": 5e-05, + "loss": 0.6781, + "step": 2340 + }, + { + "epoch": 2.862888482632541, + "grad_norm": 0.2110157161951065, + "learning_rate": 5e-05, + "loss": 0.6765, + "step": 2350 + }, + { + "epoch": 2.875076173065204, + "grad_norm": 0.2045946568250656, + "learning_rate": 5e-05, + "loss": 0.6855, + "step": 2360 + }, + { + "epoch": 2.887263863497867, + "grad_norm": 0.15799324214458466, + "learning_rate": 5e-05, + "loss": 0.6747, + "step": 2370 + }, + { + "epoch": 2.89945155393053, + "grad_norm": 0.17158573865890503, + "learning_rate": 5e-05, + "loss": 0.6735, + "step": 2380 + }, + { + "epoch": 2.9116392443631933, + "grad_norm": 0.16838699579238892, + "learning_rate": 5e-05, + "loss": 0.6717, + "step": 2390 + }, + { + "epoch": 2.9238269347958563, + "grad_norm": 0.23229676485061646, + "learning_rate": 5e-05, + "loss": 0.6609, + "step": 2400 + }, + { + "epoch": 2.9360146252285193, + "grad_norm": 0.1893588900566101, + "learning_rate": 5e-05, + "loss": 0.6676, + "step": 2410 + }, + { + "epoch": 2.9482023156611823, + "grad_norm": 0.17633271217346191, + "learning_rate": 5e-05, + "loss": 0.6556, + "step": 2420 + }, + { + "epoch": 2.9603900060938453, + "grad_norm": 0.18573276698589325, + "learning_rate": 5e-05, + "loss": 0.6624, + "step": 2430 + }, + { + "epoch": 2.9725776965265083, + "grad_norm": 0.16133904457092285, + "learning_rate": 5e-05, + "loss": 0.662, + "step": 2440 + }, + { + "epoch": 2.9847653869591713, + "grad_norm": 0.1573924720287323, + "learning_rate": 5e-05, + "loss": 0.6613, + "step": 2450 + }, + { + "epoch": 2.9969530773918343, + "grad_norm": 0.17543254792690277, + "learning_rate": 5e-05, + "loss": 0.6895, + "step": 2460 + } + ], + "logging_steps": 10, + "max_steps": 2463, + "num_input_tokens_seen": 0, + "num_train_epochs": 3, + "save_steps": 205, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 1.0100178660314055e+19, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2463/training_args.bin b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2463/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..23ae0ec8a4226be5457f61c137abab2b2c10a0cf --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2463/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56896ea21940e33d7e2934267dff2a5b4a4a80195a0f165245f6617a0f88b90b +size 7377 diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2463/zero_to_fp32.py b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2463/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..5995d6e6f04e43b989587aa9022a3aef0c66d694 --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-2463/zero_to_fp32.py @@ -0,0 +1,760 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: +# python zero_to_fp32.py . output_dir/ +# or +# python zero_to_fp32.py . output_dir/ --safe_serialization + +import argparse +import torch +import glob +import math +import os +import re +import gc +import json +import numpy as np +from tqdm import tqdm +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device, weights_only=False) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + total_files = len(files) + state_dicts = [] + for f in tqdm(files, desc='Loading checkpoint shards'): + state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + model_files = get_model_state_files(ds_checkpoint_dir) + + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +class GatheredTensor: + """ + A pseudo tensor that collects partitioned weights. + It is more memory efficient when there are multiple groups. + """ + + def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape): + self.flat_groups = flat_groups + self.flat_groups_offset = flat_groups_offset + self.offset = offset + self.partitioned_numel = partitioned_numel + self.shape = shape + self.dtype = self.flat_groups[0][0].dtype + + def contiguous(self): + """ + Merge partitioned weights from flat_groups into a single tensor. + """ + end_idx = self.offset + self.partitioned_numel + world_size = len(self.flat_groups) + pad_flat_param_chunks = [] + + for rank_i in range(world_size): + # for each rank, we need to collect weights from related group/groups + flat_groups_at_rank_i = self.flat_groups[rank_i] + start_group_id = None + end_group_id = None + for group_id in range(len(self.flat_groups_offset)): + if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]: + start_group_id = group_id + if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]: + end_group_id = group_id + break + # collect weights from related group/groups + for group_id in range(start_group_id, end_group_id + 1): + flat_tensor = flat_groups_at_rank_i[group_id] + start_offset = self.offset - self.flat_groups_offset[group_id] + end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id] + pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset]) + + # collect weights from all ranks + pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0) + param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous() + return param + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size + + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]])) + for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'): + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # memory efficient tensor + tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape) + state_dict[name] = tensor + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def to_torch_tensor(state_dict, return_empty_tensor=False): + """ + Convert state_dict of GatheredTensor to torch tensor + """ + torch_state_dict = {} + converted_tensors = {} + for name, tensor in state_dict.items(): + tensor_id = id(tensor) + if tensor_id in converted_tensors: # shared tensors + shared_tensor = torch_state_dict[converted_tensors[tensor_id]] + torch_state_dict[name] = shared_tensor + else: + converted_tensors[tensor_id] = name + if return_empty_tensor: + torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype) + else: + torch_state_dict[name] = tensor.contiguous() + return torch_state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag=None, + exclude_frozen_parameters=False, + lazy_mode=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient. + Convert the pesduo tensor to torch tensor by ``.contiguous()`` + + Returns: + - pytorch ``state_dict`` + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + Note: the above usage may not work if your application doesn't have sufficient free CPU memory. + You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. Or you can load state_dict in lazy mode :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu + for name, lazy_tensor in state_dict.item(): + tensor = lazy_tensor.contiguous() # to cpu + print(name, tensor) + # del tensor to release memory if it no longer in use + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + if lazy_mode: + return state_dict + else: + return to_torch_tensor(state_dict) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, + output_dir, + max_shard_size="5GB", + safe_serialization=False, + tag=None, + exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_dir``: directory to the pytorch fp32 state_dict output files + - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB + - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + + # Dependency pre-check + if safe_serialization: + try: + from safetensors.torch import save_file + except ImportError: + print('If you want to use `safe_serialization`, please `pip install safetensors`') + raise + if max_shard_size is not None: + try: + from huggingface_hub import split_torch_state_dict_into_shards + except ImportError: + print('If you want to use `max_shard_size`, please `pip install huggingface_hub`') + raise + + # Convert zero checkpoint to state_dict + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag, + exclude_frozen_parameters, + lazy_mode=True) + + # Shard the model if it is too big. + weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin" + if max_shard_size is not None: + filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors") + # an memory-efficient approach for sharding + empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True) + state_dict_split = split_torch_state_dict_into_shards(empty_state_dict, + filename_pattern=filename_pattern, + max_shard_size=max_shard_size) + else: + from collections import namedtuple + StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"]) + state_dict_split = StateDictSplit(is_sharded=False, + filename_to_tensors={weights_name: list(state_dict.keys())}) + + # Save the model by shard + os.makedirs(output_dir, exist_ok=True) + filename_to_tensors = state_dict_split.filename_to_tensors.items() + for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"): + shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors} + shard_state_dict = to_torch_tensor(shard_state_dict) + output_path = os.path.join(output_dir, shard_file) + if safe_serialization: + save_file(shard_state_dict, output_path, metadata={"format": "pt"}) + else: + torch.save(shard_state_dict, output_path) + # release the memory of current shard + for tensor_name in list(shard_state_dict.keys()): + del state_dict[tensor_name] + del shard_state_dict[tensor_name] + del shard_state_dict + gc.collect() + + # Save index if sharded + if state_dict_split.is_sharded: + index = { + "metadata": state_dict_split.metadata, + "weight_map": state_dict_split.tensor_to_filename, + } + save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json" + save_index_file = os.path.join(output_dir, save_index_file) + with open(save_index_file, "w", encoding="utf-8") as f: + content = json.dumps(index, indent=2, sort_keys=True) + "\n" + f.write(content) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info("Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info("Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument("output_dir", + type=str, + help="directory to the pytorch fp32 state_dict output files" + "(e.g. path/checkpoint-12-output/)") + parser.add_argument( + "--max_shard_size", + type=str, + default="5GB", + help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size" + "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`" + "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances" + "without CPU OOM issues.") + parser.add_argument( + "--safe_serialization", + default=False, + action='store_true', + help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_dir, + max_shard_size=args.max_shard_size, + safe_serialization=args.safe_serialization, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-410/config.json b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-410/config.json new file mode 100644 index 0000000000000000000000000000000000000000..18bd58d561964a16b60687f1b04aefdfed0b9b20 --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-410/config.json @@ -0,0 +1,39 @@ +{ + "architectures": [ + "LlamaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "dtype": "bfloat16", + "eos_token_id": [ + 128001, + 128008, + 128009 + ], + "head_dim": 64, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 8192, + "max_position_embeddings": 131072, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 32, + "num_hidden_layers": 16, + "num_key_value_heads": 8, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": { + "factor": 32.0, + "high_freq_factor": 4.0, + "low_freq_factor": 1.0, + "original_max_position_embeddings": 8192, + "rope_type": "llama3" + }, + "rope_theta": 500000.0, + "tie_word_embeddings": true, + "transformers_version": "4.57.1", + "use_cache": false, + "vocab_size": 128256 +} diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-410/generation_config.json b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-410/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..125cc74759bc705ab8a0c48d07d7eac083f56011 --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-410/generation_config.json @@ -0,0 +1,12 @@ +{ + "bos_token_id": 128000, + "do_sample": true, + "eos_token_id": [ + 128001, + 128008, + 128009 + ], + "temperature": 0.6, + "top_p": 0.9, + "transformers_version": "4.57.1" +} diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-410/latest b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-410/latest new file mode 100644 index 0000000000000000000000000000000000000000..c4ebe9e99e7ad00183b3b16fab7bbf0230d6d177 --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-410/latest @@ -0,0 +1 @@ +global_step410 \ No newline at end of file diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-410/model.safetensors b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-410/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f63849344ab494a4e8ad3e772dd561f2002a7f64 --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-410/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:36866398af6830b706a91072bcd7090cc7fff4d2a0e6fe8bd04e08e72a9bfc91 +size 2996982344 diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-410/scheduler.pt b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-410/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..fdc35c29ab4de0a3ac1c0efc73af1eb7668b60d9 --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-410/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:18d1eccded40950799ee287edf98b9f237a5743b7d38a53b5ce1558a9fe17e59 +size 1465 diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-410/trainer_state.json b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-410/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..81ce948e80ae46f1f9293fba901b1d3295bd1c93 --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-410/trainer_state.json @@ -0,0 +1,321 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.4996953077391834, + "eval_steps": 500, + "global_step": 410, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.01218769043266301, + "grad_norm": 0.36486390233039856, + "learning_rate": 5e-05, + "loss": 1.2213, + "step": 10 + }, + { + "epoch": 0.02437538086532602, + "grad_norm": 0.2953564524650574, + "learning_rate": 5e-05, + "loss": 1.0223, + "step": 20 + }, + { + "epoch": 0.03656307129798903, + "grad_norm": 0.2766735851764679, + "learning_rate": 5e-05, + "loss": 0.9589, + "step": 30 + }, + { + "epoch": 0.04875076173065204, + "grad_norm": 0.32770997285842896, + "learning_rate": 5e-05, + "loss": 0.9081, + "step": 40 + }, + { + "epoch": 0.06093845216331505, + "grad_norm": 0.3052276372909546, + "learning_rate": 5e-05, + "loss": 0.8752, + "step": 50 + }, + { + "epoch": 0.07312614259597806, + "grad_norm": 0.240146666765213, + "learning_rate": 5e-05, + "loss": 0.873, + "step": 60 + }, + { + "epoch": 0.08531383302864107, + "grad_norm": 0.23872187733650208, + "learning_rate": 5e-05, + "loss": 0.8587, + "step": 70 + }, + { + "epoch": 0.09750152346130408, + "grad_norm": 0.32638487219810486, + "learning_rate": 5e-05, + "loss": 0.8656, + "step": 80 + }, + { + "epoch": 0.10968921389396709, + "grad_norm": 0.25019028782844543, + "learning_rate": 5e-05, + "loss": 0.8685, + "step": 90 + }, + { + "epoch": 0.1218769043266301, + "grad_norm": 0.2507675886154175, + "learning_rate": 5e-05, + "loss": 0.8662, + "step": 100 + }, + { + "epoch": 0.1340645947592931, + "grad_norm": 0.27966755628585815, + "learning_rate": 5e-05, + "loss": 0.8428, + "step": 110 + }, + { + "epoch": 0.14625228519195613, + "grad_norm": 0.2622673213481903, + "learning_rate": 5e-05, + "loss": 0.8416, + "step": 120 + }, + { + "epoch": 0.15843997562461914, + "grad_norm": 0.3003966510295868, + "learning_rate": 5e-05, + "loss": 0.8528, + "step": 130 + }, + { + "epoch": 0.17062766605728213, + "grad_norm": 0.32308319211006165, + "learning_rate": 5e-05, + "loss": 0.8357, + "step": 140 + }, + { + "epoch": 0.18281535648994515, + "grad_norm": 0.27279096841812134, + "learning_rate": 5e-05, + "loss": 0.8432, + "step": 150 + }, + { + "epoch": 0.19500304692260817, + "grad_norm": 0.27976104617118835, + "learning_rate": 5e-05, + "loss": 0.8119, + "step": 160 + }, + { + "epoch": 0.2071907373552712, + "grad_norm": 0.290866881608963, + "learning_rate": 5e-05, + "loss": 0.8096, + "step": 170 + }, + { + "epoch": 0.21937842778793418, + "grad_norm": 0.29430949687957764, + "learning_rate": 5e-05, + "loss": 0.819, + "step": 180 + }, + { + "epoch": 0.2315661182205972, + "grad_norm": 0.27714136242866516, + "learning_rate": 5e-05, + "loss": 0.8187, + "step": 190 + }, + { + "epoch": 0.2437538086532602, + "grad_norm": 0.2997029423713684, + "learning_rate": 5e-05, + "loss": 0.808, + "step": 200 + }, + { + "epoch": 0.25594149908592323, + "grad_norm": 0.2539573609828949, + "learning_rate": 5e-05, + "loss": 0.7943, + "step": 210 + }, + { + "epoch": 0.2681291895185862, + "grad_norm": 0.23294171690940857, + "learning_rate": 5e-05, + "loss": 0.8188, + "step": 220 + }, + { + "epoch": 0.28031687995124926, + "grad_norm": 0.25016292929649353, + "learning_rate": 5e-05, + "loss": 0.8035, + "step": 230 + }, + { + "epoch": 0.29250457038391225, + "grad_norm": 0.37125322222709656, + "learning_rate": 5e-05, + "loss": 0.785, + "step": 240 + }, + { + "epoch": 0.30469226081657524, + "grad_norm": 0.21520452201366425, + "learning_rate": 5e-05, + "loss": 0.8088, + "step": 250 + }, + { + "epoch": 0.3168799512492383, + "grad_norm": 0.20990602672100067, + "learning_rate": 5e-05, + "loss": 0.7813, + "step": 260 + }, + { + "epoch": 0.3290676416819013, + "grad_norm": 0.24193961918354034, + "learning_rate": 5e-05, + "loss": 0.7979, + "step": 270 + }, + { + "epoch": 0.34125533211456427, + "grad_norm": 0.22766050696372986, + "learning_rate": 5e-05, + "loss": 0.785, + "step": 280 + }, + { + "epoch": 0.3534430225472273, + "grad_norm": 0.2662702202796936, + "learning_rate": 5e-05, + "loss": 0.7766, + "step": 290 + }, + { + "epoch": 0.3656307129798903, + "grad_norm": 0.23020495474338531, + "learning_rate": 5e-05, + "loss": 0.7687, + "step": 300 + }, + { + "epoch": 0.3778184034125533, + "grad_norm": 0.21770520508289337, + "learning_rate": 5e-05, + "loss": 0.7815, + "step": 310 + }, + { + "epoch": 0.39000609384521634, + "grad_norm": 0.22677403688430786, + "learning_rate": 5e-05, + "loss": 0.7999, + "step": 320 + }, + { + "epoch": 0.40219378427787933, + "grad_norm": 0.24891860783100128, + "learning_rate": 5e-05, + "loss": 0.7946, + "step": 330 + }, + { + "epoch": 0.4143814747105424, + "grad_norm": 0.2506854832172394, + "learning_rate": 5e-05, + "loss": 0.7889, + "step": 340 + }, + { + "epoch": 0.42656916514320536, + "grad_norm": 0.24388180673122406, + "learning_rate": 5e-05, + "loss": 0.79, + "step": 350 + }, + { + "epoch": 0.43875685557586835, + "grad_norm": 0.22329650819301605, + "learning_rate": 5e-05, + "loss": 0.7632, + "step": 360 + }, + { + "epoch": 0.4509445460085314, + "grad_norm": 0.2240927815437317, + "learning_rate": 5e-05, + "loss": 0.8042, + "step": 370 + }, + { + "epoch": 0.4631322364411944, + "grad_norm": 0.2578203082084656, + "learning_rate": 5e-05, + "loss": 0.7667, + "step": 380 + }, + { + "epoch": 0.4753199268738574, + "grad_norm": 0.25349217653274536, + "learning_rate": 5e-05, + "loss": 0.7911, + "step": 390 + }, + { + "epoch": 0.4875076173065204, + "grad_norm": 0.254446804523468, + "learning_rate": 5e-05, + "loss": 0.7909, + "step": 400 + }, + { + "epoch": 0.4996953077391834, + "grad_norm": 0.24253171682357788, + "learning_rate": 5e-05, + "loss": 0.7896, + "step": 410 + } + ], + "logging_steps": 10, + "max_steps": 2463, + "num_input_tokens_seen": 0, + "num_train_epochs": 3, + "save_steps": 205, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.6804064871359447e+18, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-410/training_args.bin b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-410/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..23ae0ec8a4226be5457f61c137abab2b2c10a0cf --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-410/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56896ea21940e33d7e2934267dff2a5b4a4a80195a0f165245f6617a0f88b90b +size 7377 diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-410/zero_to_fp32.py b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-410/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..5995d6e6f04e43b989587aa9022a3aef0c66d694 --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-410/zero_to_fp32.py @@ -0,0 +1,760 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: +# python zero_to_fp32.py . output_dir/ +# or +# python zero_to_fp32.py . output_dir/ --safe_serialization + +import argparse +import torch +import glob +import math +import os +import re +import gc +import json +import numpy as np +from tqdm import tqdm +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device, weights_only=False) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + total_files = len(files) + state_dicts = [] + for f in tqdm(files, desc='Loading checkpoint shards'): + state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + model_files = get_model_state_files(ds_checkpoint_dir) + + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +class GatheredTensor: + """ + A pseudo tensor that collects partitioned weights. + It is more memory efficient when there are multiple groups. + """ + + def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape): + self.flat_groups = flat_groups + self.flat_groups_offset = flat_groups_offset + self.offset = offset + self.partitioned_numel = partitioned_numel + self.shape = shape + self.dtype = self.flat_groups[0][0].dtype + + def contiguous(self): + """ + Merge partitioned weights from flat_groups into a single tensor. + """ + end_idx = self.offset + self.partitioned_numel + world_size = len(self.flat_groups) + pad_flat_param_chunks = [] + + for rank_i in range(world_size): + # for each rank, we need to collect weights from related group/groups + flat_groups_at_rank_i = self.flat_groups[rank_i] + start_group_id = None + end_group_id = None + for group_id in range(len(self.flat_groups_offset)): + if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]: + start_group_id = group_id + if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]: + end_group_id = group_id + break + # collect weights from related group/groups + for group_id in range(start_group_id, end_group_id + 1): + flat_tensor = flat_groups_at_rank_i[group_id] + start_offset = self.offset - self.flat_groups_offset[group_id] + end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id] + pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset]) + + # collect weights from all ranks + pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0) + param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous() + return param + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size + + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]])) + for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'): + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # memory efficient tensor + tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape) + state_dict[name] = tensor + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def to_torch_tensor(state_dict, return_empty_tensor=False): + """ + Convert state_dict of GatheredTensor to torch tensor + """ + torch_state_dict = {} + converted_tensors = {} + for name, tensor in state_dict.items(): + tensor_id = id(tensor) + if tensor_id in converted_tensors: # shared tensors + shared_tensor = torch_state_dict[converted_tensors[tensor_id]] + torch_state_dict[name] = shared_tensor + else: + converted_tensors[tensor_id] = name + if return_empty_tensor: + torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype) + else: + torch_state_dict[name] = tensor.contiguous() + return torch_state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag=None, + exclude_frozen_parameters=False, + lazy_mode=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient. + Convert the pesduo tensor to torch tensor by ``.contiguous()`` + + Returns: + - pytorch ``state_dict`` + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + Note: the above usage may not work if your application doesn't have sufficient free CPU memory. + You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. Or you can load state_dict in lazy mode :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu + for name, lazy_tensor in state_dict.item(): + tensor = lazy_tensor.contiguous() # to cpu + print(name, tensor) + # del tensor to release memory if it no longer in use + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + if lazy_mode: + return state_dict + else: + return to_torch_tensor(state_dict) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, + output_dir, + max_shard_size="5GB", + safe_serialization=False, + tag=None, + exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_dir``: directory to the pytorch fp32 state_dict output files + - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB + - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + + # Dependency pre-check + if safe_serialization: + try: + from safetensors.torch import save_file + except ImportError: + print('If you want to use `safe_serialization`, please `pip install safetensors`') + raise + if max_shard_size is not None: + try: + from huggingface_hub import split_torch_state_dict_into_shards + except ImportError: + print('If you want to use `max_shard_size`, please `pip install huggingface_hub`') + raise + + # Convert zero checkpoint to state_dict + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag, + exclude_frozen_parameters, + lazy_mode=True) + + # Shard the model if it is too big. + weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin" + if max_shard_size is not None: + filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors") + # an memory-efficient approach for sharding + empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True) + state_dict_split = split_torch_state_dict_into_shards(empty_state_dict, + filename_pattern=filename_pattern, + max_shard_size=max_shard_size) + else: + from collections import namedtuple + StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"]) + state_dict_split = StateDictSplit(is_sharded=False, + filename_to_tensors={weights_name: list(state_dict.keys())}) + + # Save the model by shard + os.makedirs(output_dir, exist_ok=True) + filename_to_tensors = state_dict_split.filename_to_tensors.items() + for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"): + shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors} + shard_state_dict = to_torch_tensor(shard_state_dict) + output_path = os.path.join(output_dir, shard_file) + if safe_serialization: + save_file(shard_state_dict, output_path, metadata={"format": "pt"}) + else: + torch.save(shard_state_dict, output_path) + # release the memory of current shard + for tensor_name in list(shard_state_dict.keys()): + del state_dict[tensor_name] + del shard_state_dict[tensor_name] + del shard_state_dict + gc.collect() + + # Save index if sharded + if state_dict_split.is_sharded: + index = { + "metadata": state_dict_split.metadata, + "weight_map": state_dict_split.tensor_to_filename, + } + save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json" + save_index_file = os.path.join(output_dir, save_index_file) + with open(save_index_file, "w", encoding="utf-8") as f: + content = json.dumps(index, indent=2, sort_keys=True) + "\n" + f.write(content) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info("Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info("Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument("output_dir", + type=str, + help="directory to the pytorch fp32 state_dict output files" + "(e.g. path/checkpoint-12-output/)") + parser.add_argument( + "--max_shard_size", + type=str, + default="5GB", + help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size" + "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`" + "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances" + "without CPU OOM issues.") + parser.add_argument( + "--safe_serialization", + default=False, + action='store_true', + help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_dir, + max_shard_size=args.max_shard_size, + safe_serialization=args.safe_serialization, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-615/config.json b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-615/config.json new file mode 100644 index 0000000000000000000000000000000000000000..18bd58d561964a16b60687f1b04aefdfed0b9b20 --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-615/config.json @@ -0,0 +1,39 @@ +{ + "architectures": [ + "LlamaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "dtype": "bfloat16", + "eos_token_id": [ + 128001, + 128008, + 128009 + ], + "head_dim": 64, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 8192, + "max_position_embeddings": 131072, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 32, + "num_hidden_layers": 16, + "num_key_value_heads": 8, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": { + "factor": 32.0, + "high_freq_factor": 4.0, + "low_freq_factor": 1.0, + "original_max_position_embeddings": 8192, + "rope_type": "llama3" + }, + "rope_theta": 500000.0, + "tie_word_embeddings": true, + "transformers_version": "4.57.1", + "use_cache": false, + "vocab_size": 128256 +} diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-615/generation_config.json b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-615/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..125cc74759bc705ab8a0c48d07d7eac083f56011 --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-615/generation_config.json @@ -0,0 +1,12 @@ +{ + "bos_token_id": 128000, + "do_sample": true, + "eos_token_id": [ + 128001, + 128008, + 128009 + ], + "temperature": 0.6, + "top_p": 0.9, + "transformers_version": "4.57.1" +} diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-615/latest b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-615/latest new file mode 100644 index 0000000000000000000000000000000000000000..7b476e7dbfce116a45d8b4e65e0b67eabcb59194 --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-615/latest @@ -0,0 +1 @@ +global_step615 \ No newline at end of file diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-615/model.safetensors b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-615/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..4d69dbbce79422e7ad116c912600d5a1f768d0f5 --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-615/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:077cbef3ea148e975f12b1bcaf46b40839a059a67b74685eda9207e15450552e +size 2996982344 diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-615/scheduler.pt b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-615/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..37e3ae2c3bab0e19d7c424a68a63f68698ba4623 --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-615/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a714b933e7526fb4956ee63066321504219c6cf740d2b6027514ad5ba6633f29 +size 1465 diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-615/trainer_state.json b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-615/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..9e86ace96b0935778f13a59910182cb2ec75dd52 --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-615/trainer_state.json @@ -0,0 +1,461 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.7495429616087751, + "eval_steps": 500, + "global_step": 615, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.01218769043266301, + "grad_norm": 0.36486390233039856, + "learning_rate": 5e-05, + "loss": 1.2213, + "step": 10 + }, + { + "epoch": 0.02437538086532602, + "grad_norm": 0.2953564524650574, + "learning_rate": 5e-05, + "loss": 1.0223, + "step": 20 + }, + { + "epoch": 0.03656307129798903, + "grad_norm": 0.2766735851764679, + "learning_rate": 5e-05, + "loss": 0.9589, + "step": 30 + }, + { + "epoch": 0.04875076173065204, + "grad_norm": 0.32770997285842896, + "learning_rate": 5e-05, + "loss": 0.9081, + "step": 40 + }, + { + "epoch": 0.06093845216331505, + "grad_norm": 0.3052276372909546, + "learning_rate": 5e-05, + "loss": 0.8752, + "step": 50 + }, + { + "epoch": 0.07312614259597806, + "grad_norm": 0.240146666765213, + "learning_rate": 5e-05, + "loss": 0.873, + "step": 60 + }, + { + "epoch": 0.08531383302864107, + "grad_norm": 0.23872187733650208, + "learning_rate": 5e-05, + "loss": 0.8587, + "step": 70 + }, + { + "epoch": 0.09750152346130408, + "grad_norm": 0.32638487219810486, + "learning_rate": 5e-05, + "loss": 0.8656, + "step": 80 + }, + { + "epoch": 0.10968921389396709, + "grad_norm": 0.25019028782844543, + "learning_rate": 5e-05, + "loss": 0.8685, + "step": 90 + }, + { + "epoch": 0.1218769043266301, + "grad_norm": 0.2507675886154175, + "learning_rate": 5e-05, + "loss": 0.8662, + "step": 100 + }, + { + "epoch": 0.1340645947592931, + "grad_norm": 0.27966755628585815, + "learning_rate": 5e-05, + "loss": 0.8428, + "step": 110 + }, + { + "epoch": 0.14625228519195613, + "grad_norm": 0.2622673213481903, + "learning_rate": 5e-05, + "loss": 0.8416, + "step": 120 + }, + { + "epoch": 0.15843997562461914, + "grad_norm": 0.3003966510295868, + "learning_rate": 5e-05, + "loss": 0.8528, + "step": 130 + }, + { + "epoch": 0.17062766605728213, + "grad_norm": 0.32308319211006165, + "learning_rate": 5e-05, + "loss": 0.8357, + "step": 140 + }, + { + "epoch": 0.18281535648994515, + "grad_norm": 0.27279096841812134, + "learning_rate": 5e-05, + "loss": 0.8432, + "step": 150 + }, + { + "epoch": 0.19500304692260817, + "grad_norm": 0.27976104617118835, + "learning_rate": 5e-05, + "loss": 0.8119, + "step": 160 + }, + { + "epoch": 0.2071907373552712, + "grad_norm": 0.290866881608963, + "learning_rate": 5e-05, + "loss": 0.8096, + "step": 170 + }, + { + "epoch": 0.21937842778793418, + "grad_norm": 0.29430949687957764, + "learning_rate": 5e-05, + "loss": 0.819, + "step": 180 + }, + { + "epoch": 0.2315661182205972, + "grad_norm": 0.27714136242866516, + "learning_rate": 5e-05, + "loss": 0.8187, + "step": 190 + }, + { + "epoch": 0.2437538086532602, + "grad_norm": 0.2997029423713684, + "learning_rate": 5e-05, + "loss": 0.808, + "step": 200 + }, + { + "epoch": 0.25594149908592323, + "grad_norm": 0.2539573609828949, + "learning_rate": 5e-05, + "loss": 0.7943, + "step": 210 + }, + { + "epoch": 0.2681291895185862, + "grad_norm": 0.23294171690940857, + "learning_rate": 5e-05, + "loss": 0.8188, + "step": 220 + }, + { + "epoch": 0.28031687995124926, + "grad_norm": 0.25016292929649353, + "learning_rate": 5e-05, + "loss": 0.8035, + "step": 230 + }, + { + "epoch": 0.29250457038391225, + "grad_norm": 0.37125322222709656, + "learning_rate": 5e-05, + "loss": 0.785, + "step": 240 + }, + { + "epoch": 0.30469226081657524, + "grad_norm": 0.21520452201366425, + "learning_rate": 5e-05, + "loss": 0.8088, + "step": 250 + }, + { + "epoch": 0.3168799512492383, + "grad_norm": 0.20990602672100067, + "learning_rate": 5e-05, + "loss": 0.7813, + "step": 260 + }, + { + "epoch": 0.3290676416819013, + "grad_norm": 0.24193961918354034, + "learning_rate": 5e-05, + "loss": 0.7979, + "step": 270 + }, + { + "epoch": 0.34125533211456427, + "grad_norm": 0.22766050696372986, + "learning_rate": 5e-05, + "loss": 0.785, + "step": 280 + }, + { + "epoch": 0.3534430225472273, + "grad_norm": 0.2662702202796936, + "learning_rate": 5e-05, + "loss": 0.7766, + "step": 290 + }, + { + "epoch": 0.3656307129798903, + "grad_norm": 0.23020495474338531, + "learning_rate": 5e-05, + "loss": 0.7687, + "step": 300 + }, + { + "epoch": 0.3778184034125533, + "grad_norm": 0.21770520508289337, + "learning_rate": 5e-05, + "loss": 0.7815, + "step": 310 + }, + { + "epoch": 0.39000609384521634, + "grad_norm": 0.22677403688430786, + "learning_rate": 5e-05, + "loss": 0.7999, + "step": 320 + }, + { + "epoch": 0.40219378427787933, + "grad_norm": 0.24891860783100128, + "learning_rate": 5e-05, + "loss": 0.7946, + "step": 330 + }, + { + "epoch": 0.4143814747105424, + "grad_norm": 0.2506854832172394, + "learning_rate": 5e-05, + "loss": 0.7889, + "step": 340 + }, + { + "epoch": 0.42656916514320536, + "grad_norm": 0.24388180673122406, + "learning_rate": 5e-05, + "loss": 0.79, + "step": 350 + }, + { + "epoch": 0.43875685557586835, + "grad_norm": 0.22329650819301605, + "learning_rate": 5e-05, + "loss": 0.7632, + "step": 360 + }, + { + "epoch": 0.4509445460085314, + "grad_norm": 0.2240927815437317, + "learning_rate": 5e-05, + "loss": 0.8042, + "step": 370 + }, + { + "epoch": 0.4631322364411944, + "grad_norm": 0.2578203082084656, + "learning_rate": 5e-05, + "loss": 0.7667, + "step": 380 + }, + { + "epoch": 0.4753199268738574, + "grad_norm": 0.25349217653274536, + "learning_rate": 5e-05, + "loss": 0.7911, + "step": 390 + }, + { + "epoch": 0.4875076173065204, + "grad_norm": 0.254446804523468, + "learning_rate": 5e-05, + "loss": 0.7909, + "step": 400 + }, + { + "epoch": 0.4996953077391834, + "grad_norm": 0.24253171682357788, + "learning_rate": 5e-05, + "loss": 0.7896, + "step": 410 + }, + { + "epoch": 0.5118829981718465, + "grad_norm": 0.27901458740234375, + "learning_rate": 5e-05, + "loss": 0.7828, + "step": 420 + }, + { + "epoch": 0.5240706886045094, + "grad_norm": 0.31884434819221497, + "learning_rate": 5e-05, + "loss": 0.7919, + "step": 430 + }, + { + "epoch": 0.5362583790371724, + "grad_norm": 0.22920799255371094, + "learning_rate": 5e-05, + "loss": 0.7777, + "step": 440 + }, + { + "epoch": 0.5484460694698354, + "grad_norm": 0.22606618702411652, + "learning_rate": 5e-05, + "loss": 0.7661, + "step": 450 + }, + { + "epoch": 0.5606337599024985, + "grad_norm": 0.21604052186012268, + "learning_rate": 5e-05, + "loss": 0.769, + "step": 460 + }, + { + "epoch": 0.5728214503351615, + "grad_norm": 0.2296101152896881, + "learning_rate": 5e-05, + "loss": 0.7626, + "step": 470 + }, + { + "epoch": 0.5850091407678245, + "grad_norm": 0.24162150919437408, + "learning_rate": 5e-05, + "loss": 0.7831, + "step": 480 + }, + { + "epoch": 0.5971968312004875, + "grad_norm": 0.22364799678325653, + "learning_rate": 5e-05, + "loss": 0.7737, + "step": 490 + }, + { + "epoch": 0.6093845216331505, + "grad_norm": 0.23953403532505035, + "learning_rate": 5e-05, + "loss": 0.787, + "step": 500 + }, + { + "epoch": 0.6215722120658135, + "grad_norm": 0.22349311411380768, + "learning_rate": 5e-05, + "loss": 0.7763, + "step": 510 + }, + { + "epoch": 0.6337599024984766, + "grad_norm": 0.21996377408504486, + "learning_rate": 5e-05, + "loss": 0.7596, + "step": 520 + }, + { + "epoch": 0.6459475929311396, + "grad_norm": 0.245538130402565, + "learning_rate": 5e-05, + "loss": 0.7785, + "step": 530 + }, + { + "epoch": 0.6581352833638026, + "grad_norm": 0.23442520201206207, + "learning_rate": 5e-05, + "loss": 0.7633, + "step": 540 + }, + { + "epoch": 0.6703229737964655, + "grad_norm": 0.22274106740951538, + "learning_rate": 5e-05, + "loss": 0.7756, + "step": 550 + }, + { + "epoch": 0.6825106642291285, + "grad_norm": 0.21717530488967896, + "learning_rate": 5e-05, + "loss": 0.7726, + "step": 560 + }, + { + "epoch": 0.6946983546617916, + "grad_norm": 0.2465955913066864, + "learning_rate": 5e-05, + "loss": 0.7476, + "step": 570 + }, + { + "epoch": 0.7068860450944546, + "grad_norm": 0.2318575084209442, + "learning_rate": 5e-05, + "loss": 0.7584, + "step": 580 + }, + { + "epoch": 0.7190737355271176, + "grad_norm": 0.25114378333091736, + "learning_rate": 5e-05, + "loss": 0.7625, + "step": 590 + }, + { + "epoch": 0.7312614259597806, + "grad_norm": 0.19145755469799042, + "learning_rate": 5e-05, + "loss": 0.7577, + "step": 600 + }, + { + "epoch": 0.7434491163924436, + "grad_norm": 0.19129066169261932, + "learning_rate": 5e-05, + "loss": 0.7772, + "step": 610 + } + ], + "logging_steps": 10, + "max_steps": 2463, + "num_input_tokens_seen": 0, + "num_train_epochs": 3, + "save_steps": 205, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 2.5207491367524106e+18, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-615/training_args.bin b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-615/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..23ae0ec8a4226be5457f61c137abab2b2c10a0cf --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-615/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56896ea21940e33d7e2934267dff2a5b4a4a80195a0f165245f6617a0f88b90b +size 7377 diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-615/zero_to_fp32.py b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-615/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..5995d6e6f04e43b989587aa9022a3aef0c66d694 --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-615/zero_to_fp32.py @@ -0,0 +1,760 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: +# python zero_to_fp32.py . output_dir/ +# or +# python zero_to_fp32.py . output_dir/ --safe_serialization + +import argparse +import torch +import glob +import math +import os +import re +import gc +import json +import numpy as np +from tqdm import tqdm +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device, weights_only=False) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + total_files = len(files) + state_dicts = [] + for f in tqdm(files, desc='Loading checkpoint shards'): + state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + model_files = get_model_state_files(ds_checkpoint_dir) + + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +class GatheredTensor: + """ + A pseudo tensor that collects partitioned weights. + It is more memory efficient when there are multiple groups. + """ + + def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape): + self.flat_groups = flat_groups + self.flat_groups_offset = flat_groups_offset + self.offset = offset + self.partitioned_numel = partitioned_numel + self.shape = shape + self.dtype = self.flat_groups[0][0].dtype + + def contiguous(self): + """ + Merge partitioned weights from flat_groups into a single tensor. + """ + end_idx = self.offset + self.partitioned_numel + world_size = len(self.flat_groups) + pad_flat_param_chunks = [] + + for rank_i in range(world_size): + # for each rank, we need to collect weights from related group/groups + flat_groups_at_rank_i = self.flat_groups[rank_i] + start_group_id = None + end_group_id = None + for group_id in range(len(self.flat_groups_offset)): + if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]: + start_group_id = group_id + if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]: + end_group_id = group_id + break + # collect weights from related group/groups + for group_id in range(start_group_id, end_group_id + 1): + flat_tensor = flat_groups_at_rank_i[group_id] + start_offset = self.offset - self.flat_groups_offset[group_id] + end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id] + pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset]) + + # collect weights from all ranks + pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0) + param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous() + return param + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size + + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]])) + for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'): + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # memory efficient tensor + tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape) + state_dict[name] = tensor + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def to_torch_tensor(state_dict, return_empty_tensor=False): + """ + Convert state_dict of GatheredTensor to torch tensor + """ + torch_state_dict = {} + converted_tensors = {} + for name, tensor in state_dict.items(): + tensor_id = id(tensor) + if tensor_id in converted_tensors: # shared tensors + shared_tensor = torch_state_dict[converted_tensors[tensor_id]] + torch_state_dict[name] = shared_tensor + else: + converted_tensors[tensor_id] = name + if return_empty_tensor: + torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype) + else: + torch_state_dict[name] = tensor.contiguous() + return torch_state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag=None, + exclude_frozen_parameters=False, + lazy_mode=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient. + Convert the pesduo tensor to torch tensor by ``.contiguous()`` + + Returns: + - pytorch ``state_dict`` + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + Note: the above usage may not work if your application doesn't have sufficient free CPU memory. + You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. Or you can load state_dict in lazy mode :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu + for name, lazy_tensor in state_dict.item(): + tensor = lazy_tensor.contiguous() # to cpu + print(name, tensor) + # del tensor to release memory if it no longer in use + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + if lazy_mode: + return state_dict + else: + return to_torch_tensor(state_dict) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, + output_dir, + max_shard_size="5GB", + safe_serialization=False, + tag=None, + exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_dir``: directory to the pytorch fp32 state_dict output files + - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB + - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + + # Dependency pre-check + if safe_serialization: + try: + from safetensors.torch import save_file + except ImportError: + print('If you want to use `safe_serialization`, please `pip install safetensors`') + raise + if max_shard_size is not None: + try: + from huggingface_hub import split_torch_state_dict_into_shards + except ImportError: + print('If you want to use `max_shard_size`, please `pip install huggingface_hub`') + raise + + # Convert zero checkpoint to state_dict + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag, + exclude_frozen_parameters, + lazy_mode=True) + + # Shard the model if it is too big. + weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin" + if max_shard_size is not None: + filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors") + # an memory-efficient approach for sharding + empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True) + state_dict_split = split_torch_state_dict_into_shards(empty_state_dict, + filename_pattern=filename_pattern, + max_shard_size=max_shard_size) + else: + from collections import namedtuple + StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"]) + state_dict_split = StateDictSplit(is_sharded=False, + filename_to_tensors={weights_name: list(state_dict.keys())}) + + # Save the model by shard + os.makedirs(output_dir, exist_ok=True) + filename_to_tensors = state_dict_split.filename_to_tensors.items() + for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"): + shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors} + shard_state_dict = to_torch_tensor(shard_state_dict) + output_path = os.path.join(output_dir, shard_file) + if safe_serialization: + save_file(shard_state_dict, output_path, metadata={"format": "pt"}) + else: + torch.save(shard_state_dict, output_path) + # release the memory of current shard + for tensor_name in list(shard_state_dict.keys()): + del state_dict[tensor_name] + del shard_state_dict[tensor_name] + del shard_state_dict + gc.collect() + + # Save index if sharded + if state_dict_split.is_sharded: + index = { + "metadata": state_dict_split.metadata, + "weight_map": state_dict_split.tensor_to_filename, + } + save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json" + save_index_file = os.path.join(output_dir, save_index_file) + with open(save_index_file, "w", encoding="utf-8") as f: + content = json.dumps(index, indent=2, sort_keys=True) + "\n" + f.write(content) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info("Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info("Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument("output_dir", + type=str, + help="directory to the pytorch fp32 state_dict output files" + "(e.g. path/checkpoint-12-output/)") + parser.add_argument( + "--max_shard_size", + type=str, + default="5GB", + help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size" + "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`" + "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances" + "without CPU OOM issues.") + parser.add_argument( + "--safe_serialization", + default=False, + action='store_true', + help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_dir, + max_shard_size=args.max_shard_size, + safe_serialization=args.safe_serialization, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-820/config.json b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-820/config.json new file mode 100644 index 0000000000000000000000000000000000000000..18bd58d561964a16b60687f1b04aefdfed0b9b20 --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-820/config.json @@ -0,0 +1,39 @@ +{ + "architectures": [ + "LlamaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "dtype": "bfloat16", + "eos_token_id": [ + 128001, + 128008, + 128009 + ], + "head_dim": 64, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 8192, + "max_position_embeddings": 131072, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 32, + "num_hidden_layers": 16, + "num_key_value_heads": 8, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": { + "factor": 32.0, + "high_freq_factor": 4.0, + "low_freq_factor": 1.0, + "original_max_position_embeddings": 8192, + "rope_type": "llama3" + }, + "rope_theta": 500000.0, + "tie_word_embeddings": true, + "transformers_version": "4.57.1", + "use_cache": false, + "vocab_size": 128256 +} diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-820/generation_config.json b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-820/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..125cc74759bc705ab8a0c48d07d7eac083f56011 --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-820/generation_config.json @@ -0,0 +1,12 @@ +{ + "bos_token_id": 128000, + "do_sample": true, + "eos_token_id": [ + 128001, + 128008, + 128009 + ], + "temperature": 0.6, + "top_p": 0.9, + "transformers_version": "4.57.1" +} diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-820/latest b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-820/latest new file mode 100644 index 0000000000000000000000000000000000000000..026d35acfc74da9ae339bd105301986a135e3c2d --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-820/latest @@ -0,0 +1 @@ +global_step820 \ No newline at end of file diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-820/model.safetensors b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-820/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e6c93f3c1ec02f7840c30e62e15c59bd22c301a5 --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-820/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4df606fd9001d31c4fc2a9e2f704a4e542b39cc099e06624a4c4c999201e9bed +size 2996982344 diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-820/scheduler.pt b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-820/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..54069d03174715d523bd6da940d9c2b3221a8a77 --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-820/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:23ef1f12530afb9136c938d977e57c45052b40c0b9edb55e2104067ed7cddad9 +size 1465 diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-820/trainer_state.json b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-820/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..f4b303c256fbab5706cfb4090c6fa0f4acdad420 --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-820/trainer_state.json @@ -0,0 +1,608 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.9993906154783668, + "eval_steps": 500, + "global_step": 820, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.01218769043266301, + "grad_norm": 0.36486390233039856, + "learning_rate": 5e-05, + "loss": 1.2213, + "step": 10 + }, + { + "epoch": 0.02437538086532602, + "grad_norm": 0.2953564524650574, + "learning_rate": 5e-05, + "loss": 1.0223, + "step": 20 + }, + { + "epoch": 0.03656307129798903, + "grad_norm": 0.2766735851764679, + "learning_rate": 5e-05, + "loss": 0.9589, + "step": 30 + }, + { + "epoch": 0.04875076173065204, + "grad_norm": 0.32770997285842896, + "learning_rate": 5e-05, + "loss": 0.9081, + "step": 40 + }, + { + "epoch": 0.06093845216331505, + "grad_norm": 0.3052276372909546, + "learning_rate": 5e-05, + "loss": 0.8752, + "step": 50 + }, + { + "epoch": 0.07312614259597806, + "grad_norm": 0.240146666765213, + "learning_rate": 5e-05, + "loss": 0.873, + "step": 60 + }, + { + "epoch": 0.08531383302864107, + "grad_norm": 0.23872187733650208, + "learning_rate": 5e-05, + "loss": 0.8587, + "step": 70 + }, + { + "epoch": 0.09750152346130408, + "grad_norm": 0.32638487219810486, + "learning_rate": 5e-05, + "loss": 0.8656, + "step": 80 + }, + { + "epoch": 0.10968921389396709, + "grad_norm": 0.25019028782844543, + "learning_rate": 5e-05, + "loss": 0.8685, + "step": 90 + }, + { + "epoch": 0.1218769043266301, + "grad_norm": 0.2507675886154175, + "learning_rate": 5e-05, + "loss": 0.8662, + "step": 100 + }, + { + "epoch": 0.1340645947592931, + "grad_norm": 0.27966755628585815, + "learning_rate": 5e-05, + "loss": 0.8428, + "step": 110 + }, + { + "epoch": 0.14625228519195613, + "grad_norm": 0.2622673213481903, + "learning_rate": 5e-05, + "loss": 0.8416, + "step": 120 + }, + { + "epoch": 0.15843997562461914, + "grad_norm": 0.3003966510295868, + "learning_rate": 5e-05, + "loss": 0.8528, + "step": 130 + }, + { + "epoch": 0.17062766605728213, + "grad_norm": 0.32308319211006165, + "learning_rate": 5e-05, + "loss": 0.8357, + "step": 140 + }, + { + "epoch": 0.18281535648994515, + "grad_norm": 0.27279096841812134, + "learning_rate": 5e-05, + "loss": 0.8432, + "step": 150 + }, + { + "epoch": 0.19500304692260817, + "grad_norm": 0.27976104617118835, + "learning_rate": 5e-05, + "loss": 0.8119, + "step": 160 + }, + { + "epoch": 0.2071907373552712, + "grad_norm": 0.290866881608963, + "learning_rate": 5e-05, + "loss": 0.8096, + "step": 170 + }, + { + "epoch": 0.21937842778793418, + "grad_norm": 0.29430949687957764, + "learning_rate": 5e-05, + "loss": 0.819, + "step": 180 + }, + { + "epoch": 0.2315661182205972, + "grad_norm": 0.27714136242866516, + "learning_rate": 5e-05, + "loss": 0.8187, + "step": 190 + }, + { + "epoch": 0.2437538086532602, + "grad_norm": 0.2997029423713684, + "learning_rate": 5e-05, + "loss": 0.808, + "step": 200 + }, + { + "epoch": 0.25594149908592323, + "grad_norm": 0.2539573609828949, + "learning_rate": 5e-05, + "loss": 0.7943, + "step": 210 + }, + { + "epoch": 0.2681291895185862, + "grad_norm": 0.23294171690940857, + "learning_rate": 5e-05, + "loss": 0.8188, + "step": 220 + }, + { + "epoch": 0.28031687995124926, + "grad_norm": 0.25016292929649353, + "learning_rate": 5e-05, + "loss": 0.8035, + "step": 230 + }, + { + "epoch": 0.29250457038391225, + "grad_norm": 0.37125322222709656, + "learning_rate": 5e-05, + "loss": 0.785, + "step": 240 + }, + { + "epoch": 0.30469226081657524, + "grad_norm": 0.21520452201366425, + "learning_rate": 5e-05, + "loss": 0.8088, + "step": 250 + }, + { + "epoch": 0.3168799512492383, + "grad_norm": 0.20990602672100067, + "learning_rate": 5e-05, + "loss": 0.7813, + "step": 260 + }, + { + "epoch": 0.3290676416819013, + "grad_norm": 0.24193961918354034, + "learning_rate": 5e-05, + "loss": 0.7979, + "step": 270 + }, + { + "epoch": 0.34125533211456427, + "grad_norm": 0.22766050696372986, + "learning_rate": 5e-05, + "loss": 0.785, + "step": 280 + }, + { + "epoch": 0.3534430225472273, + "grad_norm": 0.2662702202796936, + "learning_rate": 5e-05, + "loss": 0.7766, + "step": 290 + }, + { + "epoch": 0.3656307129798903, + "grad_norm": 0.23020495474338531, + "learning_rate": 5e-05, + "loss": 0.7687, + "step": 300 + }, + { + "epoch": 0.3778184034125533, + "grad_norm": 0.21770520508289337, + "learning_rate": 5e-05, + "loss": 0.7815, + "step": 310 + }, + { + "epoch": 0.39000609384521634, + "grad_norm": 0.22677403688430786, + "learning_rate": 5e-05, + "loss": 0.7999, + "step": 320 + }, + { + "epoch": 0.40219378427787933, + "grad_norm": 0.24891860783100128, + "learning_rate": 5e-05, + "loss": 0.7946, + "step": 330 + }, + { + "epoch": 0.4143814747105424, + "grad_norm": 0.2506854832172394, + "learning_rate": 5e-05, + "loss": 0.7889, + "step": 340 + }, + { + "epoch": 0.42656916514320536, + "grad_norm": 0.24388180673122406, + "learning_rate": 5e-05, + "loss": 0.79, + "step": 350 + }, + { + "epoch": 0.43875685557586835, + "grad_norm": 0.22329650819301605, + "learning_rate": 5e-05, + "loss": 0.7632, + "step": 360 + }, + { + "epoch": 0.4509445460085314, + "grad_norm": 0.2240927815437317, + "learning_rate": 5e-05, + "loss": 0.8042, + "step": 370 + }, + { + "epoch": 0.4631322364411944, + "grad_norm": 0.2578203082084656, + "learning_rate": 5e-05, + "loss": 0.7667, + "step": 380 + }, + { + "epoch": 0.4753199268738574, + "grad_norm": 0.25349217653274536, + "learning_rate": 5e-05, + "loss": 0.7911, + "step": 390 + }, + { + "epoch": 0.4875076173065204, + "grad_norm": 0.254446804523468, + "learning_rate": 5e-05, + "loss": 0.7909, + "step": 400 + }, + { + "epoch": 0.4996953077391834, + "grad_norm": 0.24253171682357788, + "learning_rate": 5e-05, + "loss": 0.7896, + "step": 410 + }, + { + "epoch": 0.5118829981718465, + "grad_norm": 0.27901458740234375, + "learning_rate": 5e-05, + "loss": 0.7828, + "step": 420 + }, + { + "epoch": 0.5240706886045094, + "grad_norm": 0.31884434819221497, + "learning_rate": 5e-05, + "loss": 0.7919, + "step": 430 + }, + { + "epoch": 0.5362583790371724, + "grad_norm": 0.22920799255371094, + "learning_rate": 5e-05, + "loss": 0.7777, + "step": 440 + }, + { + "epoch": 0.5484460694698354, + "grad_norm": 0.22606618702411652, + "learning_rate": 5e-05, + "loss": 0.7661, + "step": 450 + }, + { + "epoch": 0.5606337599024985, + "grad_norm": 0.21604052186012268, + "learning_rate": 5e-05, + "loss": 0.769, + "step": 460 + }, + { + "epoch": 0.5728214503351615, + "grad_norm": 0.2296101152896881, + "learning_rate": 5e-05, + "loss": 0.7626, + "step": 470 + }, + { + "epoch": 0.5850091407678245, + "grad_norm": 0.24162150919437408, + "learning_rate": 5e-05, + "loss": 0.7831, + "step": 480 + }, + { + "epoch": 0.5971968312004875, + "grad_norm": 0.22364799678325653, + "learning_rate": 5e-05, + "loss": 0.7737, + "step": 490 + }, + { + "epoch": 0.6093845216331505, + "grad_norm": 0.23953403532505035, + "learning_rate": 5e-05, + "loss": 0.787, + "step": 500 + }, + { + "epoch": 0.6215722120658135, + "grad_norm": 0.22349311411380768, + "learning_rate": 5e-05, + "loss": 0.7763, + "step": 510 + }, + { + "epoch": 0.6337599024984766, + "grad_norm": 0.21996377408504486, + "learning_rate": 5e-05, + "loss": 0.7596, + "step": 520 + }, + { + "epoch": 0.6459475929311396, + "grad_norm": 0.245538130402565, + "learning_rate": 5e-05, + "loss": 0.7785, + "step": 530 + }, + { + "epoch": 0.6581352833638026, + "grad_norm": 0.23442520201206207, + "learning_rate": 5e-05, + "loss": 0.7633, + "step": 540 + }, + { + "epoch": 0.6703229737964655, + "grad_norm": 0.22274106740951538, + "learning_rate": 5e-05, + "loss": 0.7756, + "step": 550 + }, + { + "epoch": 0.6825106642291285, + "grad_norm": 0.21717530488967896, + "learning_rate": 5e-05, + "loss": 0.7726, + "step": 560 + }, + { + "epoch": 0.6946983546617916, + "grad_norm": 0.2465955913066864, + "learning_rate": 5e-05, + "loss": 0.7476, + "step": 570 + }, + { + "epoch": 0.7068860450944546, + "grad_norm": 0.2318575084209442, + "learning_rate": 5e-05, + "loss": 0.7584, + "step": 580 + }, + { + "epoch": 0.7190737355271176, + "grad_norm": 0.25114378333091736, + "learning_rate": 5e-05, + "loss": 0.7625, + "step": 590 + }, + { + "epoch": 0.7312614259597806, + "grad_norm": 0.19145755469799042, + "learning_rate": 5e-05, + "loss": 0.7577, + "step": 600 + }, + { + "epoch": 0.7434491163924436, + "grad_norm": 0.19129066169261932, + "learning_rate": 5e-05, + "loss": 0.7772, + "step": 610 + }, + { + "epoch": 0.7556368068251066, + "grad_norm": 0.2014475166797638, + "learning_rate": 5e-05, + "loss": 0.7601, + "step": 620 + }, + { + "epoch": 0.7678244972577697, + "grad_norm": 0.21480509638786316, + "learning_rate": 5e-05, + "loss": 0.7665, + "step": 630 + }, + { + "epoch": 0.7800121876904327, + "grad_norm": 0.2633228302001953, + "learning_rate": 5e-05, + "loss": 0.7565, + "step": 640 + }, + { + "epoch": 0.7921998781230957, + "grad_norm": 0.24633203446865082, + "learning_rate": 5e-05, + "loss": 0.7602, + "step": 650 + }, + { + "epoch": 0.8043875685557587, + "grad_norm": 0.19770534336566925, + "learning_rate": 5e-05, + "loss": 0.7685, + "step": 660 + }, + { + "epoch": 0.8165752589884216, + "grad_norm": 0.2267930805683136, + "learning_rate": 5e-05, + "loss": 0.7516, + "step": 670 + }, + { + "epoch": 0.8287629494210847, + "grad_norm": 0.238111674785614, + "learning_rate": 5e-05, + "loss": 0.7487, + "step": 680 + }, + { + "epoch": 0.8409506398537477, + "grad_norm": 0.19528904557228088, + "learning_rate": 5e-05, + "loss": 0.7467, + "step": 690 + }, + { + "epoch": 0.8531383302864107, + "grad_norm": 0.21305914223194122, + "learning_rate": 5e-05, + "loss": 0.7527, + "step": 700 + }, + { + "epoch": 0.8653260207190737, + "grad_norm": 0.2205205112695694, + "learning_rate": 5e-05, + "loss": 0.7639, + "step": 710 + }, + { + "epoch": 0.8775137111517367, + "grad_norm": 0.22429127991199493, + "learning_rate": 5e-05, + "loss": 0.7744, + "step": 720 + }, + { + "epoch": 0.8897014015843998, + "grad_norm": 0.23755772411823273, + "learning_rate": 5e-05, + "loss": 0.7446, + "step": 730 + }, + { + "epoch": 0.9018890920170628, + "grad_norm": 0.17592693865299225, + "learning_rate": 5e-05, + "loss": 0.7442, + "step": 740 + }, + { + "epoch": 0.9140767824497258, + "grad_norm": 0.2266898900270462, + "learning_rate": 5e-05, + "loss": 0.7326, + "step": 750 + }, + { + "epoch": 0.9262644728823888, + "grad_norm": 0.23541569709777832, + "learning_rate": 5e-05, + "loss": 0.7338, + "step": 760 + }, + { + "epoch": 0.9384521633150518, + "grad_norm": 0.22233888506889343, + "learning_rate": 5e-05, + "loss": 0.7385, + "step": 770 + }, + { + "epoch": 0.9506398537477148, + "grad_norm": 0.23714767396450043, + "learning_rate": 5e-05, + "loss": 0.7306, + "step": 780 + }, + { + "epoch": 0.9628275441803779, + "grad_norm": 0.21191060543060303, + "learning_rate": 5e-05, + "loss": 0.736, + "step": 790 + }, + { + "epoch": 0.9750152346130408, + "grad_norm": 0.22759106755256653, + "learning_rate": 5e-05, + "loss": 0.7505, + "step": 800 + }, + { + "epoch": 0.9872029250457038, + "grad_norm": 0.19895903766155243, + "learning_rate": 5e-05, + "loss": 0.7536, + "step": 810 + }, + { + "epoch": 0.9993906154783668, + "grad_norm": 0.3061685860157013, + "learning_rate": 5e-05, + "loss": 0.7493, + "step": 820 + } + ], + "logging_steps": 10, + "max_steps": 2463, + "num_input_tokens_seen": 0, + "num_train_epochs": 3, + "save_steps": 205, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 3.365008964046553e+18, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-820/training_args.bin b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-820/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..23ae0ec8a4226be5457f61c137abab2b2c10a0cf --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-820/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56896ea21940e33d7e2934267dff2a5b4a4a80195a0f165245f6617a0f88b90b +size 7377 diff --git a/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-820/zero_to_fp32.py b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-820/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..5995d6e6f04e43b989587aa9022a3aef0c66d694 --- /dev/null +++ b/openai_gpt_oss_120b_Qwen_Qwen3_235B_A22B_fp8_tput_deepseek_ai_DeepSeek_R1_Distill_Qwen_14B/checkpoint-820/zero_to_fp32.py @@ -0,0 +1,760 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: +# python zero_to_fp32.py . output_dir/ +# or +# python zero_to_fp32.py . output_dir/ --safe_serialization + +import argparse +import torch +import glob +import math +import os +import re +import gc +import json +import numpy as np +from tqdm import tqdm +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device, weights_only=False) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + total_files = len(files) + state_dicts = [] + for f in tqdm(files, desc='Loading checkpoint shards'): + state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + model_files = get_model_state_files(ds_checkpoint_dir) + + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +class GatheredTensor: + """ + A pseudo tensor that collects partitioned weights. + It is more memory efficient when there are multiple groups. + """ + + def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape): + self.flat_groups = flat_groups + self.flat_groups_offset = flat_groups_offset + self.offset = offset + self.partitioned_numel = partitioned_numel + self.shape = shape + self.dtype = self.flat_groups[0][0].dtype + + def contiguous(self): + """ + Merge partitioned weights from flat_groups into a single tensor. + """ + end_idx = self.offset + self.partitioned_numel + world_size = len(self.flat_groups) + pad_flat_param_chunks = [] + + for rank_i in range(world_size): + # for each rank, we need to collect weights from related group/groups + flat_groups_at_rank_i = self.flat_groups[rank_i] + start_group_id = None + end_group_id = None + for group_id in range(len(self.flat_groups_offset)): + if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]: + start_group_id = group_id + if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]: + end_group_id = group_id + break + # collect weights from related group/groups + for group_id in range(start_group_id, end_group_id + 1): + flat_tensor = flat_groups_at_rank_i[group_id] + start_offset = self.offset - self.flat_groups_offset[group_id] + end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id] + pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset]) + + # collect weights from all ranks + pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0) + param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous() + return param + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size + + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]])) + for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'): + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # memory efficient tensor + tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape) + state_dict[name] = tensor + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def to_torch_tensor(state_dict, return_empty_tensor=False): + """ + Convert state_dict of GatheredTensor to torch tensor + """ + torch_state_dict = {} + converted_tensors = {} + for name, tensor in state_dict.items(): + tensor_id = id(tensor) + if tensor_id in converted_tensors: # shared tensors + shared_tensor = torch_state_dict[converted_tensors[tensor_id]] + torch_state_dict[name] = shared_tensor + else: + converted_tensors[tensor_id] = name + if return_empty_tensor: + torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype) + else: + torch_state_dict[name] = tensor.contiguous() + return torch_state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag=None, + exclude_frozen_parameters=False, + lazy_mode=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient. + Convert the pesduo tensor to torch tensor by ``.contiguous()`` + + Returns: + - pytorch ``state_dict`` + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + Note: the above usage may not work if your application doesn't have sufficient free CPU memory. + You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. Or you can load state_dict in lazy mode :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu + for name, lazy_tensor in state_dict.item(): + tensor = lazy_tensor.contiguous() # to cpu + print(name, tensor) + # del tensor to release memory if it no longer in use + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + if lazy_mode: + return state_dict + else: + return to_torch_tensor(state_dict) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, + output_dir, + max_shard_size="5GB", + safe_serialization=False, + tag=None, + exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_dir``: directory to the pytorch fp32 state_dict output files + - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB + - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + + # Dependency pre-check + if safe_serialization: + try: + from safetensors.torch import save_file + except ImportError: + print('If you want to use `safe_serialization`, please `pip install safetensors`') + raise + if max_shard_size is not None: + try: + from huggingface_hub import split_torch_state_dict_into_shards + except ImportError: + print('If you want to use `max_shard_size`, please `pip install huggingface_hub`') + raise + + # Convert zero checkpoint to state_dict + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag, + exclude_frozen_parameters, + lazy_mode=True) + + # Shard the model if it is too big. + weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin" + if max_shard_size is not None: + filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors") + # an memory-efficient approach for sharding + empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True) + state_dict_split = split_torch_state_dict_into_shards(empty_state_dict, + filename_pattern=filename_pattern, + max_shard_size=max_shard_size) + else: + from collections import namedtuple + StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"]) + state_dict_split = StateDictSplit(is_sharded=False, + filename_to_tensors={weights_name: list(state_dict.keys())}) + + # Save the model by shard + os.makedirs(output_dir, exist_ok=True) + filename_to_tensors = state_dict_split.filename_to_tensors.items() + for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"): + shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors} + shard_state_dict = to_torch_tensor(shard_state_dict) + output_path = os.path.join(output_dir, shard_file) + if safe_serialization: + save_file(shard_state_dict, output_path, metadata={"format": "pt"}) + else: + torch.save(shard_state_dict, output_path) + # release the memory of current shard + for tensor_name in list(shard_state_dict.keys()): + del state_dict[tensor_name] + del shard_state_dict[tensor_name] + del shard_state_dict + gc.collect() + + # Save index if sharded + if state_dict_split.is_sharded: + index = { + "metadata": state_dict_split.metadata, + "weight_map": state_dict_split.tensor_to_filename, + } + save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json" + save_index_file = os.path.join(output_dir, save_index_file) + with open(save_index_file, "w", encoding="utf-8") as f: + content = json.dumps(index, indent=2, sort_keys=True) + "\n" + f.write(content) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info("Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info("Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument("output_dir", + type=str, + help="directory to the pytorch fp32 state_dict output files" + "(e.g. path/checkpoint-12-output/)") + parser.add_argument( + "--max_shard_size", + type=str, + default="5GB", + help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size" + "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`" + "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances" + "without CPU OOM issues.") + parser.add_argument( + "--safe_serialization", + default=False, + action='store_true', + help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_dir, + max_shard_size=args.max_shard_size, + safe_serialization=args.safe_serialization, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters)