diff --git a/.cache_old/huggingface/upload/checkpoint_26/26/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt.lock b/.cache_old/huggingface/upload/checkpoint_26/26/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt.lock new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/.cache_old/huggingface/upload/checkpoint_26/26/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt.metadata b/.cache_old/huggingface/upload/checkpoint_26/26/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt.metadata new file mode 100644 index 0000000000000000000000000000000000000000..5c8c1a8fe95d77abb0e9185a42693c5a9eba253d --- /dev/null +++ b/.cache_old/huggingface/upload/checkpoint_26/26/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt.metadata @@ -0,0 +1,7 @@ +1741207632.1092665 +4653767351 +0 +fae3140d17ce4f82c7a0c4eb77a18c45e61155058559180d6c475b20aef5bf49 +lfs +1 +1 diff --git a/.cache_old/huggingface/upload/checkpoint_26/26/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt.lock b/.cache_old/huggingface/upload/checkpoint_26/26/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt.lock new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt.lock b/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt.lock new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt.metadata b/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt.metadata new file mode 100644 index 0000000000000000000000000000000000000000..18f116ac731d7000ae67ba8eb924432487016755 --- /dev/null +++ b/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt.metadata @@ -0,0 +1,7 @@ +1741207632.2765524 +4653763632 +0 +ba052d008b60bda98d01ddc1f8e212471756f6a555d9b2dd26a2f75f71cda44a +lfs +1 +1 diff --git a/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt.lock b/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt.lock new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt.metadata b/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt.metadata new file mode 100644 index 0000000000000000000000000000000000000000..68460422abaffa9ef0da7098df00ed2e4305bbed --- /dev/null +++ b/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt.metadata @@ -0,0 +1,7 @@ +1741207632.2595308 +4653763255 +0 +08b956fd952afe729daddf1f7b8e898cea441406ec0e2f8d9e134883b8ecc343 +lfs +1 +1 diff --git a/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt.lock b/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt.lock new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt.metadata b/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt.metadata new file mode 100644 index 0000000000000000000000000000000000000000..6f1117eee0ff93b487336fcf6c0bcafef5e1c436 --- /dev/null +++ b/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt.metadata @@ -0,0 +1,7 @@ +1741207632.2798781 +4653767415 +0 +c0e8e90dc4dd6a3f82d79d5e9f0f28ec6e2790ab70476855c7870ea324ab1139 +lfs +1 +1 diff --git a/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt.lock b/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt.lock new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt.metadata b/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt.metadata new file mode 100644 index 0000000000000000000000000000000000000000..8eb08a859b9bdf226e7176c970b55d7c0f397b10 --- /dev/null +++ b/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt.metadata @@ -0,0 +1,7 @@ +1741207632.2172365 +4653767415 +0 +d2871a33d5b7f9a2dad2a7e6bc69a10205c420b3db5502d3f5ad4f5e41774762 +lfs +1 +1 diff --git a/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt.lock b/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt.lock new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt.metadata b/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt.metadata new file mode 100644 index 0000000000000000000000000000000000000000..946e64d6f2814afa67d6561109f749a5cb31a066 --- /dev/null +++ b/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt.metadata @@ -0,0 +1,7 @@ +1741207632.2502685 +4653767287 +0 +d48d6a6ee87dd7778c5db49d34c16ea9ebd7eab872abc2f9906909e21db11c9a +lfs +1 +1 diff --git a/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt.lock b/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt.lock new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt.metadata b/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt.metadata new file mode 100644 index 0000000000000000000000000000000000000000..29126e456b15445c952c7c73373288fd06dc9d7a --- /dev/null +++ b/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt.metadata @@ -0,0 +1,7 @@ +1741207632.2649295 +4653767351 +0 +1f7429f06bec9109fcc46f5b185e3ce486ceeae05699d47ae8fa7d22950febaf +lfs +1 +1 diff --git a/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt.lock b/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt.lock new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt.metadata b/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt.metadata new file mode 100644 index 0000000000000000000000000000000000000000..8953e73f5a9ab4ffbbf48a9f50ee07647c450ee3 --- /dev/null +++ b/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt.metadata @@ -0,0 +1,7 @@ +1741207632.302208 +4653767351 +0 +86263705905e35f34d9361e694744155252330a8a95991cf49b0b8b4b312f3ba +lfs +1 +1 diff --git a/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt.lock b/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt.lock new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt.metadata b/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt.metadata new file mode 100644 index 0000000000000000000000000000000000000000..6911600d2f99dc3599a4db8f52cc0adae32f5912 --- /dev/null +++ b/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt.metadata @@ -0,0 +1,7 @@ +1741207632.213711 +4653767351 +0 +acae12738d59b09c697d2d52e865af17238a358072132c8a02922ebf65aba8bd +lfs +1 +1 diff --git a/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt.lock b/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt.lock new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt.metadata b/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt.metadata new file mode 100644 index 0000000000000000000000000000000000000000..36949ec0c9a014f709268168adf45f92ff6d709a --- /dev/null +++ b/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt.metadata @@ -0,0 +1,7 @@ +1741207632.2204654 +4653767543 +0 +3f2bb8ec9e972ea8900bb66c5bd17226c905ff094fd7341e7e9d91d64b02dc19 +lfs +1 +1 diff --git a/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt.lock b/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt.lock new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt.metadata b/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt.metadata new file mode 100644 index 0000000000000000000000000000000000000000..064502df73b2716cf9f8a6d8e2bb46e8e0552538 --- /dev/null +++ b/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt.metadata @@ -0,0 +1,7 @@ +1741207632.2958696 +4653767351 +0 +bc54d9d4c68c4133b399a6201f25040ab655817e252ddc47e1d7fffbe04414d3 +lfs +1 +1 diff --git a/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt.lock b/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt.lock new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt.metadata b/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt.metadata new file mode 100644 index 0000000000000000000000000000000000000000..5f5fe631efd7538ac6d5734967743e0113c892c0 --- /dev/null +++ b/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt.metadata @@ -0,0 +1,7 @@ +1741207632.2101996 +4653767351 +0 +ebedc947914285acc239c0101fdcd862e6416fa79be96389da8668a2297d0d1b +lfs +1 +1 diff --git a/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt.lock b/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt.lock new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt.metadata b/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt.metadata new file mode 100644 index 0000000000000000000000000000000000000000..6fb1bff69a74c636a2daa88c92061d060dd05c5b --- /dev/null +++ b/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt.metadata @@ -0,0 +1,7 @@ +1741207632.121467 +4653766128 +0 +b6b9242619c015dbee6407fdf764035a2c88bdfe75eb9251e3063af0b55bedb3 +lfs +1 +1 diff --git a/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt.lock b/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt.lock new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt.metadata b/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt.metadata new file mode 100644 index 0000000000000000000000000000000000000000..c0c1bcdb0d79870174b7cfd1b480e9d402a60d1c --- /dev/null +++ b/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt.metadata @@ -0,0 +1,7 @@ +1741207632.255535 +4653767351 +0 +11da330fc5c1b1aa5c59c3d2e7a5126c66c086e3c623e987098a94fb47bcab9e +lfs +1 +1 diff --git a/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt.lock b/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt.lock new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt.metadata b/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt.metadata new file mode 100644 index 0000000000000000000000000000000000000000..2b90e9e44db8098fbaf781797ef55f7d58fc444a --- /dev/null +++ b/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt.metadata @@ -0,0 +1,7 @@ +1741207632.2896628 +4653767351 +0 +5595734924a7d1821bd88b0f1eb0e99111236901af0e8ba67d6b1bc3fcd80fcf +lfs +1 +1 diff --git a/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt.lock b/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt.lock new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt.metadata b/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt.metadata new file mode 100644 index 0000000000000000000000000000000000000000..bdae478d40036b689f5252f29851e986a17c7f47 --- /dev/null +++ b/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt.metadata @@ -0,0 +1,7 @@ +1741207632.2064705 +4653767927 +0 +5c92573d4b5afd68e670bbd30f9eec1b645b1e488b281c8e4383a48368101bbb +lfs +1 +1 diff --git a/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt.lock b/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt.lock new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt.metadata b/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt.metadata new file mode 100644 index 0000000000000000000000000000000000000000..39bbca8bb468913c5125443b757dfc807be5a1d7 --- /dev/null +++ b/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt.metadata @@ -0,0 +1,7 @@ +1741207632.29267 +4653765239 +0 +57402b9bc1c61bb58d4788ca4af955d6a6da00bdff07d4887069d63a6bc8864f +lfs +1 +1 diff --git a/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt.lock b/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt.lock new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt.metadata b/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt.metadata new file mode 100644 index 0000000000000000000000000000000000000000..ba4bdd54ba1573c293d6c08d08969fe228f3c46f --- /dev/null +++ b/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt.metadata @@ -0,0 +1,7 @@ +1741207632.298999 +4653766128 +0 +4110ba426b2a7e62e7460d91a58891dea1bbcc28aa53da9f06a0a045d4254cee +lfs +1 +1 diff --git a/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt.lock b/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt.lock new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt.metadata b/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt.metadata new file mode 100644 index 0000000000000000000000000000000000000000..5e1d4a9b531c94493ac59d5dea0b559cb09545db --- /dev/null +++ b/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt.metadata @@ -0,0 +1,7 @@ +1741207632.3057327 +4653766192 +0 +b3cb2b40b7a1ae2b9ae14a84b2d477e713a3b9fd41309c9e47246214c7971233 +lfs +1 +1 diff --git a/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt.lock b/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt.lock new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt.metadata b/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt.metadata new file mode 100644 index 0000000000000000000000000000000000000000..2383a9a09fcb17fb11e9975a520c13c87c1a64d8 --- /dev/null +++ b/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt.metadata @@ -0,0 +1,7 @@ +1741207632.2835815 +4653766640 +0 +cd94ff1de7cf7fd60c7d8e57a1e986c74c126c37d9e578ea4202a6a38b5a6e53 +lfs +1 +1 diff --git a/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt.lock b/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt.lock new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt.metadata b/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt.metadata new file mode 100644 index 0000000000000000000000000000000000000000..7a97bacd684f9458133de95bd48ef6bd52e4c6bd --- /dev/null +++ b/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt.metadata @@ -0,0 +1,7 @@ +1741207632.2866957 +4653766384 +0 +593c9a5ba0dc6080e518b29ba70077fba90b3493ef749a7509178f6a37b3bd50 +lfs +1 +1 diff --git a/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt.lock b/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt.lock new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt.metadata b/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt.metadata new file mode 100644 index 0000000000000000000000000000000000000000..270e6ae3487ed1ce5ae86c40b7214096ad30b692 --- /dev/null +++ b/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt.metadata @@ -0,0 +1,7 @@ +1741207632.2688208 +4653766384 +0 +02c31e091e11ff1c984c13a2bc9e6c113a651a4a1da89c6fcad11d8a6f899b2c +lfs +1 +1 diff --git a/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt.lock b/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt.lock new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt.metadata b/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt.metadata new file mode 100644 index 0000000000000000000000000000000000000000..f281a11af0870a46566aa1071281b425aa903504 --- /dev/null +++ b/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt.metadata @@ -0,0 +1,7 @@ +1741207632.1279056 +4653766384 +0 +f5c26d4729b9722422dae303a760baf354b013f49ab1e3908d9d9dce7ed33e0e +lfs +1 +1 diff --git a/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt.lock b/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt.lock new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt.metadata b/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt.metadata new file mode 100644 index 0000000000000000000000000000000000000000..6d922cb2574d7b5d8fb58150d2994c1f8d06074c --- /dev/null +++ b/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt.metadata @@ -0,0 +1,7 @@ +1741207632.2726192 +4653766192 +0 +2e826ba2363771ebf5dc2180faefa8724360a50f22b8cb0399321e284ecfa19d +lfs +1 +1 diff --git a/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt.lock b/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt.lock new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt.metadata b/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt.metadata new file mode 100644 index 0000000000000000000000000000000000000000..3e8dc1274203f789a0c6dd467a0394675e77c7f8 --- /dev/null +++ b/.cache_old/huggingface/upload/checkpoint_30/30/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt.metadata @@ -0,0 +1,7 @@ +1741207632.2239954 +4653766384 +0 +889fdaaa529fab2cd4fcf004074c4b738c48d6cda546da4f923f112e5f739cc2 +lfs +1 +1 diff --git a/.cache_old/huggingface/upload/checkpoint_30/30/mp_rank_00_model_states.pt.lock b/.cache_old/huggingface/upload/checkpoint_30/30/mp_rank_00_model_states.pt.lock new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/.cache_old/huggingface/upload/checkpoint_30/30/mp_rank_00_model_states.pt.metadata b/.cache_old/huggingface/upload/checkpoint_30/30/mp_rank_00_model_states.pt.metadata new file mode 100644 index 0000000000000000000000000000000000000000..0fcf7a50cd550275a78d12f631c1a830056dc3d9 --- /dev/null +++ b/.cache_old/huggingface/upload/checkpoint_30/30/mp_rank_00_model_states.pt.metadata @@ -0,0 +1,7 @@ +1741207632.3208716 +18615550404 +0 +b85296c8162aa9bd07ebc55dc1f77175f4d6e3ec0d87fc259a76791bf4880fa7 +lfs +1 +1 diff --git a/.cache_old/huggingface/upload/checkpoint_30/latest.lock b/.cache_old/huggingface/upload/checkpoint_30/latest.lock new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/.cache_old/huggingface/upload/checkpoint_30/latest.metadata b/.cache_old/huggingface/upload/checkpoint_30/latest.metadata new file mode 100644 index 0000000000000000000000000000000000000000..b95f7bce64d34eb8f3fb09509d31eb8ff1e0c270 --- /dev/null +++ b/.cache_old/huggingface/upload/checkpoint_30/latest.metadata @@ -0,0 +1,7 @@ +1739336410.0340304 +2 +0 +624b60c58c9d8bfb6ff1886c2fd605d2adeb6ea4da576068201b6c6958ce93f4 +regular +0 +1 diff --git a/.cache_old/huggingface/upload/checkpoint_30/zero_to_fp32.py.lock b/.cache_old/huggingface/upload/checkpoint_30/zero_to_fp32.py.lock new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/.cache_old/huggingface/upload/checkpoint_30/zero_to_fp32.py.metadata b/.cache_old/huggingface/upload/checkpoint_30/zero_to_fp32.py.metadata new file mode 100644 index 0000000000000000000000000000000000000000..376e54cbebc1920edce5d29517b7973cf25a9e19 --- /dev/null +++ b/.cache_old/huggingface/upload/checkpoint_30/zero_to_fp32.py.metadata @@ -0,0 +1,7 @@ +1739336410.024512 +29219 +0 +92dcd9cc03d6010b4d060b73c8826d71ff9e29a030154e9f192e34ad80457f3a +regular +0 +1 diff --git a/checkpoint_15/15/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt b/checkpoint_15/15/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a8521ee0e4eb2a53ba9b44289775093e96e15849 --- /dev/null +++ b/checkpoint_15/15/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be7a8bc295a3b911513f0c56d651c29b7d4f9d6db67ba5fda0a31f087a4ebe79 +size 4653767351 diff --git a/checkpoint_15/15/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt b/checkpoint_15/15/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e46d14bb26a86f1d29477e4641607dfb8724343f --- /dev/null +++ b/checkpoint_15/15/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:965a92dcb06f87501066d9dac23abf23e7b2561d31f09f640b5b8df9c4d27fd9 +size 4653765239 diff --git a/checkpoint_15/15/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt b/checkpoint_15/15/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..537d923d5a48dad1b00228453289f5a8465973e3 --- /dev/null +++ b/checkpoint_15/15/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:00b1d4a4496c05cfb7606108f638fb307d1a2c0de7eb1f205f7946e33a53a6c6 +size 4653766384 diff --git a/checkpoint_20/latest b/checkpoint_20/latest new file mode 100644 index 0000000000000000000000000000000000000000..2edeafb09db0093bae6ff060e2dcd2166f5c9387 --- /dev/null +++ b/checkpoint_20/latest @@ -0,0 +1 @@ +20 \ No newline at end of file diff --git a/checkpoint_20/zero_to_fp32.py b/checkpoint_20/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..e69ecd9acb5a235ffbf927091051106d902b3d39 --- /dev/null +++ b/checkpoint_20/zero_to_fp32.py @@ -0,0 +1,674 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: +# python zero_to_fp32.py . output_dir/ +# or +# python zero_to_fp32.py . output_dir/ --safe_serialization + +import argparse +import torch +import glob +import math +import os +import re +import json +from tqdm import tqdm +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + total_files = len(files) + state_dicts = [] + for f in files: + state_dict = torch.load(f, map_location=device) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + if zero_stage <= 2: + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + elif zero_stage == 3: + # if there is more than one param group, there will be multiple flattened tensors - one + # flattened tensor per group - for simplicity merge them into a single tensor + # + # XXX: could make the script more memory efficient for when there are multiple groups - it + # will require matching the sub-lists of param_shapes for each param group flattened tensor + + fp32_flat_groups = [ + torch.cat(state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key], 0) for i in range(len(state_dicts)) + ] + + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + model_files = get_model_state_files(ds_checkpoint_dir) + + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = fp32_flat_groups[0].numel() * world_size + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + for name, shape in tqdm(param_shapes.items(), desc='Gathering Sharded Weights'): + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # XXX: memory usage doubles here + state_dict[name] = torch.cat( + tuple(fp32_flat_groups[i].narrow(0, offset, partitioned_numel) for i in range(world_size)), + 0).narrow(0, 0, unpartitioned_numel).view(shape) + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag=None, exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + + Returns: + - pytorch ``state_dict`` + + Note: this approach may not work if your application doesn't have sufficient free CPU memory and + you may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + return _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, + output_dir, + max_shard_size="5GB", + safe_serialization=False, + tag=None, + exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_dir``: directory to the pytorch fp32 state_dict output files + - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB + - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + # Dependency pre-check + if safe_serialization: + try: + from safetensors.torch import save_file + except ImportError: + print('If you want to use `safe_serialization`, please `pip install safetensors`') + raise + if max_shard_size is not None: + try: + from huggingface_hub import split_torch_state_dict_into_shards + except ImportError: + print('If you want to use `max_shard_size`, please `pip install huggingface_hub`') + raise + + # Convert zero checkpoint to state_dict + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag, exclude_frozen_parameters) + + # Shard the model if it is too big. + weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin" + if max_shard_size is not None: + filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors") + state_dict_split = split_torch_state_dict_into_shards(state_dict, + filename_pattern=filename_pattern, + max_shard_size=max_shard_size) + else: + from collections import namedtuple + StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"]) + state_dict_split = StateDictSplit(is_sharded=False, + filename_to_tensors={weights_name: list(state_dict.keys())}) + + # Save the model + filename_to_tensors = state_dict_split.filename_to_tensors.items() + for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"): + shard = {tensor: state_dict[tensor].contiguous() for tensor in tensors} + output_path = os.path.join(output_dir, shard_file) + if safe_serialization: + save_file(shard, output_path, metadata={"format": "pt"}) + else: + torch.save(shard, output_path) + + # Save index if sharded + if state_dict_split.is_sharded: + index = { + "metadata": state_dict_split.metadata, + "weight_map": state_dict_split.tensor_to_filename, + } + save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json" + save_index_file = os.path.join(output_dir, save_index_file) + with open(save_index_file, "w", encoding="utf-8") as f: + content = json.dumps(index, indent=2, sort_keys=True) + "\n" + f.write(content) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info(f"Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info(f"Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument("output_dir", + type=str, + help="directory to the pytorch fp32 state_dict output files" + "(e.g. path/checkpoint-12-output/)") + parser.add_argument( + "--max_shard_size", + type=str, + default="5GB", + help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size" + "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`" + "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances" + "without CPU OOM issues.") + parser.add_argument( + "--safe_serialization", + default=False, + action='store_true', + help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_dir, + max_shard_size=args.max_shard_size, + safe_serialization=args.safe_serialization, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/checkpoint_28/28/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt b/checkpoint_28/28/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..42846f4bd3f46c8af954ba59550a0b99bb5a4e5d --- /dev/null +++ b/checkpoint_28/28/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c908b78cd8a5d70ba73454676b07c94d6e1ecdf0375855f561153c4a5a230291 +size 4653767351 diff --git a/checkpoint_4/latest b/checkpoint_4/latest new file mode 100644 index 0000000000000000000000000000000000000000..bf0d87ab1b2b0ec1a11a3973d2845b42413d9767 --- /dev/null +++ b/checkpoint_4/latest @@ -0,0 +1 @@ +4 \ No newline at end of file diff --git a/checkpoint_4/zero_to_fp32.py b/checkpoint_4/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..e69ecd9acb5a235ffbf927091051106d902b3d39 --- /dev/null +++ b/checkpoint_4/zero_to_fp32.py @@ -0,0 +1,674 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: +# python zero_to_fp32.py . output_dir/ +# or +# python zero_to_fp32.py . output_dir/ --safe_serialization + +import argparse +import torch +import glob +import math +import os +import re +import json +from tqdm import tqdm +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + total_files = len(files) + state_dicts = [] + for f in files: + state_dict = torch.load(f, map_location=device) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + if zero_stage <= 2: + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + elif zero_stage == 3: + # if there is more than one param group, there will be multiple flattened tensors - one + # flattened tensor per group - for simplicity merge them into a single tensor + # + # XXX: could make the script more memory efficient for when there are multiple groups - it + # will require matching the sub-lists of param_shapes for each param group flattened tensor + + fp32_flat_groups = [ + torch.cat(state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key], 0) for i in range(len(state_dicts)) + ] + + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + model_files = get_model_state_files(ds_checkpoint_dir) + + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = fp32_flat_groups[0].numel() * world_size + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + for name, shape in tqdm(param_shapes.items(), desc='Gathering Sharded Weights'): + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # XXX: memory usage doubles here + state_dict[name] = torch.cat( + tuple(fp32_flat_groups[i].narrow(0, offset, partitioned_numel) for i in range(world_size)), + 0).narrow(0, 0, unpartitioned_numel).view(shape) + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag=None, exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + + Returns: + - pytorch ``state_dict`` + + Note: this approach may not work if your application doesn't have sufficient free CPU memory and + you may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + return _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, + output_dir, + max_shard_size="5GB", + safe_serialization=False, + tag=None, + exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_dir``: directory to the pytorch fp32 state_dict output files + - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB + - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + # Dependency pre-check + if safe_serialization: + try: + from safetensors.torch import save_file + except ImportError: + print('If you want to use `safe_serialization`, please `pip install safetensors`') + raise + if max_shard_size is not None: + try: + from huggingface_hub import split_torch_state_dict_into_shards + except ImportError: + print('If you want to use `max_shard_size`, please `pip install huggingface_hub`') + raise + + # Convert zero checkpoint to state_dict + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag, exclude_frozen_parameters) + + # Shard the model if it is too big. + weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin" + if max_shard_size is not None: + filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors") + state_dict_split = split_torch_state_dict_into_shards(state_dict, + filename_pattern=filename_pattern, + max_shard_size=max_shard_size) + else: + from collections import namedtuple + StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"]) + state_dict_split = StateDictSplit(is_sharded=False, + filename_to_tensors={weights_name: list(state_dict.keys())}) + + # Save the model + filename_to_tensors = state_dict_split.filename_to_tensors.items() + for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"): + shard = {tensor: state_dict[tensor].contiguous() for tensor in tensors} + output_path = os.path.join(output_dir, shard_file) + if safe_serialization: + save_file(shard, output_path, metadata={"format": "pt"}) + else: + torch.save(shard, output_path) + + # Save index if sharded + if state_dict_split.is_sharded: + index = { + "metadata": state_dict_split.metadata, + "weight_map": state_dict_split.tensor_to_filename, + } + save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json" + save_index_file = os.path.join(output_dir, save_index_file) + with open(save_index_file, "w", encoding="utf-8") as f: + content = json.dumps(index, indent=2, sort_keys=True) + "\n" + f.write(content) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info(f"Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info(f"Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument("output_dir", + type=str, + help="directory to the pytorch fp32 state_dict output files" + "(e.g. path/checkpoint-12-output/)") + parser.add_argument( + "--max_shard_size", + type=str, + default="5GB", + help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size" + "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`" + "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances" + "without CPU OOM issues.") + parser.add_argument( + "--safe_serialization", + default=False, + action='store_true', + help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_dir, + max_shard_size=args.max_shard_size, + safe_serialization=args.safe_serialization, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/checkpoint_41/41/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt b/checkpoint_41/41/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..208b15b87966dd329b476694ce0bd99523c14190 --- /dev/null +++ b/checkpoint_41/41/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:86cf55968fc98cb0a6fb11939bc60700b66652f356e4019d932d578ca5d7014a +size 4653763191 diff --git a/checkpoint_41/41/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt b/checkpoint_41/41/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2a5a3b8b89b7de9fcb040c1d77cf754d5713ee93 --- /dev/null +++ b/checkpoint_41/41/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b3bc685e10c658853bb2c6e063bbca8f18d728bf5495fd4e35764f9a0693e0da +size 4653767415 diff --git a/checkpoint_41/41/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt b/checkpoint_41/41/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2f25cd0b471f49babedd6605b17395413ba55413 --- /dev/null +++ b/checkpoint_41/41/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:567672e91561f5f9a15b3cb18d9cd52694ecaec63c9c549ecb5971bbe1cb45a9 +size 4653767415 diff --git a/checkpoint_41/41/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt b/checkpoint_41/41/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3baf05a4dc9188018b57deb5db4000bd662ac467 --- /dev/null +++ b/checkpoint_41/41/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f8cf32159b4bf35ed606838b5c0b32470638196e8b4f9b1896d55b054b04e6f +size 4653767287 diff --git a/checkpoint_41/41/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt b/checkpoint_41/41/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..463ae157d53446ba667fd090bdcf6e3b873a2785 --- /dev/null +++ b/checkpoint_41/41/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:830e1a7739150a24b2ad2384025979ed0deb51ac0c2ae93087938777e6f2abd5 +size 4653767351 diff --git a/checkpoint_41/41/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt b/checkpoint_41/41/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7c98bbccdfb99787a0eb5d0da45d6d77f39ec3b1 --- /dev/null +++ b/checkpoint_41/41/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:21982a1c3f0d82275ba9fed56a17b367d52bd48723026e87b2d53901b9e31ea2 +size 4653767351 diff --git a/checkpoint_41/41/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt b/checkpoint_41/41/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..62dacbd0f404a7ab33f3394c291fcf2734f9de2b --- /dev/null +++ b/checkpoint_41/41/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d3513b72d4deb36c622d5de1cbfca3f8cf8a5fd36e78b94372dbad3e2a5bb2dc +size 4653767351 diff --git a/checkpoint_41/41/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt b/checkpoint_41/41/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..044e962aa25b8c253c456d4c8ba9a749b14c372c --- /dev/null +++ b/checkpoint_41/41/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:610a9f15f9df5926138db65f7f535191ca09146c3ce86b23021b15e379b068aa +size 4653767543 diff --git a/checkpoint_41/41/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt b/checkpoint_41/41/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6ed84b13810eec4825426bf88e43e519f5967847 --- /dev/null +++ b/checkpoint_41/41/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:133a6bd2a5eb4a99e0fe95e02b0f3447025e40708d86979070e9e37f41e68ed1 +size 4653767351 diff --git a/checkpoint_41/41/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/checkpoint_41/41/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fa30de039cc4905a17ab09a4511d9f2ea9db4ea9 --- /dev/null +++ b/checkpoint_41/41/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f105f1a9923eda858028a7eca216b3881b79662447458b2fccddb297255072d +size 4653766128 diff --git a/checkpoint_41/41/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt b/checkpoint_41/41/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..078fe251673fc471be434c9c5da693b931924775 --- /dev/null +++ b/checkpoint_41/41/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f9f6012f7cd69bc0928178ec4e5d10e8116a324735ed7a6164c25f412fc7d980 +size 4653767351 diff --git a/checkpoint_41/41/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt b/checkpoint_41/41/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4bdd661004db30361d30ffeccc5ca9a2c4d2679d --- /dev/null +++ b/checkpoint_41/41/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5230eee6a983dcfd6e39dcf1871ca1b15a80f89cb015e876b2312d7d7adfccea +size 4653767927 diff --git a/checkpoint_41/41/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt b/checkpoint_41/41/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..becce48acd775ffbe3f234eec5fe15e246d1c49e --- /dev/null +++ b/checkpoint_41/41/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:604b42b4de6b971f9d2f4f74c0978827b27194e77796226f70bccbf8152e8ac0 +size 4653766128 diff --git a/checkpoint_41/41/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt b/checkpoint_41/41/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1bf8bf437c35ddb16ab3a950996e468ce85feaa8 --- /dev/null +++ b/checkpoint_41/41/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fea0392260ffb3590a4bb9a4eaad746de553bd58fbc8faa74a795b1f409e054c +size 4653766192 diff --git a/checkpoint_41/41/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt b/checkpoint_41/41/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5e34fdc8ab5101d86ae38b635536fc60e6f9e194 --- /dev/null +++ b/checkpoint_41/41/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:462380536ebe897fba0676edfc77f69e75efb9fc13e75b07ae7f7f6f3fad5003 +size 4653766640 diff --git a/checkpoint_41/41/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt b/checkpoint_41/41/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d7fd13719b457321d3edf295c0d5a3966b20835d --- /dev/null +++ b/checkpoint_41/41/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b12f61097b3ed3ce3ce7a18d30d56116e19e3c373bf9d01e0dee25981a7a7f1c +size 4653766384 diff --git a/checkpoint_41/41/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt b/checkpoint_41/41/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..58ccbccb68e43cbcd227b75097ce2a3e212e62c8 --- /dev/null +++ b/checkpoint_41/41/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56f61d7587e1755ebc2345c99866cb7736774d8714da89a6f02cf17f65a478d9 +size 4653766384 diff --git a/checkpoint_41/41/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt b/checkpoint_41/41/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2954dd808ab8cbe9c64c89a6ad377e40e102e9f3 --- /dev/null +++ b/checkpoint_41/41/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5b6a9e6db5b3cda4eb4beb9d241e855c249dabda6273aedf6daf25a4bacfa1bf +size 4653766192 diff --git a/checkpoint_41/41/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt b/checkpoint_41/41/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..83497834f2db6395fb0f8cdf0b9691380f23df09 --- /dev/null +++ b/checkpoint_41/41/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:42f5194bfc46aeadde6b54f3cf64efbe5ca4216069a1eff66427c6005318eef6 +size 4653766384