diff --git a/global_step194526/_universal/mp_rank_00_model_states.pt b/global_step194526/_universal/mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3951d6e6a69d875eac133f13760e5359d93d7ad5
--- /dev/null
+++ b/global_step194526/_universal/mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:85186ad6e40f2b727e0fdd1a1db6e2ebc9cfe7b6c6ad89f5fc521017fe415fda
+size 4468641136
diff --git a/global_step194526/_universal/zero/lm_head_alpha/exp_avg.pt b/global_step194526/_universal/zero/lm_head_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c38a92c8b3694faa254b458e46413d23d8062e5b
--- /dev/null
+++ b/global_step194526/_universal/zero/lm_head_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:44e27e60a26845755e0ea5a9c7fcb25ee5d3bad9e52379e8de8747012237f437
+size 1180
diff --git a/global_step194526/_universal/zero/lm_head_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/lm_head_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3b543a67ab4df1753d1f153008dd696a1eafc922
--- /dev/null
+++ b/global_step194526/_universal/zero/lm_head_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b9e3245da8de7a52375647e2741c51cd6532962c5737cdf8b28ed278a18050c7
+size 1195
diff --git a/global_step194526/_universal/zero/lm_head_alpha/fp32.pt b/global_step194526/_universal/zero/lm_head_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..82e8a2e40c42c7876869ba178c4a5c5d7fc0c104
--- /dev/null
+++ b/global_step194526/_universal/zero/lm_head_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e1c631eef5796548952c721e08ce2377365a050fd9ce0df60c3c2fa184b2fd12
+size 1165
diff --git a/global_step194526/_universal/zero/lm_head_alpha/step.pt b/global_step194526/_universal/zero/lm_head_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/lm_head_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.embed_tokens.weight/exp_avg.pt b/global_step194526/_universal/zero/model.embed_tokens.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b60e082b3ee2b1a1e65216fb0150ead89d798672
--- /dev/null
+++ b/global_step194526/_universal/zero/model.embed_tokens.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:10d2e801309758b28dcbc1ecb2dbf63a1c7b2583b1aa7d287f3fa14b4d084645
+size 760321244
diff --git a/global_step194526/_universal/zero/model.embed_tokens.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.embed_tokens.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f98f2ef5f90536f1d233cb0bfd66db25af197fd9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.embed_tokens.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f41c44bed5c3af11ee5a0aad2daf430d703f7168e1e316c9981f68deb5800346
+size 760321259
diff --git a/global_step194526/_universal/zero/model.embed_tokens.weight/fp32.pt b/global_step194526/_universal/zero/model.embed_tokens.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e203b8434e2486fd5b7de8e60cdc8a52c87e8278
--- /dev/null
+++ b/global_step194526/_universal/zero/model.embed_tokens.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aa86e53ab786197ec85ced876e210309c58f62c91c0cc4f43921e0ae86d1b8e1
+size 760321165
diff --git a/global_step194526/_universal/zero/model.embed_tokens.weight/step.pt b/global_step194526/_universal/zero/model.embed_tokens.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.embed_tokens.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.0.down_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.0.down_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..585beee0c459f48e2155b6933fec40334c46a759
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.0.down_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6e1df87e179d69c0a3ca46c59e7f83b895669dd4160d37ec84404c6619df24ba
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.0.down_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.0.down_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c9510b23fc28572dc35c3585d30f795d8bf997ac
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.0.down_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dad56fb6f64434c63e40e3010956d61e967f8d0bafc3b82ab6c696f95ad3317d
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.0.down_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.0.down_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..114ff931e6a1a52c24fd1cb103d16103e145ac3c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.0.down_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f6be0c1f29f9e939dbe70caad9294d16b17dde174b3aa8b2ff36933c9727519b
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.0.down_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.0.down_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.0.down_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.0.gate_up_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.0.gate_up_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..776d31371acd89388c3cc7fd0deb8e37caf1e934
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.0.gate_up_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9206e46264e8246c1a4c554e7932e92fd391d473ac0451e95924779dc72fb1dc
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.0.gate_up_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.0.gate_up_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ec3bbb8898ce3839bbdbb349ed20eb632aa746dd
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.0.gate_up_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:00d85b77edcc355ea18b4058fa3b094acd03bfe210af0078e5884b9ca75e75e0
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.0.gate_up_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.0.gate_up_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f0745067b490df31b945a5bfd5afe04a946d831a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.0.gate_up_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:33e2c23374a4035ef35dd71e2b36695a72651f450b4b5bd43b246d78bc072e46
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.0.gate_up_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.0.gate_up_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.0.gate_up_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.0.input_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.0.input_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fca10a68c06956d095d7f01bb0a662759d74e41b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.0.input_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a69ce8aa089d6bbcd2d66dcc98041f2ff78493b89034b32ae6934d986551503a
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.0.input_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.0.input_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a4523358cb5cb2004213be9b9c3c21ac8a6967b2
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.0.input_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f5a3076d74c1c1266f3a28764d143bc05df4df47016758fdca9d4ffef31d3ab7
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.0.input_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.0.input_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..335748c902e317d5b420f11c54aa116821f4a918
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.0.input_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6c8f5ae38b705f0622e145d277bfc1e73d12caa45c36656bc6830b36d67cb38e
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.0.input_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.0.input_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.0.input_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.0.input_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.0.input_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9e290433b077790920629bba138c1f35b745e5ee
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.0.input_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4e52ef2e5bc0f3bb7c1f8f6567f2ec8b223702149608dded3670a8e8e64b7817
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.0.input_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.0.input_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..54a0a2f0cc42a95438488508d7f278847fb23a35
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.0.input_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a041bd18b8dd2fe32447795a201d0680edf5efc7035b5a1e4e18ce13f00cc930
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.0.input_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.0.input_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7b0a5162f053510e3e465c1e0a18c190bf35a46d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.0.input_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f81d0cc93eacd56c792c3fb6e8ca0b363b7d0293279dc2cf7f6657f6ce0e8a79
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.0.input_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.0.input_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.0.input_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.0.mlp.down_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.0.mlp.down_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8c2e71a1592bc57a6163dc6f4bd0b8a1a322dc1a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.0.mlp.down_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea18a24176f8044224a8b3249c294283c8fdfed253e71db89b0a5a4edd7e9ae4
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.0.mlp.down_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.0.mlp.down_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7c2160e01f896c13baed10729063ad9a282dda2a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.0.mlp.down_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8a4186ad14ebbde998a6b5d9d7fdf663cc5237d73d7fdd39cde1d4b6291547ec
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.0.mlp.down_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.0.mlp.down_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c6fc8accd3e2df0c15ed75dfa9aa160ed4dd5f94
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.0.mlp.down_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29d5e080efde6fbe6809d6c4473b9d53e593d69d43686bfa6d348b313fb0b682
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.0.mlp.down_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.0.mlp.down_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.0.mlp.down_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.0.mlp.gate_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.0.mlp.gate_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e9a828ac8bb66b30147b5d11bd7f1a35463c4d9f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.0.mlp.gate_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1a5cad3dd65e5b180b05690fc63a0d920a285eaa432df7291761d5e64b252868
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.0.mlp.gate_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.0.mlp.gate_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..033d3a6e5519ca02b371c55943a2b26d251a7945
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.0.mlp.gate_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:94296b1947f16d7e70fc4f6b76e1fb10c2f3505ac87920d9822f4abdf2b95dec
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.0.mlp.gate_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.0.mlp.gate_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..307feee1c536a94bad798c7836557b78cb55b2b0
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.0.mlp.gate_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2c5b0785d5a9ef8e228e7b7f80c1cd5b05faa9fd3e5dca4823c4db3dfb250ffa
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.0.mlp.gate_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.0.mlp.gate_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.0.mlp.gate_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.0.mlp.up_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.0.mlp.up_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..bf3367493665d3775c14e2503366ae7ee5d916f5
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.0.mlp.up_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cee4a898e2a6456a89850b70ac139f82cbca9fc467920541ce376f65c2fb72fd
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.0.mlp.up_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.0.mlp.up_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2048cd33b6e2a873790ca6adc1848df03faa15ba
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.0.mlp.up_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:77c6d1cf1bc5c1ac5b5a7dde12dda38649f069b2522a4cf77d7c92e607f5e297
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.0.mlp.up_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.0.mlp.up_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..35964c896405d7b6f1e2261c5918ee35aa6b23a8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.0.mlp.up_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0ca1503396d758c66079c5c3d02b81e9bc07a9d9d26e215f159822664888ac29
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.0.mlp.up_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.0.mlp.up_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.0.mlp.up_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.0.post_attention_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.0.post_attention_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..267c3c1e1a21d98022bd825d69a481bfdd46cd48
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.0.post_attention_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:605ca6cb7acabf480e5fd13d7181a50d6d44fa430668256d5bed8461a67a7205
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.0.post_attention_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.0.post_attention_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c6c0dcca1ee8aa71d0c900783d5fdf791d524df4
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.0.post_attention_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7e90916fa36f3f8c4b09490df6f2523a071a2bec74b81bba5694faef81281fb9
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.0.post_attention_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.0.post_attention_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c660a3ad2f2fe441e5320e3a4beaa4f1fdee2b39
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.0.post_attention_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b6750caef4e4fa484fae665776735711f6c2795aa3c963939faf2283b4579882
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.0.post_attention_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.0.post_attention_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.0.post_attention_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.0.post_attention_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.0.post_attention_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..07b394aa55599568672bf91e365285f3f5a3097d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.0.post_attention_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2a7adb468c8f8fc719aa904ca6e14587b55a81afb2fa1710a56038ea9ba1db97
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.0.post_attention_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.0.post_attention_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f0c8e16dc93a1a1aa41485e38022e221a1cd2339
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.0.post_attention_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5639d6916497d5fcd972815be66a6c54f68b0c601e3d59f8439c5a6ef60094b8
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.0.post_attention_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.0.post_attention_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..701d0815c0f3a35dbe201678ec7dedbd7c693ef4
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.0.post_attention_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7543a5b0dbc91a24cf9dcb69445ca4e332a147d78caf4e2bae2f544ac4ba7d81
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.0.post_attention_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.0.post_attention_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.0.post_attention_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.0.self_attn.k_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.0.self_attn.k_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..67a2452e4334252c9a086190f5ab7ddb743e0e64
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.0.self_attn.k_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe3ab141014811704f3bc12b36e8560bd977f90d8de446dc08d024c522b0344c
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.0.self_attn.k_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.0.self_attn.k_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2cefc4c3e3d91de8306cee544284ad0cb499585a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.0.self_attn.k_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f40ae5ec4f67092863981b549d3c381701b17441a328d518c46498aa4c097439
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.0.self_attn.k_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.0.self_attn.k_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..bf5551008ac279bcf75742cca82a4ad415149c39
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.0.self_attn.k_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4c92ee0f202e8e56ae6877484766fe71a8aa204a84d8d1908b5454dfc408877e
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.0.self_attn.k_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.0.self_attn.k_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.0.self_attn.k_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.0.self_attn.k_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.0.self_attn.k_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3b5ebd8be8ac140b6b8315e24d30d0f0995e3a09
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.0.self_attn.k_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:72bfe626ea5dbeb49a49fd2bf4ce1db3f92c4116c45421dcf0d3f8be7f112789
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.0.self_attn.k_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.0.self_attn.k_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..db0c96d398e29ecf20c1b693123e31386a1fa6fa
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.0.self_attn.k_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc41a3bcdb275f916de6ade7bec4349b865c9ef71b6c069d9b3c723d81e51638
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.0.self_attn.k_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.0.self_attn.k_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c40ebd5aacd5f415486d48b4bb9c9dc6750ce8de
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.0.self_attn.k_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c2aa7db70f4995ae82c6262402132e8e5d3df035bd53da1a282c85e031cee017
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.0.self_attn.k_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.0.self_attn.k_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.0.self_attn.k_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.0.self_attn.k_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.0.self_attn.k_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..dc6ee89dca0ecb692a886087e25e23ad35a6aa53
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.0.self_attn.k_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e3f1511b877c52e5c0e855452707ffd9514a8e70cae7ab3b87b14ba46043004c
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.0.self_attn.k_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.0.self_attn.k_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b2aa63b3c012b3de2122e40e2ffec625a837389b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.0.self_attn.k_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e6e8e0d413857cbba12d5d40c7fb54e355be1bab1edabfe23bf5aca747fda7b0
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.0.self_attn.k_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.0.self_attn.k_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..73db2ecb408ee34db43998929c82c5ae70458466
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.0.self_attn.k_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:48b475d8d8474826538dbca276e0a7ef958dcc7ed6f0a42634601562714d4c6d
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.0.self_attn.k_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.0.self_attn.k_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.0.self_attn.k_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.0.self_attn.o_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.0.self_attn.o_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7a8ba745cdcfb3e6117870ee18fb67bcf5231748
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.0.self_attn.o_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2ad2f67417e0c1152594b73383dbb648d327d18868386ac2cbf4478dae38b862
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.0.self_attn.o_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.0.self_attn.o_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cca7c11d82d726b6b33d0821a01aa8575df381f1
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.0.self_attn.o_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c6335d0342e5911db5140b9d108e2ef9d42cca14b883965e4fcd7bf49ba46bf
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.0.self_attn.o_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.0.self_attn.o_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..597eba170c42a9f8c4eeb35aae64acc4ebf54467
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.0.self_attn.o_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a87a43a6ecda49bd5ea8f1fdaef0dd285d3207d13c1fd0f94b8d7cc05f1b6aaa
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.0.self_attn.o_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.0.self_attn.o_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.0.self_attn.o_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.0.self_attn.o_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.0.self_attn.o_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..59ba7c4457706e17ca2c7b232995e8c59684f399
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.0.self_attn.o_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:041ee0c58f6871e7de0a55f56c5ac1b0205af2c7ee79dcf9e0e90ebd5a4d4605
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.0.self_attn.o_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.0.self_attn.o_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6dfcd4ddff28da616d3d78218f41c4d765c55d3d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.0.self_attn.o_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e5653611e2da3f68c53e00eaa4757b51f9fec76e6897ad6851a759bbe31be7a6
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.0.self_attn.o_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.0.self_attn.o_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3ee2e83ef97764db88c385049719ecaebe5dec04
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.0.self_attn.o_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b557b2540092e40ca2195ebb87e98bc20e9e43044a6c702dcb558b4947bca21d
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.0.self_attn.o_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.0.self_attn.o_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.0.self_attn.o_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.0.self_attn.q_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.0.self_attn.q_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7059426e388cb39ba873a907a96a113108689266
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.0.self_attn.q_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6da464120a8b9c52ab8896d645bf8f7af69c694175a4a3429d3cf94292e199ac
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.0.self_attn.q_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.0.self_attn.q_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..aaf933281ed0558b8c089dc97e18720182c96c5e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.0.self_attn.q_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:82858823fd249d5c1d8ed8fcf681793d21602d910021ca5f550b3d21b50ef302
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.0.self_attn.q_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.0.self_attn.q_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3860be75805491a9e5c5c9115f18e3a25652ceb9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.0.self_attn.q_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be6572ca0b89a220d1f78ab1ab8167f29bffb68c86adc4443aeeb4c55475855f
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.0.self_attn.q_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.0.self_attn.q_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.0.self_attn.q_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.0.self_attn.q_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.0.self_attn.q_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3b8363d4a1f3f43e4ba8e50628db517874cc9005
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.0.self_attn.q_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f015dc9dc0d197aba3394b5c83124aa779b6a676952fa3b5833fc651338db409
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.0.self_attn.q_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.0.self_attn.q_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..73be5fb560773897a8f121e92ef921c9510ba504
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.0.self_attn.q_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0f2ab4744b50365a2e75cb43c8bf3812cb26d708d6377fa62e713950f2ed5fac
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.0.self_attn.q_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.0.self_attn.q_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ff8370bfbeaf5ed6e786bc2388c07fc0f16c2c9c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.0.self_attn.q_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:182a48b5d3a8a7ed655c22c2eb9579fa29b6e872facefe3e35a7eac2f7e8b88c
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.0.self_attn.q_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.0.self_attn.q_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.0.self_attn.q_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.0.self_attn.q_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.0.self_attn.q_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2aaf86c04da2640391d985255483db2f7ebf30f4
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.0.self_attn.q_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e9576ecd2872c2081bb93c242f9e7af7308b80aaa2c0126cab44f40a26040ed5
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.0.self_attn.q_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.0.self_attn.q_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b1d3f4e2319551dbf2fd86d5e46010408d5486cf
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.0.self_attn.q_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:91caf47e3c978d750e7b08759f3d8188be27c5273ad856ac080a8fcde613d47b
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.0.self_attn.q_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.0.self_attn.q_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ca360fdf8bdac130e6fb3b6636d7ab391a47fd53
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.0.self_attn.q_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7d2493293d986d33186d93a13211cbba499f294160308c4423c72d2737a43b4c
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.0.self_attn.q_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.0.self_attn.q_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.0.self_attn.q_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.0.self_attn.v_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.0.self_attn.v_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d858f3c1f7f0bb64d87d63abbf3d9debb8dd2785
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.0.self_attn.v_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1411e7426db0ed62ed0989542fe5933fbb86b3e5bcdacb017b97a4cf52b92c89
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.0.self_attn.v_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.0.self_attn.v_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6d1b1929099335f5ec826debeeb90ee503ac536d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.0.self_attn.v_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0bf88c38ae8e3c3743d26f4950da20903c6bcc77f1f39d6da305c4faf267e899
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.0.self_attn.v_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.0.self_attn.v_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..39e0b6f28f8e2ec5d097a462ec02899b93dcc9e9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.0.self_attn.v_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:72a9712d58d0fdb75e0a907b3da6f08ac59b6eab86a5b07f8ec17869a78896ab
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.0.self_attn.v_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.0.self_attn.v_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.0.self_attn.v_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.0.self_attn.v_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.0.self_attn.v_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..06e34020ca478cfe418d09f4e6d1ca57fe2c9af0
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.0.self_attn.v_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:015d2df759db1a254fa56adeaa8d74c8bf9b1c6c66c082df0354e209948b8609
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.0.self_attn.v_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.0.self_attn.v_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5a6e4f3e4b3dfff5af9ca7d7391cc2c6c55f6013
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.0.self_attn.v_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:067b976e804f4c37e76a51bd253dff03a15906bb52dbf8c9ee77997aba615a65
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.0.self_attn.v_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.0.self_attn.v_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..60d8952d8906bb12d27302605be1806fcf5172c4
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.0.self_attn.v_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f44321bd60d89eab0c99371ffa6b06b7721971bcd20656ba871c0aa61e1a54bb
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.0.self_attn.v_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.0.self_attn.v_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.0.self_attn.v_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.0.self_attn.v_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.0.self_attn.v_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..192a47efbd770bf06ad1887474b0cad5244e96f7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.0.self_attn.v_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b52d6eb01fa11573b00fdfffad7b16d4ee924fd64e67f034ad22c16b8d1699ac
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.0.self_attn.v_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.0.self_attn.v_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c52397373d70c13fb68d58efbb4e533174bea7d3
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.0.self_attn.v_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f90a12bf613210ed227245ced2275d5871bb15ba42e0c03e9970905276b8b46e
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.0.self_attn.v_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.0.self_attn.v_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c8138f06d8ee37fbdd10d288194798517ca31025
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.0.self_attn.v_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ead0ea09c956af11cd07f1b2452865c81a8d66996f1d3e8523f13c89e123bd73
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.0.self_attn.v_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.0.self_attn.v_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.0.self_attn.v_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.1.down_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.1.down_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4cec3aeead396223160136fe4f2cedc8f8493d11
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.1.down_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef48e9aac519e210833a5a9fb46be8cdc4656066682d1a30cbe716b7a3a5bfad
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.1.down_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.1.down_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a2dc43576a107538333224dba6e2251a2babb25f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.1.down_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b095f7a3d77be8b4ebf3f39dae5fe78f8182b780ab5dd96fd9860b27781214f6
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.1.down_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.1.down_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..00690153431eaa518c6f8279a5543ee4707811c2
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.1.down_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:230eff2c70e8ca8dc2d62f17d98357267827f13f762bfa8828cecd39229c3737
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.1.down_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.1.down_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.1.down_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.1.gate_up_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.1.gate_up_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..bba5fe0bcfa8a21ba0c1bd568a3a42b3de91e261
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.1.gate_up_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e86460df0ba8ea59b828a80bd3693bbaa031f1c109ce7f5f18ae92d0470bc4b9
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.1.gate_up_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.1.gate_up_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a3cc09dc151e2b31120adae34144cce629bcc451
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.1.gate_up_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9813b48f08cec25440ca08d6c2337f72ebf7fe92677b7143c6d6f735fa760f54
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.1.gate_up_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.1.gate_up_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..55cebd45b7fa99be13b39b6b5980f5fa320467f2
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.1.gate_up_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6c7eccaf4d3a0c5b06cc47372ca44188163f3c7bdb0671354665da9b037a3256
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.1.gate_up_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.1.gate_up_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.1.gate_up_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.1.input_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.1.input_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cd61c94357aae922ee5719e1601e9ffab316f9cd
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.1.input_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4223471e30ef3a05661884295207ff5c58e39b0863c5fb2852d94d22cc21f272
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.1.input_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.1.input_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ae773e8e6d4c7c43382751309d2309224236bd50
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.1.input_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:28c68f25a8c1e8fbf92181af68183700b83ea48f167f184d4851255c35dc33ad
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.1.input_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.1.input_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d987650d509c821f26d900d0ba0168409bf0d816
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.1.input_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f5d3004aa7bedb1a866242ab927bea64cf5968f2e12d0233ba960957d6981640
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.1.input_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.1.input_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.1.input_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.1.input_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.1.input_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..016dd896e788e3b3ae649c9cc40a5324191bcc81
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.1.input_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eea0dbfe6d7ebc20dbd204438b057f35f86342ffe3b16910755b6dc975a3f94a
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.1.input_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.1.input_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..81fba4ecdedd9bc0a27db1f761636419436bda86
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.1.input_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a112e7e7348dd8ecfc70178b242d6b60c8556bf3340f72f80288e835ebc9bd09
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.1.input_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.1.input_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ea21d2ef9c465a25e3cfdcaeaf2f34f08da4f85e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.1.input_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c34f797367c76d05172fa2176e6f6a2599b19e59688a94481cd7ddd7db7383a6
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.1.input_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.1.input_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.1.input_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.1.mlp.down_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.1.mlp.down_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1fd44359d4db5346dbf7c1a8f26cd8dd51ac1911
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.1.mlp.down_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e46715255bea7b204580594154595575541dfc4a5cf7628d2cafd102557d6b7b
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.1.mlp.down_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.1.mlp.down_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..97ba8d4e95e0c23d5e628e0231a5d59ac14291d4
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.1.mlp.down_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1a2ad7862ec6dcb372983577d0766cb301cade1f94940eb085d0159551a68f79
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.1.mlp.down_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.1.mlp.down_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..24b3f2d889a6e6fc87ed116e9d40030048c983a5
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.1.mlp.down_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:96f25a55c20c390dd88acf03710865b6081a866d044c658cf9887ebd03096ccb
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.1.mlp.down_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.1.mlp.down_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.1.mlp.down_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.1.mlp.gate_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.1.mlp.gate_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0917f00457aa71e3b89ec2119b5e66311e1be231
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.1.mlp.gate_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bcc7f59788cb4d3604630eacfca4ea6f64adb7b0162198b12f71d4398472b07d
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.1.mlp.gate_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.1.mlp.gate_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d1f3df851275806c625e5b3455eca977cb01bc68
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.1.mlp.gate_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9ef3f482426a0cc83e743c63bb75f541a7fcbafb966d40adca925d5d24282a21
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.1.mlp.gate_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.1.mlp.gate_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e9992052e025440259ad485ac1deee57758307b9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.1.mlp.gate_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b798953e78950a578fc990485dbc5d266b5d97456c356ca098f13bc7c1ccb17b
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.1.mlp.gate_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.1.mlp.gate_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.1.mlp.gate_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.1.mlp.up_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.1.mlp.up_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a6b70e098ca0bd4c26fb271048e6906d27b584ea
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.1.mlp.up_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eea41533b78045ea4cdd53147c7fd5bc521c44d104e0ce619749481f73ac4772
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.1.mlp.up_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.1.mlp.up_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f4c613edd7cea930f373becb13663d4385abc316
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.1.mlp.up_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7acb3a588de1224c550afd76b1ec4494a8b9bbf361664b2da8d81996fcf563c3
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.1.mlp.up_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.1.mlp.up_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5b83f11a605b913ee132a21b63e2828b6425be79
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.1.mlp.up_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ed46455ce19c46a3b84e178f4580413deb16553eb42f760ee98bec7bcf7a1aee
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.1.mlp.up_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.1.mlp.up_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.1.mlp.up_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.1.post_attention_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.1.post_attention_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..407dd5621796acd69a7546e2b0682ba1822eed2a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.1.post_attention_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5bf823e6f13d82bd19a1806bb4abaf33daa4fa87dcc85321e0d88cdb664a1408
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.1.post_attention_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.1.post_attention_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..648cf6d29db040c004134b345b0212f4fb1b33bc
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.1.post_attention_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ed9e9de193a47174bf7b40e77299f913b7dd5bfce3855230828bebec1ca2a865
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.1.post_attention_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.1.post_attention_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cfcba79b6fd58c9070534548c480676103d9dbfa
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.1.post_attention_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:82486e2fc55cc14e7e3745e399224e0d790b10ae3911373be7b95b000fe2345d
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.1.post_attention_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.1.post_attention_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.1.post_attention_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.1.post_attention_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.1.post_attention_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e8468520d29c994fb91bbe58e8073a018cff8666
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.1.post_attention_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c063a3258d84cbe5032083990b48e685b9aa81ae86c7a073730cfa170b54f684
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.1.post_attention_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.1.post_attention_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6e8ab4595b336ed20faa3aac6ac830ea03a00bfe
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.1.post_attention_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fcf051fd00d88a90ed3fc5952123d633e5df16c2643696837df4072b69a5490e
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.1.post_attention_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.1.post_attention_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0834a5c08a9c0e72afb766fedb41bd60a81d9b8a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.1.post_attention_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:22e2e4585317b49cf08e9807250731da383aaf558fa3f0bdabc58923f47d12b4
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.1.post_attention_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.1.post_attention_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.1.post_attention_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.1.self_attn.k_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.1.self_attn.k_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6953e2b439c211c879da39673ffdaec3fd8ba2af
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.1.self_attn.k_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8ba7373d519d4a28f8ee4e519a66334854a1b29f28b5898f3614db9dad5c0cba
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.1.self_attn.k_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.1.self_attn.k_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f08272fd4af5cfeabf204fa821b64ae8913b4f29
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.1.self_attn.k_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d496323e7a57ed9c28e511126946c26c8c17427b6616dd3256b23dcfd1a81eb6
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.1.self_attn.k_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.1.self_attn.k_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7b8b8de0005380d67e95cbaee96ceb2b14f45665
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.1.self_attn.k_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9a92aa9fee667d950470f77eec6af9e2b58c054d2bbc08c8f8a9380e214724c6
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.1.self_attn.k_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.1.self_attn.k_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.1.self_attn.k_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.1.self_attn.k_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.1.self_attn.k_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..26c2d370a138b81d35614e6d1e73603ead60565e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.1.self_attn.k_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6fa0e7c9142b72e69dc3ea3ea6fff3c795b649ad7767689ec44a9a20223d2cbd
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.1.self_attn.k_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.1.self_attn.k_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8c980869e4f31d3143ca3ec6c9a51864812cb619
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.1.self_attn.k_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a03415489c9548f3c14a3b6c3b9b5fd1997befd541f6911e88e2e75c8ad80845
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.1.self_attn.k_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.1.self_attn.k_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..49a423a4c6e961386fdf3139971a31281bd9ba92
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.1.self_attn.k_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f4bbe84c846de9efde2db8c639e790bc0ab5589dcd0c68c18af15a7df7bad1bf
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.1.self_attn.k_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.1.self_attn.k_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.1.self_attn.k_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.1.self_attn.k_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.1.self_attn.k_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3b811eb8413cad4bbe1cccffb53bda48d3fdbb63
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.1.self_attn.k_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2a35b44530179224bb9656eb2076c869577866055673b0aa13b80e981a800f3f
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.1.self_attn.k_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.1.self_attn.k_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1b749c8aae5b91cfd954dc4dc38a1c1010d0d368
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.1.self_attn.k_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eda71eb95a5c9fc0395c7f1e8222bdea43b128ffb21249c8a2c0b99230e28578
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.1.self_attn.k_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.1.self_attn.k_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..97750ecd9b4648b08ca2517969fac00b0ba6c205
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.1.self_attn.k_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:727e3f8f4db0a9ca4799ba4abb6f3ef3787d9ebd544cc4012b3da264da1d31c0
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.1.self_attn.k_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.1.self_attn.k_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.1.self_attn.k_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.1.self_attn.o_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.1.self_attn.o_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d002aeab77e0625933cac8b5bfc823abd86a9054
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.1.self_attn.o_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4829e83e17cd115f9e122f055490c1de199e36a031c2d49f66359e56fe95a820
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.1.self_attn.o_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.1.self_attn.o_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..885728ed86c0ce10ec07d34d3873e449364e195f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.1.self_attn.o_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9edab2d4a71cc95f3593c7f42a712134843f40fc22d0c8699d4750b1e6e4461b
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.1.self_attn.o_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.1.self_attn.o_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5dfa55a4b87a800e599a3f60bab443d8a2bfabde
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.1.self_attn.o_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cf66fa7de4a4da619bf04506dd8baf631d576d48877742e6cb29ba80ee9348be
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.1.self_attn.o_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.1.self_attn.o_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.1.self_attn.o_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.1.self_attn.o_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.1.self_attn.o_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2755533b8913592866ddb3b040bb1d3efc6114dc
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.1.self_attn.o_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd7943ce0b6fd34b80129aaa1dba95ea6ccfbf39808215e791ba1c53339fab43
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.1.self_attn.o_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.1.self_attn.o_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..89ae24c7989ad86ef6fe5f263cfcf261de76741e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.1.self_attn.o_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:46262c557f8e43dbed25bafd39b30d9fd485b7a47adeabc10ac31e7a3a49c2c5
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.1.self_attn.o_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.1.self_attn.o_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..142f63aeaba7d06684be8460d7e6270faa3d154e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.1.self_attn.o_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ab7a2f0aab66db113f397d78cc49e60d0d09de8ed6ae904aa57c1f0bb762570d
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.1.self_attn.o_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.1.self_attn.o_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.1.self_attn.o_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.1.self_attn.q_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.1.self_attn.q_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..41859c3540c4ff8c8acb30b7490e079aabb3acca
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.1.self_attn.q_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a572db65e6e32c05eeec028e4144ef56a6a85637e6e8beeb761e3c970beb5327
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.1.self_attn.q_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.1.self_attn.q_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c545f2086410d133fd06065a8cda18de68a7c5d8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.1.self_attn.q_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7858b4b009b873fac14a9952d0fdbb6a760b90f5a39ff4cc897881b7ebb78020
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.1.self_attn.q_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.1.self_attn.q_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7d4e6811916be8e4fe0e7472cd74c382db7aeb1f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.1.self_attn.q_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b5ece7fefc47a33f236ad23f27396003bd72988d065bbebf645a1f4d178af320
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.1.self_attn.q_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.1.self_attn.q_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.1.self_attn.q_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.1.self_attn.q_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.1.self_attn.q_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..96c2b7a58398a029568be7d9da6c735dc3d1ba35
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.1.self_attn.q_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:895e7e7becd0db51ff3439c72db2b092c6af4883c46639debb35edfcdf3207c8
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.1.self_attn.q_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.1.self_attn.q_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5f7bb3e3003803684c98af48e89c4da8497910b5
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.1.self_attn.q_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d21ca215eab0a7131a35c8e37ec7a01931672ce99fef79e3501c3d27339ac089
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.1.self_attn.q_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.1.self_attn.q_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c6bf62fb421ffe9472d1f2ec78be899de8d3dc48
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.1.self_attn.q_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ddcfb4ea2e15406a65126e90be67adfae4956a825fb2dd317476ce1a1b10101d
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.1.self_attn.q_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.1.self_attn.q_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.1.self_attn.q_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.1.self_attn.q_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.1.self_attn.q_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f5cc3b9844a62725231f6ccb5636811f346e6e6e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.1.self_attn.q_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d0980a896abf8ea824bc37f5e7ff34dc3db15c1e12407ae2180f0fc9828e5444
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.1.self_attn.q_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.1.self_attn.q_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c52bc74be1a22153269b445101d5d2e92467d6ba
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.1.self_attn.q_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:929a953187eaa54c471e8ed0c7c4504ee1b9a7e39e12734506ef133c17578cc5
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.1.self_attn.q_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.1.self_attn.q_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ef681c6b0df9a4c811983b80ced79f8252b3744b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.1.self_attn.q_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:279c4bd75035dc2ac64e68c367640dd8ffa33087309976f7b3f0fd6457f169c6
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.1.self_attn.q_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.1.self_attn.q_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.1.self_attn.q_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.1.self_attn.v_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.1.self_attn.v_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..63dae48701fd2d90bb06ee8d999300bf89e55f90
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.1.self_attn.v_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c9d9b968cae6ea63bec280eca5c4f915b8bb297c3f8c7ef8422e9923f1eefbfe
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.1.self_attn.v_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.1.self_attn.v_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9cd3f3198994e3b33986e66b501662a810887a31
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.1.self_attn.v_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:264fcf603e9a7230b618274f6be29a194855de411a0319527adca806deb3d1eb
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.1.self_attn.v_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.1.self_attn.v_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..aecd33e1c2f629298056edaa7aa997a92a45898f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.1.self_attn.v_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:187de1c102506a23c32d62aad7d7e629f991989b7a80cf27ff54ca3aeb19ffc2
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.1.self_attn.v_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.1.self_attn.v_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.1.self_attn.v_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.1.self_attn.v_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.1.self_attn.v_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7d6f4be9c4d4c8965b7ea36927061ed5e08675a9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.1.self_attn.v_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca72044615eb1c5b004e037c8a29a9fd12e0d74cd10d97ddbbefb0d9031584b3
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.1.self_attn.v_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.1.self_attn.v_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9791a1351e9b69a55ff7a9fce6758c8694f4de4b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.1.self_attn.v_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3ee9db74938c92c7b6003dd01850a31bcd4769b92bf6c27a67c3fb491159cc19
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.1.self_attn.v_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.1.self_attn.v_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6609ef407094d059fda57d54a52985b6ff807de7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.1.self_attn.v_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a9bf001d9a9815cfad825b2efbbcbc6c2fc5fe2c99b712e9ea2026f20d5724b4
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.1.self_attn.v_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.1.self_attn.v_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.1.self_attn.v_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.1.self_attn.v_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.1.self_attn.v_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b49f903f46546a812c31a16e5566ffc7606453cb
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.1.self_attn.v_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fa6eed4e94ffcea10a9de04f41b78df34585a7c75f473f1d12b9e5c7492a7a08
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.1.self_attn.v_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.1.self_attn.v_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1cfd3992eed8dce0a0e9d18bf4921a005e02a7f7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.1.self_attn.v_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ecf88f281e63b86278417f451ed89ca9fada58901232e1e8e7969e4da7bfb5db
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.1.self_attn.v_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.1.self_attn.v_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..df36b99916d0bcff40946e1676a28c417181069a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.1.self_attn.v_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d94b702459ab19614c84478906047c99c0e106d4f9cca6432a78dd93e9874bb4
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.1.self_attn.v_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.1.self_attn.v_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.1.self_attn.v_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.10.down_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.10.down_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0ad1f222bf6e7a272be7dbd5c09a50b25b864083
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.10.down_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1c6d75753ed3fc389bc90d675df4bb3366769ed6f0a73cb3a03b984b1ee170d7
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.10.down_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.10.down_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c315d0525fcc120b3511741c0f657fa855369dab
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.10.down_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f1ffc0814ce87ef7114c17c12ed1ca43cdbf849c75c97d9eeccdee597f970f54
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.10.down_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.10.down_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..62aa23f8d98f2c55e981b98ea1e4af31d6f405ae
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.10.down_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8aa71b6002ae376e097b6e501787d29ea20ba8db53c23a82a17a5937e1fc59fb
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.10.down_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.10.down_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.10.down_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.10.gate_up_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.10.gate_up_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..342c25f4203893517a08e669f2d0a31132abcfab
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.10.gate_up_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e84546a546ce673d752c72beba529f359a01cfa093fef70448b681c9cfca635c
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.10.gate_up_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.10.gate_up_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..294c185274696093ad1e2c158d6babf40271c01c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.10.gate_up_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:092f8aed956aeeac1d5deebd00562000e94c67eaf17f8404df1ada0267287822
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.10.gate_up_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.10.gate_up_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..197dbb0152e029b6db0f6dc615b277664b215cee
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.10.gate_up_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c1534486e06b75c91282b034fbc69ce2fcfea386c560eb3e1e6de7a937c850d1
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.10.gate_up_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.10.gate_up_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.10.gate_up_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.10.input_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.10.input_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..968eba31a3676504dcd33812f35f773237f67be3
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.10.input_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc65062b5bd4aa632881682b7ddf7cb7111420a1bb9483f8b4323ac9843845f9
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.10.input_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.10.input_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..42cb9525ce94277fbab156e14871c4cb2a7bcb2a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.10.input_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aec4a6b9af4b97fa5b45dd83eda4b8add75fae00f42784e4f81b7260082b63be
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.10.input_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.10.input_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..777ae9014059f094c7285ec61ec1995d3f18462d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.10.input_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:380eaef23401ef5385707aaa9cdbaec9bbb51969a687b9ce8c965a5ad8d72d26
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.10.input_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.10.input_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.10.input_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.10.input_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.10.input_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ce830b7f005da1784bd3cc7b826245586240c981
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.10.input_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dce76e5ec53ecd83ecee33d3223b26fbb6b428031f0c507f77cdd245de0467ac
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.10.input_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.10.input_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5aa7ec1327f915a096f7c727eea0fe03ae395673
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.10.input_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b848d385c594c433e4bf7554b9717e17d0542920401cfbe09cec08fb6f8fd8cb
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.10.input_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.10.input_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9468cc31e4c218c9405bb0339b98e7fb8585c049
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.10.input_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5293ac1c16c78f720d5fd57711bb8a3818cb224409f4edf47f1f718c5b6dc2c2
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.10.input_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.10.input_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.10.input_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.10.mlp.down_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.10.mlp.down_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0d25d54e0ffcb7a3de724d98cf0d12641d743345
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.10.mlp.down_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:27598bf129b6162e049773a827c28999053a5de5da7aed214aa58aee6dafa607
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.10.mlp.down_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.10.mlp.down_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..06c39a1d5156ce4c0b5673801a501d58a271584d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.10.mlp.down_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be0fbc69019a8a39dfcfbcd94631a2db90077d25da87aa7d5c2fbfdecd338135
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.10.mlp.down_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.10.mlp.down_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b88723f4e0bde0fd67c1739e92b3a968eef6448f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.10.mlp.down_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c3aba16f4751021cdbd5fd57fc1d5992322763e25655c0bd3f7bbf96a787fe8c
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.10.mlp.down_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.10.mlp.down_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.10.mlp.down_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.10.mlp.gate_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.10.mlp.gate_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8940428b2fd85924620d28e7a6f7ce4bf94c5610
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.10.mlp.gate_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:783d0fffbd1ed0a0243199fa1b6456b2edbe0e0c8a69e007576997124c49e392
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.10.mlp.gate_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.10.mlp.gate_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0336c275b66876b597a11e0f12e4e4ccb70ee654
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.10.mlp.gate_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9b3d6f8a388d0b32662e105c785dff467fb2c8f7cb3523b2ffa52992988bf2f7
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.10.mlp.gate_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.10.mlp.gate_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..12ca295e8ae18212ea1c71aa12dc9672be18220f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.10.mlp.gate_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:659d0a4dc820dcfcc311df774c7a473817c1613dc928af0d374c584733122e9a
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.10.mlp.gate_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.10.mlp.gate_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.10.mlp.gate_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.10.mlp.up_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.10.mlp.up_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0c5c7b0589774f968ea93e017f6b2b9a5d502fb8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.10.mlp.up_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:16c4c6a925909744230264b9045163a9cfe33821f93496f19acc4393c0b29b59
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.10.mlp.up_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.10.mlp.up_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0f43a7f283828fb3199a80a2157109db9ea25c2e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.10.mlp.up_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:84814033022f2ce90c1819abef33433f01a8e520e241e5678c1c58a124175308
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.10.mlp.up_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.10.mlp.up_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..484dab96bb8f216833096cf2e603cf2281938ccc
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.10.mlp.up_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:52a6139a2a18c19104d90f7e884ac9b328d15c690d77b67357a50626fdbd37c6
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.10.mlp.up_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.10.mlp.up_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.10.mlp.up_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.10.post_attention_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.10.post_attention_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..54ec2b4928992b6bc84586a743412e60fdc7bc11
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.10.post_attention_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fc1de6f8e5eb2c024b67fee2136f80cda550b4606f9dd61da3ea77f843742c85
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.10.post_attention_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.10.post_attention_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d32ed6d9013ab08b56bdbc726e8009e64d297146
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.10.post_attention_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:260c4a038ddd14c013f4f5f9a20912e087b8d72b225064e7f47c2e67bea516af
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.10.post_attention_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.10.post_attention_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5064a6a7a06efc32e0a780f327d9ea7dcfa984ef
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.10.post_attention_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6666e4c7f74c1b1cae3fea2482abdddf0dced1925b8452072aec0fd902649cae
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.10.post_attention_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.10.post_attention_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.10.post_attention_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.10.post_attention_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.10.post_attention_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e757fc12554bf23627b9d2e7c8ecdd6ab1677c7b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.10.post_attention_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3c49ea175072cf540014e6053e2713090a9edd559d57e312b986871d7d244cc1
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.10.post_attention_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.10.post_attention_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ce7388ee0cc1cf0583a44e519dfd1ffdfcb0fb38
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.10.post_attention_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0ea1c4410fab4502542585905ba57b1d21dc62faf3613698b27ed8b76391c0ef
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.10.post_attention_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.10.post_attention_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..02b87bb392928411f71c1b16a0a9e5abbdea3bc9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.10.post_attention_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7bee5870b1c54cd32dd8808ae77323516c1cd6a6b16693749195b4d9a9786be8
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.10.post_attention_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.10.post_attention_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.10.post_attention_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.10.self_attn.k_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.10.self_attn.k_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..189067890d01f6a39fd74862824b12efcaf10e9b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.10.self_attn.k_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6c7bfcf2d2740de9a2784b7c830fface5eda4f5b4415c80192955ecf9fc19050
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.10.self_attn.k_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.10.self_attn.k_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d4e596711e6ddc4ea9bd48d2a6451ff078d460cf
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.10.self_attn.k_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8fcab38eb65084c518fbe10a1d11138f51b5fcf6080d0faa2f8abe1ba537bff7
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.10.self_attn.k_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.10.self_attn.k_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..03431ff61edb2d42084890a59e3323471160b429
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.10.self_attn.k_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bd61960a3be2ee86cb2563aebe8234343d3c3112d299e061058db8cfae7aa58a
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.10.self_attn.k_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.10.self_attn.k_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.10.self_attn.k_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.10.self_attn.k_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.10.self_attn.k_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1fbc59864cd52f08431c3b5b007b88d614a8da7f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.10.self_attn.k_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6573f5585c930c44df4d1a84225827ff422541fc42a84fe7276a5de178f8771e
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.10.self_attn.k_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.10.self_attn.k_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6522e7731d94deabf249c39d5e5c7506170c355a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.10.self_attn.k_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e31e2030a59605174c185051e76f46b43c16a7d840355468e1174da523813617
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.10.self_attn.k_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.10.self_attn.k_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cf9b5e88ff6d4890b0257e6dc2a20797ffaa389f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.10.self_attn.k_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5796bbc1c62464a2498f02e5f5b9d3f6348bbf69f25b142398b9454fb360c814
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.10.self_attn.k_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.10.self_attn.k_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.10.self_attn.k_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.10.self_attn.k_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.10.self_attn.k_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c44205be356589109d6477054963fe6da141b316
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.10.self_attn.k_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d7f1ad4feb4ac8cc66245010b7494781508218b3945ac948429cbae20e2ead26
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.10.self_attn.k_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.10.self_attn.k_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9a68f73c519163e8d3910f8187928677eb032f7f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.10.self_attn.k_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2bed9e6e68f806534c1b46bd69921c3358e82ca48a8b6e3cdbe5ea565dbfa2d8
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.10.self_attn.k_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.10.self_attn.k_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e54d2fa96900abb08470b34b940f6d4bd5e7d9e0
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.10.self_attn.k_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1624ddee18c27b7f8b724bc58c3bd263ead8b80d185329668098809f5dc63cb2
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.10.self_attn.k_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.10.self_attn.k_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.10.self_attn.k_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.10.self_attn.o_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.10.self_attn.o_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c3ff1d8fea578008407fa3cfaf19cf9ef8973b55
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.10.self_attn.o_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:295e70d8b5540b34647510bacb284e04bbaebf6f2adae162103420beceb2aa30
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.10.self_attn.o_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.10.self_attn.o_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2814c4447d387ddc20bedb3b4d248eb6c25e6dc6
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.10.self_attn.o_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:347ecf966200e6570bf0cbc13c7376df9291a64eae08cf97f16e252d37ae40e2
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.10.self_attn.o_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.10.self_attn.o_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d6b686409e4b1c28cc81eccbb220651d464eaf48
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.10.self_attn.o_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ce9f50bcc89f8ebafb16d594632ba12a064549d1351cc2f5e9cc20c8136074dc
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.10.self_attn.o_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.10.self_attn.o_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.10.self_attn.o_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.10.self_attn.o_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.10.self_attn.o_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..02e90acdd74a752a304fd15bfd4c24aaf764c1cf
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.10.self_attn.o_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:01e0d464397c276f6fd3e98868c4b02db527f45d8534945664a5f912412deec3
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.10.self_attn.o_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.10.self_attn.o_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f06a5f2efe42b1373efc94eb3fd400726f5d23c9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.10.self_attn.o_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ab09b31484fdf486fa726d801354976542800b3273aef8be7e6cc6cf5bc95228
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.10.self_attn.o_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.10.self_attn.o_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b4d28d896d4c47fdc64a076426f45746fd448f54
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.10.self_attn.o_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6406e89425a5d15ff3fada5f42d69194b8008ae072ca6ce3d1cc95efc684c228
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.10.self_attn.o_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.10.self_attn.o_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.10.self_attn.o_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.10.self_attn.q_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.10.self_attn.q_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..75eaa0eeb811f095fe8bf014edb563b825a64d4d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.10.self_attn.q_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0f312c058f421aa79d353f102c04805a276d6d1213453eeb40851429f24f4172
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.10.self_attn.q_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.10.self_attn.q_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..af50744b9165593a5cf6ad703631a002befd48da
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.10.self_attn.q_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1e5d8a4ee4bb68f144056146d95039706cb639d82c885fe6d27179b892700807
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.10.self_attn.q_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.10.self_attn.q_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0ddb4308cdb20fa9e9f6ab08b93551bed1791555
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.10.self_attn.q_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:02d48232be831fbdcf252c4eb6667aa7d7179fdd467e7b366ea19e9e612fa052
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.10.self_attn.q_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.10.self_attn.q_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.10.self_attn.q_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.10.self_attn.q_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.10.self_attn.q_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..bd26376ecf49c3640cf1b20ea613265d735dd793
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.10.self_attn.q_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:efc8cb248fe1c7c43932a018764ad0eb2e6b342c0e946d74ae4fa0c34b5d6cb0
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.10.self_attn.q_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.10.self_attn.q_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a0926f545a37bf8e482bffda8c4ab29c44cc06b1
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.10.self_attn.q_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a8dea0061b757bcaa492be46619975d15cdfc1dcf7ecfe3e1b6a3c6b17ac20e9
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.10.self_attn.q_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.10.self_attn.q_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8cac59a4a77a931ccce09cc4eab176e819b24796
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.10.self_attn.q_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a426e03ca09159da888bdba79375520e78c7f9a5c8da2da33c603475c758e25b
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.10.self_attn.q_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.10.self_attn.q_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.10.self_attn.q_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.10.self_attn.q_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.10.self_attn.q_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..38b8b224f0d208d527a5b568a7203b00ad01ce3f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.10.self_attn.q_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:837c176c1d0117d3234053d4f29e0aeaece199f9449663c35b5c83702b013f63
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.10.self_attn.q_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.10.self_attn.q_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..89e7dbc0a09f3bd7598f1ebdb1687e1682200c49
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.10.self_attn.q_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:31d774ad22a9ee9d6d00aee0a2dcabe4b5ae2ec26d46461fbac068db4f6152fe
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.10.self_attn.q_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.10.self_attn.q_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e55b1d2990d771c5026b2720215b5c73b117fbc6
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.10.self_attn.q_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3f5291038e8d9c244dcee79a957c4475335c501c3c482c3cec80f0713b8b4274
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.10.self_attn.q_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.10.self_attn.q_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.10.self_attn.q_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.10.self_attn.v_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.10.self_attn.v_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2ea34bf95d9fece4bb8d543556dbfc48219c8c8f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.10.self_attn.v_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f8a604aab020dcf5c9a2e4688ca8402191b3ca12cc3c7775e6e096e0f931a310
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.10.self_attn.v_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.10.self_attn.v_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b5476b2c052d2dfc3f3ef016e365c631886af8d5
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.10.self_attn.v_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd193da863ccfdba10931adb516c312e9947b9ac21df59af4c9a8b49c8fa8a7b
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.10.self_attn.v_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.10.self_attn.v_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a05d7b4da8d7da66148df7602fafeff00dc14fd0
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.10.self_attn.v_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f5c685bd34c4fc5382ef6f472ace9f4496d3f74657cdc081027f7c6063bfdf12
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.10.self_attn.v_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.10.self_attn.v_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.10.self_attn.v_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.10.self_attn.v_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.10.self_attn.v_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b069c12ec68ef061abf49005016fedee14df78c4
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.10.self_attn.v_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d30732bbd57a7a5ad85c32ee4c0b94cde4d5108f22f59425d49e284636ea9673
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.10.self_attn.v_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.10.self_attn.v_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..84301f6a5fe5d5db6efb776706967895908bd3a6
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.10.self_attn.v_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9cf9fe1676d902b1bf48279cd57df8a871065d2e1c75e7c0e59288e871acceeb
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.10.self_attn.v_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.10.self_attn.v_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7f2aa9912e5193825b56e2a1365462561d5c7133
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.10.self_attn.v_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:371ef9884ed33851f1706618c1e4e0f8a97a2d5bb3506869fe5182fabded430c
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.10.self_attn.v_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.10.self_attn.v_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.10.self_attn.v_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.10.self_attn.v_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.10.self_attn.v_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..33cec1d764c70bcea16a2c0dda9d484838d84c55
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.10.self_attn.v_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ddb957a98695287b464e4adcd98cad017598f19e81b7cf6a7f33cffaff398b3b
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.10.self_attn.v_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.10.self_attn.v_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..13bda3567ccfd1a52637728a3dd9c4633ed6835f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.10.self_attn.v_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:883a8728e26763cf0366761b1a7569df1474f19ccf10e4752cff65af7ea0717c
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.10.self_attn.v_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.10.self_attn.v_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0371470330e4b3109713e279243b969db593c534
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.10.self_attn.v_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f0d78e02118c803752bd437f18b58eedf6160f6ed01f1ad3f84504246743470f
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.10.self_attn.v_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.10.self_attn.v_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.10.self_attn.v_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.11.down_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.11.down_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0a3f4b8114d87741fbc0425799be088369946c38
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.11.down_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:74be4b66573dd72beacfd11af96105746adc5a99cd1ddf8f5f6a5d6be5b0be30
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.11.down_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.11.down_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b714c2b22e1089cc2be62bf33a3e16b85bda8c80
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.11.down_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d3db0bd53367e9d4e364ff9dda7bc7b7bc3c9371a8d838d6887a80989186619e
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.11.down_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.11.down_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c95bc34e9bb4abdb95abed7da0a29d6dc2bece48
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.11.down_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8bfcb5cd52ae1f64c19c9daf137e337299e0d79ccbc8e3110316480ae64fc278
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.11.down_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.11.down_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.11.down_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.11.gate_up_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.11.gate_up_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9570c90916fbf599adf70144ca259dae8e49cf97
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.11.gate_up_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e71b7d6d772d4773c011ffc28b9f7e5c2c8dacb226d6466821c825d4a38b3be2
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.11.gate_up_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.11.gate_up_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d8ab4e6a3e450ce335b98f8bc14fab5cfded66ea
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.11.gate_up_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e2583dd7cedbf145477585bcbb9832cd5b7e0ba445b3199a9274875d5af501a4
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.11.gate_up_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.11.gate_up_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4c06bc769d43726bb4af28ebe74da3fc56a4bfea
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.11.gate_up_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fca5803add3314ba9e3570f2b407b40fb044b1e99660215ded31fcea9eed2e4b
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.11.gate_up_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.11.gate_up_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.11.gate_up_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.11.input_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.11.input_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..47521f78d4a432e6028f25083c9b3486ae1a41e4
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.11.input_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d79ede9e9569ef043ad7e66cc3cab5cb4dbee1c9d6de7ccabcf93ed99d55193c
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.11.input_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.11.input_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fa7343bdb12fc868f3642b06823a95f308b66b41
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.11.input_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:191d31558cb6ec49c584a6673619b15658c399614511f346a243ab0ec2cb9edd
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.11.input_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.11.input_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9ca8421406f0d04a04ad0ecca7bbf08f25f2fb17
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.11.input_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1f7cb8ab62c586cab92ea45d9308a0c4edcf980bf8c9296f5dcbd807dea59432
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.11.input_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.11.input_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.11.input_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.11.input_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.11.input_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..72400ae04869d7e5495954cf5da85a45ad91cc08
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.11.input_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a65aeed5c857071fa95cc4b0f370cbf9b0e76bfa1476562a0b1c6dc3d510364e
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.11.input_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.11.input_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2f59282977cefa88a81d6ffaaa87fcd3002034a3
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.11.input_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4742140686de555e019efe710261c0d3f7a755123a6490a46dbd1a6bdde08dec
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.11.input_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.11.input_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5a233e3861f3895d96af1e56453f186840006f07
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.11.input_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6961d039285cc10433c575d07e4858c8632b932f36b72db36b6fb40c94e76191
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.11.input_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.11.input_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.11.input_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.11.mlp.down_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.11.mlp.down_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..241f321ec2da02fd2602d109303ea5f3c48bb4c5
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.11.mlp.down_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ff478dc9058fada1c845e5648e481f72e9c50a33ce5652330e05b362935c9cee
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.11.mlp.down_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.11.mlp.down_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5b32307441679fdf56c503e51dcb2c3b26d3c1ba
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.11.mlp.down_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a951a78dc9967a90f55c3e8a87e1418bffc9337c2f77b1f081897870a6cf6a9b
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.11.mlp.down_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.11.mlp.down_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3088893079ed66ad41356788d167b4efdd1f4775
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.11.mlp.down_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:989d73e52022cc17eac336186448dc922e6f6ba4a4cc40225bf011b649e7b4a0
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.11.mlp.down_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.11.mlp.down_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.11.mlp.down_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.11.mlp.gate_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.11.mlp.gate_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..87080c0012e936a53a761b2c0c0265bf4d285ff8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.11.mlp.gate_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be5292df71abb6fd83efbb11d4d7bb4a647d327dd4190f5511a732ed1c3d1160
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.11.mlp.gate_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.11.mlp.gate_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2d22ad3fbe52d159254693d8a6bd18ec572bd5c5
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.11.mlp.gate_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a736c5e93c343a1120e74eec7de1f37d37b978e265865e682f16e0679c4128e0
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.11.mlp.gate_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.11.mlp.gate_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6fddb34bc57851dfedcf275bc4cdc0672b347825
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.11.mlp.gate_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ea86b35f9fa96cdadabdd0a48be3e7897fe8cbd82f90fb0e657a449b8522786
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.11.mlp.gate_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.11.mlp.gate_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.11.mlp.gate_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.11.mlp.up_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.11.mlp.up_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..043fb2a1d398a8cb3ae7e6a5cd2b7a47e29710d7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.11.mlp.up_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:df7bb83bd7a9fcf2bb8623fe321863b919039a9848247ba6de74557d2052d0fe
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.11.mlp.up_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.11.mlp.up_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a96e7bd8d9f9b25938faaefe17695fac5c0842c7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.11.mlp.up_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5b62aa5b5c659cd15532cafeab5e44029446c1cc4fe9336669d8b3edc105d6fd
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.11.mlp.up_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.11.mlp.up_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4345bbed0196f0afc2cbf2665d7d1596bb02678e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.11.mlp.up_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e184f185b5ffdb26ac158891a363893e901b7c06a4ec2a7e2e012f42b54b717c
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.11.mlp.up_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.11.mlp.up_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.11.mlp.up_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.11.post_attention_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.11.post_attention_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6ff3d287a29c54c2bb42a2f9576b4366c0547650
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.11.post_attention_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:22b0fe29d4c4a7f28704b800c84baa892887efa959c4cdd91cb5e40dc41274a2
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.11.post_attention_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.11.post_attention_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..df7dfd2d7dc34795d7d482370941986be15848fe
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.11.post_attention_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9db3949d289e02ce44fb94158289f1c3d8f19281d99ee614a46aa3776cff378d
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.11.post_attention_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.11.post_attention_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..df7605fbe1d07d2d81a28daa5873fd0148b2fe85
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.11.post_attention_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5a81534fbac9d4b70f2bc86da04388b1220abefdcf64a6bff11a71ec263beea3
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.11.post_attention_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.11.post_attention_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.11.post_attention_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.11.post_attention_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.11.post_attention_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..019a444f6f87b346b5b10cfb1aac14e3e9424979
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.11.post_attention_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f5388c093387c153c281a6ab7ae77c535017b158f830b36a333b81dd3cab44b4
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.11.post_attention_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.11.post_attention_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2257e7489bf9ae59bd1e84ac94e0f0d5d7c00e0b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.11.post_attention_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4decbb93e8b673bfab157e8042ae975623b15b4ff2dbe26e7d24c4be7b9f2a03
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.11.post_attention_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.11.post_attention_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6d254007af6693690501264b89329fe9c46260a1
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.11.post_attention_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d13460423584a989f4928a664135c6ef2b408a1474a7e6467ba4927b25f51498
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.11.post_attention_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.11.post_attention_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.11.post_attention_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.11.self_attn.k_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.11.self_attn.k_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..48367dc4265688f5b75733c0b365b1a74ac82825
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.11.self_attn.k_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e28d615d2aa4c2c4318a7d5bb34ef4a873742c340f47bb8e711e080e652e57e8
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.11.self_attn.k_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.11.self_attn.k_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c27e40bdd3db9c2d0ada76e090ccbdd939b98325
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.11.self_attn.k_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:38a1947fd4616293c11b78989bb3e80f42f49d2a16296a8e2d163b960c99123f
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.11.self_attn.k_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.11.self_attn.k_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1f5a860d00f07e5beaea34e685dbf773b37c05ba
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.11.self_attn.k_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0bd8200690a3f138c2ade53f7d669d5bb11131d8b90bc7adb82a5c15d3fe6f46
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.11.self_attn.k_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.11.self_attn.k_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.11.self_attn.k_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.11.self_attn.k_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.11.self_attn.k_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fb88939ea2d460080bba35cb5773dc332d0f25e7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.11.self_attn.k_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:da3b1613a82b64a5add12fb8f3c0f1513117bacb8b9d09b78b533d87c01e240f
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.11.self_attn.k_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.11.self_attn.k_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..35e200da398fb5f9783c8925d43eafc0085ff3c4
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.11.self_attn.k_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ed6a6b8acce30bdd63d0423f243d1ef5a1268f407b7769aec9378cc2b8aac69
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.11.self_attn.k_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.11.self_attn.k_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..114365aba172ed3867bdf9a4b967f7e997cb91ff
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.11.self_attn.k_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0874f2b9ae577d4ea27c94af294163a2bb2ee24b345cd7a9cf1850cdbd10bda8
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.11.self_attn.k_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.11.self_attn.k_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.11.self_attn.k_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.11.self_attn.k_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.11.self_attn.k_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fd7693c6d49658ff06fe9610d50c371098f5434b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.11.self_attn.k_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:307bec166071cb6b7008b90eb27c7ebc7aa19060b97c7a55756c88df4e85cdf0
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.11.self_attn.k_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.11.self_attn.k_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c71acf973036e0d24b69a7f7237bb2a6fed095c2
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.11.self_attn.k_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e61a53b5bee8f7ad25d96332706a8147b611f8b49b9d53b69b4fd7c75d953e51
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.11.self_attn.k_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.11.self_attn.k_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..efe4f257a5865654f54d6fa0fb851cef778a5fb5
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.11.self_attn.k_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3e40b5d1cc22b429784698a611142201c00cb6a696b8df7e0111277e58769e67
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.11.self_attn.k_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.11.self_attn.k_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.11.self_attn.k_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.11.self_attn.o_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.11.self_attn.o_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..485c9696adafe248b9aead70882a905bccd0c06a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.11.self_attn.o_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c006408fc5a915d9c6fa82b6cf2e2460046cd3b87aa6a2a1a6025ff08ca51285
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.11.self_attn.o_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.11.self_attn.o_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cf47ef513cc8827c1a89c67d2d269ed62c482fe7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.11.self_attn.o_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8a26fb081631e06b23cd6def8d43a3043075801b06dd5944b0f260b06247af17
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.11.self_attn.o_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.11.self_attn.o_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..689dad76580c5651ff635f8f008b01f0d94bd5de
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.11.self_attn.o_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aed32b748af663c933d7d5b29cb2ccc0cd442788c820d871dc9da9da8416fa49
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.11.self_attn.o_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.11.self_attn.o_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.11.self_attn.o_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.11.self_attn.o_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.11.self_attn.o_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d0fc89ed0b21e72bf9894e906830525edbcf5e35
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.11.self_attn.o_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8c9d04b9c6679a817fa679c14a724ed9270dc6ee711d73ad1a30838d3044f5bf
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.11.self_attn.o_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.11.self_attn.o_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..44173e71fbf63dd6e26f68e9cd0e57097c11611c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.11.self_attn.o_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f63e585523162cce81575b92abb6b51e62883b088b37dd50302443930cc8ef2c
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.11.self_attn.o_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.11.self_attn.o_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..158f17c6a39a5e74c5770e4cbfe26c3eb26f0fec
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.11.self_attn.o_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61343134862e3b9a82048e26bc649a4168884148c3416d098b708abaae1e7aaa
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.11.self_attn.o_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.11.self_attn.o_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.11.self_attn.o_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.11.self_attn.q_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.11.self_attn.q_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3bb9ab40a7e42faec92107e0906d9f0a41ded70c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.11.self_attn.q_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e18a953b35740e87dae31f29e7ebac4d48b819b2d6df1f24a51ab09b2bd3e47
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.11.self_attn.q_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.11.self_attn.q_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..82133f237c2ed1725f4ad6eea3391ce4312eaf5e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.11.self_attn.q_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c7ebc24480d1eaff6289b102ba2845b7f8a42954bc8013e2e8b797216b11a496
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.11.self_attn.q_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.11.self_attn.q_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4e203b09fb2fc749dbf197de66091df5cf9426a6
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.11.self_attn.q_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:130bfb034d76ecf06da956b7733d2c88aa9ce7142a998fb1624a52e739799e8c
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.11.self_attn.q_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.11.self_attn.q_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.11.self_attn.q_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.11.self_attn.q_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.11.self_attn.q_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..df6811fdf69d46c4cb376585509caf902da29163
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.11.self_attn.q_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d623ef8a93424489b42608197e0ae5a25673a498d68c374eaf6484b895b6f32e
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.11.self_attn.q_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.11.self_attn.q_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0284a144c9f06a687faffaa2105a34ed303e110b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.11.self_attn.q_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e6940ebaa829c9c77dc6d1aa1d7502f1c5e77b1e0b2faf8f3cb90eb387b7335e
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.11.self_attn.q_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.11.self_attn.q_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..02e2af0faa8425a48d5c2406376dec36e6bac28d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.11.self_attn.q_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7f33a5558356c463146cd1ad5ca5b407220ea4b524e9e2ffe77d7bcf55edeaf5
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.11.self_attn.q_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.11.self_attn.q_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.11.self_attn.q_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.11.self_attn.q_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.11.self_attn.q_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f96462c09ad06cdb5eb2785e1720a6e77f0913f8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.11.self_attn.q_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ee58cada799079d6328d07e6e167e3b7bcf5aa84997cee54f52cc184cf8b9b40
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.11.self_attn.q_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.11.self_attn.q_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..733fc7a5209dc37d0c608bfff8eb277886394e44
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.11.self_attn.q_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d5b4b71a95c0ed55f78bf0b06577633fde67649d00da6aec95aa2ec8d18746ce
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.11.self_attn.q_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.11.self_attn.q_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e42e95d05455d81af5ff4d25686da50da3390c54
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.11.self_attn.q_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d45aff269442cfc1fc357c577341b7196d67dc0394da97407d773f9a3c1ed076
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.11.self_attn.q_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.11.self_attn.q_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.11.self_attn.q_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.11.self_attn.v_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.11.self_attn.v_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d53d39a99906d8c1fa371b1203f3c8f9ea5d5471
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.11.self_attn.v_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2f6465f703147111977015280829b1975967c9146c0f5ce3dc8c4eecda11058e
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.11.self_attn.v_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.11.self_attn.v_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..40944a48eab2df3bfbfe5aae3bd51fc3e15d5148
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.11.self_attn.v_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:521140db1c610d3115f13eefba6bae7417fdff6926294e7f01efd4e33e6cb21d
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.11.self_attn.v_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.11.self_attn.v_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ecfc4d6ac4072d0c2e46d517ccd479fd633bac96
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.11.self_attn.v_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:07344eafb304293d8f7443b24e414c12f11a1125d1aef5fd97890f3aa68e86ee
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.11.self_attn.v_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.11.self_attn.v_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.11.self_attn.v_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.11.self_attn.v_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.11.self_attn.v_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7a494ccacee8de211be755af228cf1b3a9676f1d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.11.self_attn.v_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0cfbf2ac06c82b7526ef0abd036f89415f9de144d2adb7a5b0d0b9aec92d9ea4
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.11.self_attn.v_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.11.self_attn.v_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0a59dc30e48bb570d299cc8df563933fb7bc7dd4
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.11.self_attn.v_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aaea9009618bd5fec24d9735f1dede4f223230bfcbd2212a2fd8ed3bcb45ca6e
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.11.self_attn.v_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.11.self_attn.v_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5762f600bd965f7c47d958dadafc57bcf9bc7643
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.11.self_attn.v_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ec573abb8d7c1afaa7dc6437465c0a7e407a7993d559952ccebff2098ff60553
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.11.self_attn.v_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.11.self_attn.v_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.11.self_attn.v_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.11.self_attn.v_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.11.self_attn.v_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2f2d779ec310c9e8d77b472c44767638a7d6d8e9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.11.self_attn.v_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9f30a674c4a7355b787f123eb664daa72233458c8800c042e18b074f5c93ef7e
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.11.self_attn.v_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.11.self_attn.v_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..70c52c26933ffba10d8c24f2a06f034cfc1fdb53
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.11.self_attn.v_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:19b4d5ec152ebd0f5961f3ce763d76ab21801a252f56303f9cf6c412ac4d922c
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.11.self_attn.v_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.11.self_attn.v_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..00c4107a2f39b7b4c5ffc0050d3f51ddc38134f2
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.11.self_attn.v_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:37acc0d30569fd9001d6ea762100734572792e2b8b0b95bd8064a037634aa637
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.11.self_attn.v_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.11.self_attn.v_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.11.self_attn.v_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.12.down_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.12.down_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..baca699dd27de2f549662c8c9e045a053e1dff9d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.12.down_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:628de15aae91d13d292b40d505aa7849387f23247efd53dc3997a8f19ee808c9
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.12.down_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.12.down_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5c96bc516a49ef55195f324fa5c40a4f8bb27458
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.12.down_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fa09ba0a079822723e2a796e6c058b16f69498e2251a79cc276537f7af437f24
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.12.down_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.12.down_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4285fe3588bdddcf238e9025401bb0374095485b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.12.down_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f9cb730a5a50220cd49bc8b512898eff41b38f95895ce6628b57059b3154f26d
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.12.down_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.12.down_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.12.down_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.12.gate_up_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.12.gate_up_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..12f036551056e991662403b677c0e73b981b2fa1
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.12.gate_up_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:938f6c3d5dd5add37012bb5ec331ddf7c751791b80c6b3eb3387fda2f711e726
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.12.gate_up_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.12.gate_up_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..152e412943ccff13d8a1108c147deb01e18c6397
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.12.gate_up_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f8ede16789818f3d011a6d99262b54ea43bc545227ba3678ce92350849489a39
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.12.gate_up_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.12.gate_up_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..10d936fe29751b322673a7bd0731bb251ade2015
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.12.gate_up_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d496674fc6230141b761818e560b6bc923cee96257000962db28afd9f5fce3f2
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.12.gate_up_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.12.gate_up_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.12.gate_up_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.12.input_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.12.input_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a50beffdb71631d5970fe47e4ad56902777c42ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.12.input_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c0df2cc292c76a21d9e5f2d724f301d4f12b04537aa67d53829c23e663b70443
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.12.input_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.12.input_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b8c0bc919451579d02ebb21c9bc1123d045fe8c3
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.12.input_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:95bb25a5d92678372e3c701f190514d907fe860395a68d30f8519d9fc1bd379b
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.12.input_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.12.input_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2aba0807ffc333e75ad55e59a15237b13c8ec062
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.12.input_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:80f0c3c0325abea9baf166d9687ba448b9bad4e6736ac86cc3bdccba1e387e8f
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.12.input_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.12.input_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.12.input_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.12.input_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.12.input_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..abf4dede29d0251b7cf7943fb74ec2231e024fb2
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.12.input_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c88194aa0336f177ec35a5fc44e77766c38cbaf00c8982c504b108447d87fc1c
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.12.input_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.12.input_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..caa703ac58c5d2b254e56d86dcc065cd9ea18bb4
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.12.input_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6fc01d1caccd37a2b33ee289e3c935bc12b6463e0421b9cd22483c0c941bedd8
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.12.input_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.12.input_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..236fed9223671466672f05ee281f1ac4e2bc374e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.12.input_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bb97263b26561d28e2f8a15a7b9d0da12ba6fbd286a2810aab84c308df8f88a5
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.12.input_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.12.input_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.12.input_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.12.mlp.down_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.12.mlp.down_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5c145ab8fcac2f78286607c28e1367ca623b04e8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.12.mlp.down_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:86f317c86ce19e83e5d7eadd906cd61d354601872f96197568fb121d61a29c75
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.12.mlp.down_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.12.mlp.down_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..742e39718f88165094d0b3e3599d135610f22625
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.12.mlp.down_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:42cf759ab63409ba1a83505b45c92d164f476c1e485e867f51b8449728f1981b
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.12.mlp.down_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.12.mlp.down_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6d3f0c3160352acdd43bf9861ab186e0520f2320
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.12.mlp.down_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:865f627548462912b8b6341fefd5cf0edd5bf100c576d07566b3fa9cacd18656
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.12.mlp.down_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.12.mlp.down_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.12.mlp.down_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.12.mlp.gate_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.12.mlp.gate_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..210f3f40663f5f9bce5928fb58fcd96267925f5b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.12.mlp.gate_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7e399a602a16dbaf7529f144d7423a12a85b463e46cd4f90dd634c36cc841d19
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.12.mlp.gate_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.12.mlp.gate_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..02a9c4c5337d6d20c401a02446c9387f99228cf2
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.12.mlp.gate_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aea170f5944cd159928020a8b38cb15bff4908af62ce5627067e6ad9db515981
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.12.mlp.gate_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.12.mlp.gate_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ddcec084c37785f436598e85219ec50333adcc98
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.12.mlp.gate_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:01b1fce21413f40650aa2c6588d903ef1a6faf713f1ed7638bbe65752455147f
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.12.mlp.gate_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.12.mlp.gate_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.12.mlp.gate_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.12.mlp.up_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.12.mlp.up_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e034d678f2b5274ca969c34dbacd7ebb1d5b7f9a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.12.mlp.up_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3f455a61980f1f9ec5b01d852f38d832c513a5d9bc9881a7dea3a197feca1d2c
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.12.mlp.up_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.12.mlp.up_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..51923f3988d425900ef248a344e7f189cf162875
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.12.mlp.up_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a3f21bbebd781061ba4045cb9d0207d63ef8d1bda73c884511268710fac355c1
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.12.mlp.up_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.12.mlp.up_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ca97a05cc37fd3da35757440c1d2fd45ce1920d5
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.12.mlp.up_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f1f41f33e23921f16600c8a9b86175539054d538e3b1ea1f5d7c6c325d62957a
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.12.mlp.up_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.12.mlp.up_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.12.mlp.up_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.12.post_attention_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.12.post_attention_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c045234fd8c0a030d8633e024fba83c58d1f4887
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.12.post_attention_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dc254ee91b45994f1d240de7508d31fbee064faf0a1e8866b8eef605346632fb
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.12.post_attention_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.12.post_attention_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d315288d02422e3307e173723c5dac645090ee18
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.12.post_attention_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f7599e19e1d81b59e6f014f3e9001377f1c6c6eb258e0ffe07c995d6cb9d6ae4
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.12.post_attention_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.12.post_attention_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5c99ea68c3ae8688975dfd195eb75c3e040eb50d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.12.post_attention_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:641ab934bc701fdf275143c02f53194d3edd8b81fba2a49bbe277efe54803e56
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.12.post_attention_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.12.post_attention_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.12.post_attention_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.12.post_attention_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.12.post_attention_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..171421741e34e514a7d363bae2abede3ff4ff927
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.12.post_attention_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2539b356cba1db2915891ae1c8ce29b8b04bf9bada697100ea4452812bcaf398
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.12.post_attention_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.12.post_attention_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8c6642492547b8f32feccfb0a206569d60b6928b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.12.post_attention_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a9c1f659eaa768078856065f66e83314fabc81585a4e2789bf733bc999a29e70
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.12.post_attention_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.12.post_attention_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8711b49793086d3849e99e53275e5c455e16a302
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.12.post_attention_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3676ddc51104382cfdabaabb2f81703aebf5d8303ce19600ced37bbe34647fad
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.12.post_attention_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.12.post_attention_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.12.post_attention_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.12.self_attn.k_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.12.self_attn.k_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8221e143a9beea71c14abcf9c176a434e803f360
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.12.self_attn.k_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0de56ee97e71e53826663386286f0daaeded2008e8898bc9a5116a0a6c5eb65c
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.12.self_attn.k_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.12.self_attn.k_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4b79c82de8401d46962ae10504dd710b8b481154
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.12.self_attn.k_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1e573d6358838f93c43e876bf25182ab6014cffe0bd5b57e5863964bcf4c502b
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.12.self_attn.k_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.12.self_attn.k_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..36b1b56bb3f240aaa70ce46827fe826efe68e8b2
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.12.self_attn.k_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a95cd631bf37cdb4307a5b4daf768f920ab15990b6ddad146ea7139f5b70cccb
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.12.self_attn.k_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.12.self_attn.k_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.12.self_attn.k_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.12.self_attn.k_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.12.self_attn.k_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9317539c6ff8a537091dd9ff85202d78171c6e92
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.12.self_attn.k_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8045bed75bdf0d38d6b302c9afdfd2af47fcda70c9127c99327d06d76b6b4ba2
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.12.self_attn.k_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.12.self_attn.k_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..67c51900fc16812f1c7866c14769b6fbe70f39da
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.12.self_attn.k_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:609d1205d20c25285cb3576f8878230312cec37edbd6a048574331331495070b
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.12.self_attn.k_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.12.self_attn.k_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b210c2bf6c4084872b93083e4e63880e1a6e706c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.12.self_attn.k_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:89c98b34a6a4a80ed197abdc61d9c39fe8ea8a3185ecbb28c94c1f3f96b37731
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.12.self_attn.k_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.12.self_attn.k_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.12.self_attn.k_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.12.self_attn.k_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.12.self_attn.k_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cd6c816904cbb6a33fc6437e62e9a050f5215644
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.12.self_attn.k_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e918caf98779c3e409afb127ea553379502e378e6f9d70309d5db1cd9823ccc4
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.12.self_attn.k_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.12.self_attn.k_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f521ecfaa9d46b45848490159e879505d951fda4
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.12.self_attn.k_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:35821782facac6acc4e8dd6deaafbffdab284ab75bf0802cbba7e804aafc5ac7
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.12.self_attn.k_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.12.self_attn.k_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..081cf3a5efdd8f29b86e79d6914eb7eeab250e66
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.12.self_attn.k_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d47cba13f10a77dde471ab557a990b0452e2cb45ecd39ecfd2569e0b2161921d
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.12.self_attn.k_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.12.self_attn.k_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.12.self_attn.k_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.12.self_attn.o_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.12.self_attn.o_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e2131d4dbc19f70b6b14ebe54faffa2060547164
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.12.self_attn.o_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:616201c2ad2e28701a499c36e07b980ed126cdc65a98f584fb029526b6c30338
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.12.self_attn.o_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.12.self_attn.o_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..36d514ce76f3cb4a08d7979f016346f23be4bbcd
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.12.self_attn.o_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3bcb586674a42402162ca1c9874db4210d800ec598ea21489f1c73941c2b2dfb
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.12.self_attn.o_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.12.self_attn.o_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..723c322084a3efb19cfde36807af273043a811a4
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.12.self_attn.o_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:83c75e8c1ef285147be8e94b44e7c69d5da49585a143579dc4d36edcd19f897e
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.12.self_attn.o_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.12.self_attn.o_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.12.self_attn.o_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.12.self_attn.o_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.12.self_attn.o_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..486cdd6168069de6aba7f8d4adbcc84be2f4acc0
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.12.self_attn.o_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0659d4e8d0e9dae5774000fc2e125b71dc9634ba1f32ca085c4a04a49b15ab67
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.12.self_attn.o_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.12.self_attn.o_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3fa2cd3c0aead04936408f7cd5abc0baa506aa21
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.12.self_attn.o_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:487e850fb16e5d88f05ee185f7893fe93194fca0641f699a103f26c413cb78fa
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.12.self_attn.o_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.12.self_attn.o_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..206f3f9ab59ef51225fbba34755c8b19b716ef8c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.12.self_attn.o_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:42fb01267d481001c1da5afbc2d61104755f11e59d65332622b37d1a89a049bb
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.12.self_attn.o_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.12.self_attn.o_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.12.self_attn.o_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.12.self_attn.q_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.12.self_attn.q_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..dffc7f1729bdcea0a1cb25909906537dc34f8c10
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.12.self_attn.q_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:71ae28698daab572f17a61b8f27c5323cd96e08357a29601c4e66cab677b6e6b
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.12.self_attn.q_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.12.self_attn.q_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..08bdb0f0e442d29e9cbc94956f82e8bc009482e1
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.12.self_attn.q_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:942114318b28edf1b1ed7ad51ea524bcdf73a3dee90dbfab9ba421015caf0ce9
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.12.self_attn.q_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.12.self_attn.q_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b13fe42bdd8ad26fbcbde3e76396a60e431c9475
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.12.self_attn.q_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8692255d9070b569909e633d7dc635624205c6f0ff41f522941d7125fe64872c
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.12.self_attn.q_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.12.self_attn.q_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.12.self_attn.q_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.12.self_attn.q_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.12.self_attn.q_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2bb1f36c3ca6034ee1959a1f236969244bfeac81
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.12.self_attn.q_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0d93a00ae06bf8548976acacc26544cabe2e4ed32e95a01220ebfae03f360bf4
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.12.self_attn.q_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.12.self_attn.q_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..91ed538b6797f255f2aaeb6a2eabfd0d7260aae3
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.12.self_attn.q_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c473ce4870c070942c169a41bb4dfc02d424883b6a950bf20581546df51d4d21
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.12.self_attn.q_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.12.self_attn.q_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9a6a98b107daecd2873c18a00577f0bcfae48631
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.12.self_attn.q_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4039257a41214c4d848234d4623b9345f332df6ced93dc9e8a338f24dae97c21
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.12.self_attn.q_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.12.self_attn.q_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.12.self_attn.q_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.12.self_attn.q_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.12.self_attn.q_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ad62fd38b04f6cc10928a77a9c099963c98b1e37
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.12.self_attn.q_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bb2a3a31534c38c00d98be17371fd090a0ebdab43784d69b1b08718742ee37a7
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.12.self_attn.q_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.12.self_attn.q_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..bd6410c0d97bc2f44998353a95217d7fd39bf719
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.12.self_attn.q_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e2e8884db8780bc36d6e7d80baf84bf0dd378bc88985494f7fa50cb116a76c69
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.12.self_attn.q_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.12.self_attn.q_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0e3ec55ab821b171ac178b027b38a5042b19120a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.12.self_attn.q_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:93fbc8342124471fe0d76fa1a1c4833c18c5894486b33affc5a792ff7fd242b2
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.12.self_attn.q_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.12.self_attn.q_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.12.self_attn.q_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.12.self_attn.v_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.12.self_attn.v_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6075379c3465593ae4e19b1b96b32fe8beb0efab
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.12.self_attn.v_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b5959523579bb897bd89eb8029d225e110e52306c8338c54b5b0a003230be2f8
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.12.self_attn.v_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.12.self_attn.v_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a03eda7c2325bd04eeacbb961d7f2dadc6bb1da8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.12.self_attn.v_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aa110fd6b35d93caedd9c484bac848faaa59d34ad82d94da3eebf1112c767932
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.12.self_attn.v_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.12.self_attn.v_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7a06c715b68151ecb0c2863adac836dae6a5cfc9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.12.self_attn.v_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:43d91e15372ea2f48d4c9cf35921b1a284acfa4f035b5639fa08bcb567cf7c1a
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.12.self_attn.v_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.12.self_attn.v_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.12.self_attn.v_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.12.self_attn.v_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.12.self_attn.v_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fbf866ed47f9b1fa916bce87d3d3cd9ed99be1f0
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.12.self_attn.v_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a0bbb7abce550f25d0709f21709ca873c6ae71c45cf59cae97b46e9185ff5a8
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.12.self_attn.v_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.12.self_attn.v_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..17eae49e666c9d663eff4f5f94e520ae003c9719
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.12.self_attn.v_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1c2d03085a5c10b8be8cf27211a3c6adf5d88f13f33b7f53dd545e0e135f842b
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.12.self_attn.v_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.12.self_attn.v_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4f8d6529f641f99bcbcf2f9cde0a38ae2e57e0eb
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.12.self_attn.v_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:514b67721a818309cbd3ebb14651412e32c638d20306c3f43a39663f0047763a
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.12.self_attn.v_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.12.self_attn.v_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.12.self_attn.v_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.12.self_attn.v_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.12.self_attn.v_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6261809afdaa19d997ecf3db28ac29b4f79f1cc6
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.12.self_attn.v_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:338c45aedbb03124ce13db4fcc0296eb1b4481aac25b7ce833e8753c08ee2c51
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.12.self_attn.v_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.12.self_attn.v_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..527325643444ac122a9a33967d8b456d821ab1e4
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.12.self_attn.v_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:efd35de9ad13fbadde168567f481e311303fafd865683dbf798e11f696691fb4
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.12.self_attn.v_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.12.self_attn.v_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6ac7e1ee9a8817ae0f93be241b0a5abe16244b36
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.12.self_attn.v_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:992d65b0645b75a180d70e1998e9af1d57ed83130ef954fda37328dd3e2fb361
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.12.self_attn.v_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.12.self_attn.v_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.12.self_attn.v_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.13.down_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.13.down_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..12641382679617bc8a340ecb3589e15ece8b5865
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.13.down_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8e5165d004d39e1e7d11207618583e886a6990129fda5034340b177e0d95c92c
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.13.down_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.13.down_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7ff36126a151fdd57c834bfe9c62bfde535af1d5
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.13.down_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:58a94ada71caf75224870004c7b02976c032a92e365d94709c4da73111f38ae4
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.13.down_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.13.down_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..821384ae433fab039a23378c0b1aa49c8879d89a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.13.down_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4655ef6797a49d07e1560a75b26b0ac61424acc58cf7ea865f302bbb8a7b6c9a
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.13.down_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.13.down_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.13.down_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.13.gate_up_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.13.gate_up_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..91666a5e4678a935e46c125fe6e02f7ca20cbd1a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.13.gate_up_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ec38e47f968a311513da33df5bfc97bdd3a0626d4147d7cb4dcf8e512e27d01d
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.13.gate_up_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.13.gate_up_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4932c2f8d97fd449a613b1b640088febff4f7fa0
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.13.gate_up_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:85a980a6f4636cc424054c286e5d7921520bc6ac02e3a3dae0c79addb6ccca5e
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.13.gate_up_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.13.gate_up_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cb591e9447a85f4b74343b2b3f5c2cf3eb3f9be7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.13.gate_up_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8aed0ed61f5387a943e9e0f0c659ab9f6777f571af8665b7578bf6941fac6481
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.13.gate_up_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.13.gate_up_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.13.gate_up_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.13.input_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.13.input_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..db4649ae9ef31da6e65e29708f7e96b0e09621ea
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.13.input_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8008a2fae33cb61b3acf9f8c00bf93a7c42a8f3b9e6bf2b93268ba4ee43748e6
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.13.input_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.13.input_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..87c24eccbe42de1615c3ef539ab5fd26529d331c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.13.input_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cf8ee6955a098406470f78b90c6bd45350e1668d7d38dc14cc8b669b674d278d
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.13.input_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.13.input_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..19607f19f5ae751380e05f625196f6055582785a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.13.input_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:604fb96f4a6cdc56b28b9379e66fd8965604e0add8e31e9821645037132c8b6e
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.13.input_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.13.input_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.13.input_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.13.input_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.13.input_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..497e51c4aa9ebb0db578dd81c0489ee90b21e264
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.13.input_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e673ae47c45d3722e8e017f22396c73f0c4e62473f0f0de8ecaed6b626e7a477
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.13.input_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.13.input_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..22658e7e0b31765fe697c920ba9d3aba911a0ce1
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.13.input_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9cdcde9ce58fb17135797d63c735ba2d6a2d575c1460afa45fd3b579b13bd44f
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.13.input_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.13.input_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..025b0cb3f2c3922ce82bcef9c00b2ce5060620fe
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.13.input_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:51ca99dcfb7b8cbe04c181c3248e73ecb6e982fdac06dae968df788cd80a168e
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.13.input_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.13.input_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.13.input_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.13.mlp.down_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.13.mlp.down_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..00a9fa63c14288f659733ef3a19b1da6a0cce2fb
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.13.mlp.down_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d9d0ee046b2413047eed251cee555a9b3dc99790dd3e8b37f25adca6e1f67f4a
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.13.mlp.down_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.13.mlp.down_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..762c4280c624a837b3d0ec9aca0d743536fea76f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.13.mlp.down_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f57acca0da84c9ebc328290c1ccd7add3584a1747ca20802de11fcb3231e56e0
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.13.mlp.down_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.13.mlp.down_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f3a15e70ef84266fbaede9c7da638f02ead411d3
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.13.mlp.down_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7e3897944bb8252b849a0cce645f608e5a320bb2246a8b245b54e2d1b8740564
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.13.mlp.down_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.13.mlp.down_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.13.mlp.down_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.13.mlp.gate_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.13.mlp.gate_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..83aab108cf74a50b16ff200be30009b57f54bf1e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.13.mlp.gate_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e40f690ac19e6f760ad5216e30f070d86cb5c8c6055c1f9624f4d56f41b1b55b
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.13.mlp.gate_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.13.mlp.gate_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ba2d43fbb177bc2f097a9cce75c479b97bdcedcb
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.13.mlp.gate_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:60c17ade42e93304be30707a22dc7bbe24f1a6d9452ccf6888bc0bbb16a5a15e
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.13.mlp.gate_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.13.mlp.gate_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4198ee6f13db13ca6d9b11702871089e5c5f00ef
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.13.mlp.gate_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:86a1a6fa6bedcb1d948cd931dbc1f0abcf8fe4b515727eabcbe5bdf0aee15fd7
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.13.mlp.gate_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.13.mlp.gate_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.13.mlp.gate_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.13.mlp.up_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.13.mlp.up_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4500c39ba7f33cfca24165f7af58dd2c8cd19271
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.13.mlp.up_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:508bf1e5d92f2648c0f0c86b1f9675ebb6c9abd02c4eba4822de79941e5a24be
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.13.mlp.up_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.13.mlp.up_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a2a7cfe9103f91865e3f51dd89789c46adcda23a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.13.mlp.up_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7a9b6fcba552c8f4271228d02a9a7cc342e679bd5a4c1c8a9d9d64c28e4cf40a
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.13.mlp.up_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.13.mlp.up_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6b97df1de4a88a93418881432fad5cd934ee8ef6
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.13.mlp.up_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4bffb159d4152e4f3bce2c7bb55dc5a07252bca77990bb2080bd8298ee423e12
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.13.mlp.up_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.13.mlp.up_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.13.mlp.up_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.13.post_attention_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.13.post_attention_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..73f4e5480fa2f82122da727781ff4eb84accbbfe
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.13.post_attention_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:362f8a57bcdbf9ea97d3978619f76d1fe22b93fc28d216c7ca0d106716f6e095
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.13.post_attention_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.13.post_attention_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b4c6194daf7d1640e1879bc26597b50ef3435612
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.13.post_attention_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3056baa20802975fb2cf590f5cf4527d5ec873d986fea3984df4818711c33582
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.13.post_attention_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.13.post_attention_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a09d6c326c9a72de99bee4e81c0f6c02063b8e15
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.13.post_attention_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29e24d4836d98a2ed11b35e8fa987cd8cbf3d519425cfce2e772a37b1fb0bc66
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.13.post_attention_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.13.post_attention_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.13.post_attention_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.13.post_attention_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.13.post_attention_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a0fcfca034bf8337eb9a91c7a48ae7a18edfb6b1
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.13.post_attention_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7b8255b52df242b355258733fa4050f6b7019f7f935f6018a7f4e1d6b9ae23d8
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.13.post_attention_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.13.post_attention_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0746cd47a2c569c33e0867b5b202142c579919b9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.13.post_attention_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aec573e0f95618cae3aa0be34d9b9b06be3a2994726c11d1a4e26325c7a82461
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.13.post_attention_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.13.post_attention_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ea471ed8cfeedc7edfc3187987a8d43cbf401fdd
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.13.post_attention_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:36e4b2ae5f93c074d5a09c3b79aec0e1e90de22b32ea2a1ab0a085585a739c74
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.13.post_attention_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.13.post_attention_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.13.post_attention_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.13.self_attn.k_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.13.self_attn.k_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..469bbc2563cae94d97d8f921ca9bf1f209654d05
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.13.self_attn.k_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c456c2be71ffbff09dbc268645576a3be1af239cd10fbf58168170673c2d9dc1
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.13.self_attn.k_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.13.self_attn.k_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fdbc05894ed0391babc4c6414cea48d77872aa28
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.13.self_attn.k_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1bd1040cdf32ada6ccaf874b44d885b1bcddbd7ed909922c1b23acbb7fedd3d9
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.13.self_attn.k_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.13.self_attn.k_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..284c56e89a7704d73be0e9e0bb8bce6eccbbdfa2
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.13.self_attn.k_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f6e625065d723420305c917e966b8592ed58309bb9bc0ec1bd75f664cb97771a
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.13.self_attn.k_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.13.self_attn.k_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.13.self_attn.k_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.13.self_attn.k_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.13.self_attn.k_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..63807ed0dd0529cdab7a2fbcc70cbe8fc867db31
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.13.self_attn.k_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2ecae7c6f7b4b4d6b008a08f35d9636478935fa9ab9226be410689f5a15700e0
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.13.self_attn.k_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.13.self_attn.k_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5631a171664c84c9a39eb5e8cf2929c6abf1c640
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.13.self_attn.k_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6da67578589c6a641cc6e337a0059f1a52966790ef4ec405cc9a839ceb26b949
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.13.self_attn.k_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.13.self_attn.k_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..415b2aa7384852d8df4d3c02f5ae1f3e9f04e1e9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.13.self_attn.k_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ccab3b4c3e0cf79c5c637bf5af6deb846b706029e939a7f40ccfca673f645c15
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.13.self_attn.k_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.13.self_attn.k_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.13.self_attn.k_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.13.self_attn.k_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.13.self_attn.k_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..eea759cd73fb723cdfbf586b3584732ea3985659
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.13.self_attn.k_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eb641112b18ae41ea5f73812d92f0520905a87a1b7d4e65f3dc7cad0d46f46cf
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.13.self_attn.k_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.13.self_attn.k_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6feb4df7316598d5ef1c85c4531454512eaade51
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.13.self_attn.k_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:46edcd1aa2122a6b291f2139cc9224d9d9e4bd4a6420fdcad40d4c04315711b0
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.13.self_attn.k_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.13.self_attn.k_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8d17ed4253f2aad91d85b5d592160d5304517901
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.13.self_attn.k_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ff9a48dd9cb235a6887cff43b9ee86193336ba9dac4d59d41529d39969654d02
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.13.self_attn.k_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.13.self_attn.k_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.13.self_attn.k_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.13.self_attn.o_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.13.self_attn.o_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ea4669006832a1a1f252df63772842a3ae823b22
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.13.self_attn.o_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:960db9061671cf9ea3d8ab97f771993ebaddfce7bb6ede29218fe9f872ceb705
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.13.self_attn.o_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.13.self_attn.o_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b740a5e6cc5c6758521ff887dd1b4d28649db4d8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.13.self_attn.o_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b9ab734422c8362e46bb5c9752d4134a1e4033a08fb2ffcb0d68726c24842dcf
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.13.self_attn.o_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.13.self_attn.o_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..39ab3f07f5918ceb696e7f85deecfd6fdec51975
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.13.self_attn.o_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2c5ac8bdc2de598a8a816a84a3a0cf7614a28435ceb93a5632186f577377c662
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.13.self_attn.o_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.13.self_attn.o_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.13.self_attn.o_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.13.self_attn.o_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.13.self_attn.o_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ba4bda7bb23b10d70ca7741bd10e84838c7ec45d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.13.self_attn.o_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f63830dc70e16cfd4cd3c994f15c34f9788a6ba82eb5010a2462f13e3f1a22c2
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.13.self_attn.o_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.13.self_attn.o_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a071bb0e716a828975f2c6d09893a3e2c88a7a09
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.13.self_attn.o_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e276a7f77b3fbf494b963db590a71e875750abbca03387bbb0d2c786e0416d50
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.13.self_attn.o_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.13.self_attn.o_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..febbe6eaa6d6d0f2a2d4379627b8994f08f08160
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.13.self_attn.o_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6f586a8c7006ef0b572cb89f363b851de1ee0872fcf4cadf3a19781bb0ff1a96
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.13.self_attn.o_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.13.self_attn.o_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.13.self_attn.o_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.13.self_attn.q_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.13.self_attn.q_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6ec2b22ec16e63b91945e0f69205bae2edc4260c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.13.self_attn.q_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b8dde54228235843587d8aa372aa66a307bf353a5d95696ee9937c33246685ef
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.13.self_attn.q_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.13.self_attn.q_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..36e0e4b13f7f3b80f0c3f7e1c3a91cef057fad3d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.13.self_attn.q_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:233b6ca14f51b885f853326d95a2db70b44fe50884ab685632d6702d312ad5d5
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.13.self_attn.q_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.13.self_attn.q_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..67d272c6d83002f257096b75235d96ece5297226
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.13.self_attn.q_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7665f61be4dcc51f0e449e341fe23cdfb8498b2890ad430db5fd670e16157725
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.13.self_attn.q_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.13.self_attn.q_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.13.self_attn.q_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.13.self_attn.q_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.13.self_attn.q_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..24a4973c254a494ed58a626faaf12c3ee26b83f6
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.13.self_attn.q_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:97e6d0249020dc16172b077cacaedbbc33fa589271f4d08583955b42c786ba13
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.13.self_attn.q_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.13.self_attn.q_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f747653fb61a7073df4c6c605f7d80c2ab04fa02
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.13.self_attn.q_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2ad6fcf11e8a5d07e744bf5e7e80e091a1ed59c8aca7b070d7b607bda51896fd
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.13.self_attn.q_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.13.self_attn.q_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9af8b796ec8b32cdaeff93797d16a6fa94ef696f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.13.self_attn.q_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2ed1634d1e8c427db8462162d5b68c8343aaf6f80a4031a16ba2e325dc76c380
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.13.self_attn.q_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.13.self_attn.q_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.13.self_attn.q_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.13.self_attn.q_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.13.self_attn.q_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..da5e4d4ff7bc8ae1bb9267efbcbe4768ed56415e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.13.self_attn.q_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:963cf91f60b98e8643c5055cc15cd104a0b1c101fcb8ffb5aa7468eeba9fc29e
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.13.self_attn.q_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.13.self_attn.q_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fc7023d867aeb03b7d388a3abbfe093fb751d319
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.13.self_attn.q_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:40bc349214c27b05c33c5ec082c0693df63bf5373373828a096e6051afa874c9
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.13.self_attn.q_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.13.self_attn.q_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..735e3e74ea167d9a4a00c353ffb190be3a12cffa
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.13.self_attn.q_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2183cb7a8219eb4f99ffcd06a4f2df39bb225ad83724f9cd96ee8c7094d3ba37
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.13.self_attn.q_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.13.self_attn.q_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.13.self_attn.q_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.13.self_attn.v_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.13.self_attn.v_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c0ac265155a466477c0d00339ce02d7ebc5c69f5
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.13.self_attn.v_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:88f56c16731f27894c43b4cd6ceef2f6cb1fdd861cff49f93d6db6bb4eb12d08
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.13.self_attn.v_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.13.self_attn.v_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b40410c8cdbda94ef1c9fbbbb1640f5af990126c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.13.self_attn.v_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:82afcf3374d0166a145221239bc284ad3999e658cec5c9683d5e39427af3fbec
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.13.self_attn.v_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.13.self_attn.v_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..67c02827e96c5630bd7af50c5a4d773e2ab32954
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.13.self_attn.v_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0600e3b7d5103787949425fa191f66ebed74909ac6a60300d842366a45952e45
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.13.self_attn.v_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.13.self_attn.v_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.13.self_attn.v_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.13.self_attn.v_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.13.self_attn.v_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d87de9f1d5d229ca472e5106475ad517c1f1a64a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.13.self_attn.v_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b60f51ebf18d3288739f73dcb9c958ee2353aa835edaa6c43b5b5a8a7062dd11
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.13.self_attn.v_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.13.self_attn.v_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a8033dadfb691193b2727db0b4f569d1d84e5373
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.13.self_attn.v_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:435e12bea5659db586efae9bb8a728b10c76308f1fed487a107d08d4fc26b572
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.13.self_attn.v_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.13.self_attn.v_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b4c385dd67de386596ab0c7e415145bea29915bd
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.13.self_attn.v_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2364242af9053f853e9091b1ffb4190b3ac54d6fab1a6b0318e6468384049f7d
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.13.self_attn.v_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.13.self_attn.v_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.13.self_attn.v_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.13.self_attn.v_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.13.self_attn.v_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..15b550ed5fca3c87dbc72820b406034ae8034b72
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.13.self_attn.v_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f13e33ac12a857cf1e43437f07092f39cd44cb25c25a86d48067113bcbbc5ec2
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.13.self_attn.v_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.13.self_attn.v_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..152664418b2fdee7614707f075d54ba66e6e3e65
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.13.self_attn.v_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:da241a28053ba0603e1cb13e8204833bdf574693a243f6b2218aadc98ed818a9
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.13.self_attn.v_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.13.self_attn.v_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..991fac92ce86de294fd927ba5eea862c795894d9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.13.self_attn.v_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0f4ec876926258889d63f71a3537d6baae14f15cda94d4caacba26d2d17e8a0c
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.13.self_attn.v_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.13.self_attn.v_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.13.self_attn.v_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.14.down_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.14.down_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ff72a69242ec4ece09255beea0231cd76be4a989
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.14.down_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e56337af8e20390152659b2d79f7fefd2ebe03b7b3b24b5e661d49756b8571ec
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.14.down_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.14.down_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9eabc4958d00563605e708071ddcc39db3b7d3ef
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.14.down_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4ad651fd05c22954f518a2d1b5730f04bd8b3ebb66712a3097bae410aa56b9d9
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.14.down_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.14.down_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6399deb10a46c9d7e2d8b93ca75b16ccd68d9acb
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.14.down_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:990ee6d5e59732b0889a36e13cb31d7b2f5212b28f4bd500c584a5d6be0db257
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.14.down_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.14.down_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.14.down_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.14.gate_up_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.14.gate_up_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ff119b2f4f8bb4bc1381caf786846529ee085c44
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.14.gate_up_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:876b66f4bf0edd4565b3a8846cc871e55ee100d241dd977b965fd7975eec058e
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.14.gate_up_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.14.gate_up_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ed7fb6e84833e7e72f46c3895c7664c16a9bb85e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.14.gate_up_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6c50ab97db7e2de63e977e03959e05c94f92f3a330171887cb5404c6e80500e8
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.14.gate_up_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.14.gate_up_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0a361082b07fbfed214b5fa9e22e1f6523a10981
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.14.gate_up_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d7b832e955d097f52f2200efcecc1ff5b7938e11899cc35d07dd95f7915c4aa9
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.14.gate_up_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.14.gate_up_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.14.gate_up_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.14.input_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.14.input_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d1d5e9f7e25c970da29587141912d38ea06b8409
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.14.input_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d51edad24e19c9e09bd4377b28e546c4a9c62168eb4f51a50de1618f3ed86880
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.14.input_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.14.input_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6d9396a9d2177391f161b35cf747ba10697ac4d3
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.14.input_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2074b320196e9aaeb776db70bd8b1ee2ae0729f4bb5105d751fb225f682a71d1
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.14.input_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.14.input_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..affb8a0431239a910def750c0001a8cb40199c7b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.14.input_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4bae7566fa0cec3e288ffd405be3e1aba704b75bd3055218ecb09de954f580f
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.14.input_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.14.input_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.14.input_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.14.input_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.14.input_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d1014bb12621b457a7ab46efbff050bb5336be5c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.14.input_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8208efee28da3213b104aaf1ff2852713d5e88946fbeeb209bcf8f0704cc3af9
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.14.input_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.14.input_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d42ad5a9e39e55384da787969500e511c4e4f4e3
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.14.input_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3a929e59237c00531719b82e19fd947fa08d3435b8150896f6dafa2e6a8987a3
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.14.input_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.14.input_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5802694309d0a139d6992ee3cacaf07f735d3cfa
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.14.input_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bb068be95a7bc35a8dbd215c12f2e3307f0947726999dbdf6485dae8031cb549
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.14.input_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.14.input_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.14.input_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.14.mlp.down_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.14.mlp.down_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1d370621ed6cb79f7dc510091d495aab02003c21
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.14.mlp.down_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:819405ebc929d9319db13f6b9bd30157e6c327c1bf34b709c0b2811648878145
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.14.mlp.down_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.14.mlp.down_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..089c8e2cea094b92b3f3910b85bc103c51c3a9ff
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.14.mlp.down_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:95f3b10675e95ed1d15896b217d705eac88f7b1644f0097872e7afd0ba32bf34
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.14.mlp.down_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.14.mlp.down_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c42268c0e3ca4015bdf454ffd5e8ba34d6ffc39b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.14.mlp.down_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b51878d145761214cc0fc1527fdc28302ee3a9169baca7964115b3bbf1ddef19
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.14.mlp.down_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.14.mlp.down_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.14.mlp.down_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.14.mlp.gate_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.14.mlp.gate_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2a1ac2847810de9a2bda8a9dc6532c5c9bc35c3a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.14.mlp.gate_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8cd35afe8742ed2ae3cb21a6dce5499237608e5146d03d53a61f5687abdfdd2f
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.14.mlp.gate_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.14.mlp.gate_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7b65f0cb763cf510f1bd96546a72a6b449acd75a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.14.mlp.gate_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9171f470dbdc81ccb9c339dc69ef1df4281e319e2bdba24bf1e3a0ad19c69547
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.14.mlp.gate_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.14.mlp.gate_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c8569d963b13955e6cbf376f84783d55a6b2736f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.14.mlp.gate_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6a594778e0aa16a081e0ee34a454cd85b84a87d790558c148a7ef7a4dbb36b59
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.14.mlp.gate_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.14.mlp.gate_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.14.mlp.gate_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.14.mlp.up_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.14.mlp.up_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..97f45a93b5ce6ae6073fb547e24d93177cf95f16
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.14.mlp.up_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0f8a5154a92587294063b527c3825733a155415b3ce8a32837466177894d522e
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.14.mlp.up_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.14.mlp.up_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..34ba94e951866fd8974522bd48dd318221ff4157
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.14.mlp.up_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:09273f52f6863315f745fc660650996bbb38a31e6d5afbce20eaedaa139df75a
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.14.mlp.up_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.14.mlp.up_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fd0dfdfcff7b97184b5c140ca40bc19bcc107d7c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.14.mlp.up_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:63a19eee77758ae369cbef5b58a3023d7fea64d283115897dc613503a8b6c18a
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.14.mlp.up_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.14.mlp.up_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.14.mlp.up_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.14.post_attention_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.14.post_attention_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..70883467a3c17c53a949c516fea05b5750d51eea
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.14.post_attention_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc0d3fbbe2cd9e34e3a9bdf7ef1c39f40469249faa648b11ff01de3b42b6d1d3
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.14.post_attention_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.14.post_attention_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4ccd6f5c052d482fe20479e1be5fb612b932117e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.14.post_attention_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cb093b2a1b808665c74686d693bfee82091dd7a58d00fff5684a2fe5ec35116d
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.14.post_attention_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.14.post_attention_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3d4e32c6a0a13aea2b5dc584f7ba7b65722f46a4
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.14.post_attention_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6df1d08c304350741928160061a03ae87b5e4de0e8dd218774e1b15f129db47c
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.14.post_attention_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.14.post_attention_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.14.post_attention_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.14.post_attention_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.14.post_attention_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b6f7f974989320cf258deef24b40f8f3aa394620
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.14.post_attention_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4294740beb0dbbd30ecc3d39fd61de7d618645bd7fab15b40a1bfda5b45604be
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.14.post_attention_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.14.post_attention_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d4778b69599547667ff190b8319817068c9025ec
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.14.post_attention_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f31d8fcb8a2cc0381e6c09a7be60615a9c0a920125856d0af8346b57e8089abf
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.14.post_attention_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.14.post_attention_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1a738484371c8b449da6c5adcef5231eacfad6c2
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.14.post_attention_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9dc0937ac82fb2069b04adbe7406132641fb0e34cc2040026b9d4bbaf01b7eeb
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.14.post_attention_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.14.post_attention_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.14.post_attention_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.14.self_attn.k_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.14.self_attn.k_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9dbc8a810511c040debd825d4511e18a293bc65f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.14.self_attn.k_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2dc83b7d55ba2924f95c49f471327a06fef3b5a14e550ddc471a56b737dada11
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.14.self_attn.k_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.14.self_attn.k_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9b440c62fdbbdd9d32d43c77c43f94baed6cfb99
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.14.self_attn.k_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:04d7672d269c81bed731f6e93ed007c6f5fad21d29211a77f8904b7a96f27425
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.14.self_attn.k_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.14.self_attn.k_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..25e4e71b218351440720c0509f2b8b707ee9d1a8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.14.self_attn.k_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:81075edd284558302f21a9ae80c534ae6fe6292d3cf24cbc4b08deb307811859
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.14.self_attn.k_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.14.self_attn.k_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.14.self_attn.k_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.14.self_attn.k_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.14.self_attn.k_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9da96961b72196faea94974adfce82da111c01a9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.14.self_attn.k_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7f838196b5d22166d06f9a30cde5758df5bb184c78045b425a83e62640e991b5
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.14.self_attn.k_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.14.self_attn.k_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a7abc630d9edd5844d2767416452056f9a450f88
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.14.self_attn.k_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a981659bed484ab5b929209f6b276cbbfd09635c1ee9300ee9cb017763fe08c
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.14.self_attn.k_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.14.self_attn.k_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..169b180a15b85e4572343c70267256f430d164fb
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.14.self_attn.k_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4057b77e71996c7015fc2ed0c6a1003f1fb92a280d2e5d25bf2a1d670dfb9510
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.14.self_attn.k_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.14.self_attn.k_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.14.self_attn.k_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.14.self_attn.k_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.14.self_attn.k_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4e3b8af3be68bc166d5e281bd0bdd8691e9c0bcf
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.14.self_attn.k_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7c8a21866bd2ed78d4ffa1bfe5f52eaec26ff86865c03346cfda0260d1382252
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.14.self_attn.k_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.14.self_attn.k_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0982d4e858806f7ef7e21258b9f2cbfbf1fc86a0
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.14.self_attn.k_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c7f3a2596a7e382a4f0d1f1c94d157771851c65cabbf23c9b8e50502998c3db1
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.14.self_attn.k_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.14.self_attn.k_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..78d2ae4e6d0799cd03231502221f0be1c85febe1
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.14.self_attn.k_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3681485d762f8f6b94931197a1121eba467ef718c7cdffe31d235c796ea3cde6
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.14.self_attn.k_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.14.self_attn.k_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.14.self_attn.k_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.14.self_attn.o_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.14.self_attn.o_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ff8c63c73e317c4e9202ac06692caf783e9b13d0
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.14.self_attn.o_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e2901520413d4cab7e9752f04a1c9b0007ce33ad64cf75197a1b8884671beb5
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.14.self_attn.o_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.14.self_attn.o_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..35f9597d2bfb373579f79e2ee75428be5e7a3179
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.14.self_attn.o_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eae98d0705c2591d97540b351136587fbe97c22a64a3f2ee361f02fd098f984c
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.14.self_attn.o_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.14.self_attn.o_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f336758638cbaba0d55e0e997280c5a9619d256b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.14.self_attn.o_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:06182893602d5fb96f61ae3efefe18fce9d7b0accff02f964dc516ac6565ab9a
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.14.self_attn.o_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.14.self_attn.o_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.14.self_attn.o_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.14.self_attn.o_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.14.self_attn.o_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c2b8a87892d4ef039baae7461bf6e7f382aa8a4d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.14.self_attn.o_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2746cc4525044333bb9b5105098c14927ede5860069c2d5f62122d5a3930f1c3
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.14.self_attn.o_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.14.self_attn.o_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ffccc98fe6a14dc4b5108b22789e927f4acd4b33
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.14.self_attn.o_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3c3054c48a3566c15e5611ec9ed10d1779e25c74959a8cf8f4074c248b3ce676
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.14.self_attn.o_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.14.self_attn.o_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..694e3a8c4bed2afac7b3679a650cf507d68dc50f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.14.self_attn.o_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ffc0d1aa62f9f352dc02f1ddb99d6fd134dbfa079a525727f18bf5d27a44f6c4
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.14.self_attn.o_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.14.self_attn.o_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.14.self_attn.o_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.14.self_attn.q_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.14.self_attn.q_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b9b6340079690e7b03dc356b397d4b3ea42d325b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.14.self_attn.q_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc804cd5e2142209d896a8fd5ec7c7a7dddc72ae70f93479005221a8981c1acf
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.14.self_attn.q_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.14.self_attn.q_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6c152427094dc2936d381032dbe3f1ccb7831acd
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.14.self_attn.q_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3df25c663ef603ee5a8880471f91a3e256f57b4fecdce6f225ca8feb1e7dee7e
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.14.self_attn.q_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.14.self_attn.q_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8f39d33ee4f67c6418df356b47a68c7e0fd8d897
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.14.self_attn.q_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a6907526e86203b3e552f08b312ce177bdb1fd3a59f92d29d9d5434c213687d5
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.14.self_attn.q_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.14.self_attn.q_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.14.self_attn.q_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.14.self_attn.q_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.14.self_attn.q_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3cda3d9761f03809ff9d4a2e4f678d67b65c9732
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.14.self_attn.q_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d46cec782baea8533fe16b0dcf3bc3aa6db40be291565440dce26559b7e1aaf4
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.14.self_attn.q_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.14.self_attn.q_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..58020dc0139793b9beabd617f581a00c0b9435c2
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.14.self_attn.q_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6ffec21b26adbe66e7f944c8e372813fd1b5efbcc2db16dcff6a8d7c997d3429
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.14.self_attn.q_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.14.self_attn.q_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0fc4c87a15a7588947ea30398ac7972b2baada77
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.14.self_attn.q_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d1d81a738b01577478ac7793aafb1d46acf56a723dd69a8d06564cc78f31d173
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.14.self_attn.q_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.14.self_attn.q_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.14.self_attn.q_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.14.self_attn.q_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.14.self_attn.q_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b322018eec97e839632aee3694841d0efd02e59c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.14.self_attn.q_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b5140c66a0cca6bbb667935a6c89521ddf1bd7d1e538473edf95ad5d146fdb4f
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.14.self_attn.q_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.14.self_attn.q_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3e959b88caafeee0305b897712ecb8822deb7ba3
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.14.self_attn.q_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3784a007462281354d916639696873201916db4323cba0e0c7298b026c0240ce
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.14.self_attn.q_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.14.self_attn.q_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..339f15e71dc2c629390c0f72218f95198c4036a5
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.14.self_attn.q_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:084a4d8c5806150eed011328a8c1fd979ec7a2bb41c9ceaf3412e54a45f010e6
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.14.self_attn.q_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.14.self_attn.q_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.14.self_attn.q_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.14.self_attn.v_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.14.self_attn.v_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f4447eff4caec8651b5f6ad16a52da7057bbde02
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.14.self_attn.v_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:39f59eb071e4cfd8cb99862eab3f097a5b306af4c2ad5914a252ed258cd5786a
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.14.self_attn.v_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.14.self_attn.v_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0d55cc034216766727ee61f3f22b2dc894282d01
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.14.self_attn.v_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:452c8e0857f922edfb14fc234220a7a630d8e2937d1d744883af71167afe2ad8
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.14.self_attn.v_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.14.self_attn.v_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..03f4d7846b86a7d26d8313c80ace32ae0dce1293
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.14.self_attn.v_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:504d90eb7ef5685d5ef14a23a67529a640e5c54a53081d68b98612f80ace48d7
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.14.self_attn.v_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.14.self_attn.v_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.14.self_attn.v_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.14.self_attn.v_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.14.self_attn.v_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..12a39544587b93caf08b14f08d8850171d7019ec
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.14.self_attn.v_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2709a994532d181ccd278537545fe5dfd712a79fc0442a71dd1476fd43d46451
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.14.self_attn.v_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.14.self_attn.v_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ef5b62335f870c8c895eed762aed173aeeb5b44f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.14.self_attn.v_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eee57924a1c116ef3760718aebf6559607d821abbe5ade6485c3641bdcbd77a8
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.14.self_attn.v_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.14.self_attn.v_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..37ae2337b96478bc74a805b2547d20f8c9d33010
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.14.self_attn.v_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:98c6da927d6687dccfd42d5db565e86247b5c027b440e5095be86d1973bb13a6
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.14.self_attn.v_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.14.self_attn.v_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.14.self_attn.v_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.14.self_attn.v_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.14.self_attn.v_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..50764722850220e5b88f80c185089560526d149b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.14.self_attn.v_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cb82649781a82e82b94ebc3881c512aaea2d727f5733a120d62067b4723ab800
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.14.self_attn.v_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.14.self_attn.v_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..66f07d21ebb44bf505d8bfa4a01333c2a20e3f73
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.14.self_attn.v_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7f414f134838b96dd8e4ae71b6321424d1bb92a3ec33bdf9fa2e9aba010afb57
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.14.self_attn.v_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.14.self_attn.v_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..259cd26af9df1137fef337c05018029f324b779a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.14.self_attn.v_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:81b47d4c164d66c13259e77243120b3c3301267f4085163f8406767a4ee89bcf
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.14.self_attn.v_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.14.self_attn.v_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.14.self_attn.v_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.15.down_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.15.down_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..91497b611c663f36fc7f9a549c1c83a482c9e2c3
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.15.down_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d9ccfe6d67f67c2e29d01e2c89025cfb755e96bee95a960e75cd3889602e1ac8
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.15.down_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.15.down_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e1138ad14694bd095055a29a5960b5a4e3695776
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.15.down_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e0f2412af6a3d1094654b14536eb388665c25c2860b67013344f497b840b85cb
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.15.down_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.15.down_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f1fdc801601140e240238c258d680d2ff72306c7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.15.down_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ed56823a20eb48ad6aa9591701a69ef7079e241b8abe55c8114933ce2a5bbc37
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.15.down_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.15.down_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.15.down_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.15.gate_up_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.15.gate_up_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7f7b8777490c8a2073541a4d5cbf49e22cce06f1
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.15.gate_up_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:96008ca6fdf41c41ad58a3f6f18655740ffb535d7f188019a32f922b4b647bfb
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.15.gate_up_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.15.gate_up_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c82a2133e3f02e2b32c8755b6c355303bf873bac
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.15.gate_up_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6c10f68fba50f9bfa91c7f46a581b9ff4088585b9076958a6b5b0f0a6c40c794
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.15.gate_up_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.15.gate_up_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..14795d279d654aa6d3c8e1aada31cb3a7908f32f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.15.gate_up_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:47ab770735fadf43dd5ac2c3e62daa70a16841a95ef4e5370c52c4aa56110d1d
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.15.gate_up_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.15.gate_up_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.15.gate_up_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.15.input_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.15.input_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5a3093bcbe8fd3d5e2c96383cb8e3b0b4d96e5e0
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.15.input_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:39e9482c0e454b43a04509365e030c400671303eaa90a0adfd254026b08630f3
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.15.input_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.15.input_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..777c59e19661ff5d76e2a59949f9c9aaa06a562c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.15.input_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a1bb39eb66152a488f6e630ec113f43352649378ad936a802e88fdde6789e982
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.15.input_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.15.input_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..82606bcff2199069a613397c877768f1d9114d75
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.15.input_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f28f72f07008a88a324838174a5e35371ae31b9f938a53ff70cda9be6cd72627
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.15.input_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.15.input_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.15.input_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.15.input_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.15.input_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..54482824edcce74f452ac8febfa19f9b00712342
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.15.input_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3e9b22d68ca333b9a5f18874db8ca640a1daa4de2d7cb6db043e1607259c0f92
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.15.input_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.15.input_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e708d0685cf71b6e2d51e1b56aceb2ce1c4c2b26
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.15.input_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a1dd397af3007d4ab377b1ac1ea6253633ee8af7a958837f63e808ada16ec619
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.15.input_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.15.input_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b62d4be8e54cce589e8a9cb4a348cd309e69e068
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.15.input_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:75ebe75c0844bebd5a6a3810617941af18dae047f8a5fc862c8d8347a386918a
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.15.input_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.15.input_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.15.input_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.15.mlp.down_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.15.mlp.down_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ea7a485e9cbde966fe8ede9e44a10ef4ab979222
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.15.mlp.down_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc5fa835f5516178a0579b8ea6bf7cbb72b44b4a31ca35b683d7c8177eceb31d
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.15.mlp.down_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.15.mlp.down_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..bb1757569f0cb54f41b5e8fe3615b3ff9035996e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.15.mlp.down_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ccc1fdd2350d904a0d15ca53c6b897810f96557b28b74ba26e984d970dbb998a
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.15.mlp.down_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.15.mlp.down_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..92024c426ab20c2ff9bc0cf3e22333205cacbd68
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.15.mlp.down_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea6e08f9aa43d1c8cbee0a230576d6e6f06d4af6b9a9261f0c6896226a59eb36
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.15.mlp.down_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.15.mlp.down_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.15.mlp.down_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.15.mlp.gate_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.15.mlp.gate_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f00d2f627f1932e8d61b899a0de9689eb403492f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.15.mlp.gate_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:107fb596c561964bd85e4ee36a31ffc704c788e733339dc32f4124c7b3399800
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.15.mlp.gate_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.15.mlp.gate_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7846e0e196b1ee5636033c82cf4a7a00007d0399
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.15.mlp.gate_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:92cbf39cd02e555019b22c2bc9754a98119d222938c7a873d858f2b7fb9ceee5
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.15.mlp.gate_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.15.mlp.gate_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b0298cea0e583d05a7267419f01305a98e9f4f42
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.15.mlp.gate_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dc0c9ffb505bf49387303daadae5c96596e379efe33818bd3387e01da93b5b4e
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.15.mlp.gate_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.15.mlp.gate_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.15.mlp.gate_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.15.mlp.up_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.15.mlp.up_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3fff0432f19d0153f9b8f0ca485d538a4e306bff
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.15.mlp.up_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8d05ba848328beff4fd41f4548ac1d9955f48e4a23bd7cfc8d2b433069bf8819
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.15.mlp.up_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.15.mlp.up_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c74722b5593f74b13f1295d634e650513b8961b6
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.15.mlp.up_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2066c28b59537347b33a7e4b1ccce1af0b8358b2f9f298c05bfa7ccaef28ae0d
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.15.mlp.up_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.15.mlp.up_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cbb0f2b1de5e13a62a4902bb111faac58b4abeb1
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.15.mlp.up_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:45722a76594b25700349859c88d444e15e8ef0d04ba5dfdf8b65bcdd64a2eab5
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.15.mlp.up_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.15.mlp.up_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.15.mlp.up_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.15.post_attention_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.15.post_attention_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..30b0b3c18b5ba261ca91e8604dbb591fe54b75d5
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.15.post_attention_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2208a647fc6944c4aad57d775eafea6f60e5cac8d1f35955df7a045f0fe2c0ec
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.15.post_attention_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.15.post_attention_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..16fe3f7657b4b715cd19dc09573d9353e90cc880
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.15.post_attention_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9789722e5b86413d7038d9db776b1d4739fc8f4f22c0a4a0c9ef24913f58158d
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.15.post_attention_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.15.post_attention_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..52054c8e48af086acf04650d7f768015b36f190b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.15.post_attention_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1bfb019d8d116e259d49c734961ab4db551bf7bc2792e5d5c2ebb91ca457c3b9
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.15.post_attention_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.15.post_attention_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.15.post_attention_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.15.post_attention_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.15.post_attention_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8249354869798f5ee5a3759d3ad574f020d514e9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.15.post_attention_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2f4da0dc8ef0fd7ac9bd9e59b830a959cbf65608074eb1b65e9d5c0ec081d5c8
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.15.post_attention_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.15.post_attention_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5d121a301c6c538fc11fb895700973d110f7c183
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.15.post_attention_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:754c004f098e125f5c43d3344fabe13ef789b4050964e33dd5db071218b06831
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.15.post_attention_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.15.post_attention_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cdcfb9ca1115af7164fc91b035c98ec7f6efd0df
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.15.post_attention_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5f66b32e6e0ed39ad5b25bfd12b0061e4de8613f022064127b06196710091b2d
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.15.post_attention_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.15.post_attention_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.15.post_attention_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.15.self_attn.k_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.15.self_attn.k_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0ffe6d67bc80505ca24e678f47be48bee4f681eb
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.15.self_attn.k_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6a54cedf0186edecd22ba3c05830c7151eb436c8397293b63705f50a2e9ec5dd
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.15.self_attn.k_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.15.self_attn.k_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4926835d4de98fa8e71e3f913834133f9fab6124
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.15.self_attn.k_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ce37e86e7d96dd39ff1cf773c678c0a1c1167ee03d526b0e2e479447ab9e2184
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.15.self_attn.k_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.15.self_attn.k_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6d6a1705de27666185a68904b768bf370a3cf6df
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.15.self_attn.k_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:48b28dd2c15b216efd2ccc85eece4539378046184fbe2ab575b301e5a52bb2f6
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.15.self_attn.k_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.15.self_attn.k_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.15.self_attn.k_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.15.self_attn.k_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.15.self_attn.k_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..027939f2b362e5ecc3527282fa88d698e461cb5e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.15.self_attn.k_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0b2085c21c6b3602a92b03846f9ee3ef2a40234563832fc9d4c952d2dab9967b
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.15.self_attn.k_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.15.self_attn.k_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d298ebac18319b40b2408130acabc7ed4c59dbb6
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.15.self_attn.k_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:92a2e237735b73a7ffbb9ceda80975c30c1bfabf0617b103e4a9ade1928b3642
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.15.self_attn.k_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.15.self_attn.k_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..df8f2ff8844d925d58a36e974ceff27ab04a7273
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.15.self_attn.k_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a05c803b0b5764b5020fbe12f17dd3b1f3a60799e5d6bfa3b78a34b77c8bc295
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.15.self_attn.k_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.15.self_attn.k_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.15.self_attn.k_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.15.self_attn.k_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.15.self_attn.k_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2ce20007e848b32af227d55da1e5aa18f95ca970
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.15.self_attn.k_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c018b71c0bd4b5b18f9870b3b264f47e751f08918ae61cf8e820f60833c84d66
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.15.self_attn.k_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.15.self_attn.k_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b67123ecd1d0d781baff908419c95e21a61d542a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.15.self_attn.k_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e4d8aa44be2b3176e7fe1c396c2397f66acfdb3da986b829ba71654277771b62
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.15.self_attn.k_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.15.self_attn.k_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3cc824b674818e3c2dba14d7db52cc488668c891
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.15.self_attn.k_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fea3585d86eaf454892099db8d354aad3e99d13a2781fb00a95728de98749fd7
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.15.self_attn.k_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.15.self_attn.k_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.15.self_attn.k_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.15.self_attn.o_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.15.self_attn.o_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9ed8c35a65c68fdfc9ca742a03fc0df114775925
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.15.self_attn.o_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f8c93b40d1f126a36fb36381e7622ded979a6e0aefc039582dfb0ac32edb1424
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.15.self_attn.o_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.15.self_attn.o_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3a58289a7db4e59d0fd223415eef62619b49e5fe
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.15.self_attn.o_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cdf5f7ddaba456452e8f87165fdbc4aa45ebe03c6e04dbe4f31c3099de3a53bd
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.15.self_attn.o_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.15.self_attn.o_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f329882973ddfc94480f620d80a084a72e393e79
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.15.self_attn.o_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:05488e016c75c73d504ab87a1b6a641e9bdafe2f03a500abe765eba985c22b4c
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.15.self_attn.o_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.15.self_attn.o_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.15.self_attn.o_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.15.self_attn.o_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.15.self_attn.o_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..50d633207d821f5233775af1556d00a2df6f6474
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.15.self_attn.o_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:14d8c768fde8f0ad7d16b33d14aa44ee96d67a76b4cd4a40eb798e9804325dc1
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.15.self_attn.o_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.15.self_attn.o_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b5e663ef3ec39dd14b6256bc3683eed9764d7a45
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.15.self_attn.o_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:96f4a85e025112a18ce2db7c5c942b55a29d134a82c5510c9e2cb9e93a85aa53
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.15.self_attn.o_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.15.self_attn.o_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1acf039e6f06eced4baa93db34535998ec5f4efd
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.15.self_attn.o_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0e1fd5a3c22004e69ee49169dd3f2e659279f1d1552c808a160ecd8b61401e87
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.15.self_attn.o_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.15.self_attn.o_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.15.self_attn.o_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.15.self_attn.q_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.15.self_attn.q_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..19c22c3d74677af936f410948e22f9f015192995
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.15.self_attn.q_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:25837a80b507c5277a41bd952727ac3eeb17a6970d741918273b3d41216be4ae
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.15.self_attn.q_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.15.self_attn.q_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1afd180031756984f5cdfda34d306195f681d533
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.15.self_attn.q_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4239355fd74d0498dda1d9d12422c6c592d0ce41576870ce04471fa0bbfd5ffa
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.15.self_attn.q_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.15.self_attn.q_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4c681541d5c5e9302f6cbe288b33f9f19c9d3431
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.15.self_attn.q_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4f5028b1899875d02c04754b7fde34356c4b73cf3ee7e3c48a9399ab5da74d94
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.15.self_attn.q_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.15.self_attn.q_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.15.self_attn.q_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.15.self_attn.q_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.15.self_attn.q_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ed9ccdf35b6c4fd2a526f34c68aee2a1a1ad7c25
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.15.self_attn.q_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bfe98e15973cc3922a4d53c1ed37dde1cbf19ee7b2518aad51dcf9a70fe06796
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.15.self_attn.q_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.15.self_attn.q_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3eeb97625e15a01ae5f249ed5f3e321f8034cb34
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.15.self_attn.q_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:818884d28aace73e43810712a99611e28766aea92e923653afd426a12b4c06dc
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.15.self_attn.q_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.15.self_attn.q_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..84e416bc1d5fb700d362684a9465e4065bbc3e75
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.15.self_attn.q_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:16b9b0319b5bad55a29df123a87a69fbc2837bd1c167434f0c1f8febeaec7dad
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.15.self_attn.q_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.15.self_attn.q_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.15.self_attn.q_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.15.self_attn.q_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.15.self_attn.q_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3057c41c7711eab6081834a19e5c43804d237699
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.15.self_attn.q_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8e2df4f01141833239dd8790eef3c4fa3d006ec4ea41f4269d9e456e78bfe5e1
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.15.self_attn.q_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.15.self_attn.q_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..40bc5cfb8ce0491f3b200616d89f82bcf3b76b0b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.15.self_attn.q_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e9b80fbd73e12de2ef44c508ef54862c71a3078da0e3c5eeff288d23fd8b850a
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.15.self_attn.q_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.15.self_attn.q_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1e6c9c02a7118f1b593801c6cf805da28c60479e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.15.self_attn.q_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d0c4dc9d4f4cc3f48b0806aa9a6b6daf0298716ab8dfddf596a57078d81e04de
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.15.self_attn.q_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.15.self_attn.q_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.15.self_attn.q_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.15.self_attn.v_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.15.self_attn.v_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0aa4bf4f5ea918413277e4eb4253335bfd7e5d16
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.15.self_attn.v_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3ac066bb4b597e47147cade49663e9e98f68500cba6aaff87795932b36304b88
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.15.self_attn.v_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.15.self_attn.v_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9a6c4d4e2aaa49a6f940dd56c9ba6db6b2715155
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.15.self_attn.v_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c9a6725568a6e60b124a2be81e944bafe4d26eea2fe4fe4f0413124780508d21
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.15.self_attn.v_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.15.self_attn.v_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..997357359d365da040912c3e40c111977635e48c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.15.self_attn.v_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:186281959d6ad5dd1364ff052f1b6e3d4f8b5a7aa84666a1dcb84ddb0fc5ce82
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.15.self_attn.v_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.15.self_attn.v_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.15.self_attn.v_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.15.self_attn.v_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.15.self_attn.v_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5934d39dbae2965eefc7ba70d42a09cc9e3db43e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.15.self_attn.v_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:35e9456f27820bec44dc78b7da6c027d1f920cc6e5e3d37d53abca2ef188ac99
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.15.self_attn.v_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.15.self_attn.v_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6af131952389b93182b855d5245ad028071a6a3a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.15.self_attn.v_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8834e0ea0648e2b7ff8e529340db4d9dc3451fd7b013fdc1fbb7fdba836be128
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.15.self_attn.v_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.15.self_attn.v_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..873fc217f601f4718572499e1bf24bde71ecf7e1
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.15.self_attn.v_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7390df312c306a4df36e25e0f4fae95ab5583f169809fb5b94e912cd3384202d
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.15.self_attn.v_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.15.self_attn.v_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.15.self_attn.v_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.15.self_attn.v_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.15.self_attn.v_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..301e12683de984e329a2ae32233d1bf569917784
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.15.self_attn.v_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4f50fd3071187ae74678d4f49f1838ae48e182f31439dc99e63a57141488d9e4
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.15.self_attn.v_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.15.self_attn.v_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3bea8edd821a9879fcc14cb4c608e1ede29cac72
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.15.self_attn.v_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:51f0525d6bbfedf505f49d56c28e87f5c4f7e886f10decb29494c08c74c124be
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.15.self_attn.v_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.15.self_attn.v_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..56a8a93f33cdd7ac078b25296de224647dea271a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.15.self_attn.v_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:372846f26ead39986c11fdd2043ee08c9a2e820d0f66d186c6d85e1155be8c23
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.15.self_attn.v_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.15.self_attn.v_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.15.self_attn.v_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.16.down_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.16.down_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6d2050a0ba574f5250a4ae9193d599387c26e00f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.16.down_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe1ad8b6364badbb514c2059a8537c596f1b77eeba6663867022dac3c9879bcc
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.16.down_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.16.down_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..30ae01ca4060884ff4a045a2092931cd1ad6462f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.16.down_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c2856f19915538c21cd32cdda1f7d7e53c231f8190a1c363f3880ebdf0829ced
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.16.down_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.16.down_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d499dab4b652957eeeb9c2f076a54e207e7656af
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.16.down_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ee09a6590bd43b88f6b35400001aaa17b8f258820c2881f36863ddd16a6798e1
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.16.down_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.16.down_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.16.down_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.16.gate_up_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.16.gate_up_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a21f3274acf7d56d2821b74b28a2425677e9a6ae
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.16.gate_up_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ab871abf66c1b476891a007aa9fb004b87fcab4fa2066b6219906c01bb6e076
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.16.gate_up_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.16.gate_up_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..002d8be9a2fd189586397b28ae61fa7630963a20
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.16.gate_up_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7d46c930528521abaaeaa3edc3dac7f920c85298fd6892be06bb23046155b93d
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.16.gate_up_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.16.gate_up_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f61a243409c3a66b79d417cb9501ff7b0b4d2c35
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.16.gate_up_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4d9cc335d1454edb93a0a654db9488cfc1e4e0bdaf805d43b0a428097356acc2
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.16.gate_up_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.16.gate_up_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.16.gate_up_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.16.input_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.16.input_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..64029aad1fa4db65538fb87268d7eac2446dc1ac
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.16.input_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d153ef1501b139a1f8e4815c5a4e0fb838b15792b83c2fcd1051c99982c14c90
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.16.input_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.16.input_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..56bd084f9792e91d4a0ac5fd37a6fc11bc503ce2
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.16.input_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2ce9dd23e6ba050a8b4d8ce7c54c443225c21a3357713e299ffb735da84e0a40
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.16.input_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.16.input_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8fec48ea3b90ad5683b91ce868f487b56f12e810
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.16.input_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eafe4e73bddbe516c834c73cd8304f4aebd951547acf8988aabe8f19b12ac813
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.16.input_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.16.input_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.16.input_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.16.input_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.16.input_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d87073283dfa028e155a8c0ac9e0195bf082300c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.16.input_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c5951a99862d6e39c43fd1c0209635cfd9725c40f2a28719aff5280d15aa8945
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.16.input_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.16.input_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cce81213c937aa69f38be3102481757842224f45
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.16.input_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:559659a6721064e4f4019884b7ebcb63ebbfb278f59669dd0f968315be12995a
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.16.input_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.16.input_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a0e0bd65352ae4dd81f9899b3b03ae0233dbb1a2
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.16.input_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:471b37660d640a68366888f1f5fd4e826d6c0290aee394e9a4c834bf17713c98
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.16.input_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.16.input_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.16.input_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.16.mlp.down_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.16.mlp.down_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..80055173cf786755ef0eb088be587eb9e32afb0a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.16.mlp.down_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a002e7c0036a4a644c8d70c9d19c7a98c8fcb5daeccbeb382d844cef9d664c9
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.16.mlp.down_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.16.mlp.down_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fd6cde04340720071bec5fa5b5e584df5b055ffc
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.16.mlp.down_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ec8e3accf4c56528528400adb4198ea274fce491db373cd1335faa97f1183e81
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.16.mlp.down_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.16.mlp.down_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d569bd76ea2f927964d9aa150ad2fef02c0e40f9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.16.mlp.down_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6fcfab250f252978aabc9fd638dff36827e203be5343eb095cd0179662922e61
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.16.mlp.down_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.16.mlp.down_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.16.mlp.down_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.16.mlp.gate_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.16.mlp.gate_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9528d16c19f5e0e48a5a0cae3b0b81d675cedbd4
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.16.mlp.gate_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:28a95551808506d623e0162e8eda6103bb77ae88711681e1a58e06cf93da0dd1
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.16.mlp.gate_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.16.mlp.gate_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7fad07562a433cbfcd2bdedd3c355bdef7500d55
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.16.mlp.gate_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6768874002e3137e8ff29b68e48071379dc5561ea8c3a8dbe9a97c0ccb817d2b
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.16.mlp.gate_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.16.mlp.gate_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e8466be78ceefbb4e6cd12d73c20a0381c52b6ba
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.16.mlp.gate_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fc9df5be5cab1ca7406b4d6f19fdc9bb6930876004f92afed8d9011910d943cb
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.16.mlp.gate_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.16.mlp.gate_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.16.mlp.gate_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.16.mlp.up_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.16.mlp.up_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..078a9e2941bd8d4bbe380f6d871eae3e9959669d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.16.mlp.up_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d115a63dd89e3815280303250d41a1e1b4c88c7a61759aa4bb71af3bef6536de
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.16.mlp.up_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.16.mlp.up_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..408174a7681c4b66c2c880e222c0f683bfa8cb02
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.16.mlp.up_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e1ea70f1e661924ccfc1c052ee39a271d5e2b68a4f85509e19c90c2fc3521127
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.16.mlp.up_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.16.mlp.up_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e1817855c091d17bc97be42f02c014096f68c4c8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.16.mlp.up_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c8d9d19a9b5e456d5ed0cdfc35b0cef2f841209640c6bfd88ff54cc9ebfbbb93
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.16.mlp.up_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.16.mlp.up_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.16.mlp.up_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.16.post_attention_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.16.post_attention_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..132914bdda22aaf1741bccfb84d40945809a8ac8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.16.post_attention_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:50b64ec6b96c8b9d8c0842815436d274897d1ed936fb4d6e87416547277f4c87
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.16.post_attention_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.16.post_attention_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d4319dca9b799bb09a61192139e5614dec3fa37e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.16.post_attention_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c43c010536963424622e79464c7da4a5fc5e3f47630f20af8aada7999b8b2757
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.16.post_attention_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.16.post_attention_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6f11bb953a09137029518e17447603145922b144
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.16.post_attention_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5c8a35c18087c29238cd1fb7ab6f6c682c2fb97387c0a8dbe0fd59a0f5fd22bc
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.16.post_attention_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.16.post_attention_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.16.post_attention_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.16.post_attention_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.16.post_attention_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b9ec14e1b6baf2d133a93c65da5e5ca3a3d556ca
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.16.post_attention_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1542a5e2f144451319658cdf28a65b6118688ee4f0cf3a7fbe2965c78578bf09
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.16.post_attention_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.16.post_attention_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a733aca16495aa6b6eaba14afe2eb24e7ed5113e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.16.post_attention_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b05bd401db6599730c850c56cc0985d8021ed7671d1c417988b50838d1dbe2a3
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.16.post_attention_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.16.post_attention_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..40a96157fc4de67ec36dc19a4c26ab1d0afbfc75
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.16.post_attention_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0ac1500b06af976d2d6f8ea65ce8b8677539acb91f3d979708fe67e2d58ef564
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.16.post_attention_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.16.post_attention_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.16.post_attention_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.16.self_attn.k_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.16.self_attn.k_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..86e216bb6bf372abfe38b557d5c7936bc93c2a44
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.16.self_attn.k_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5ff8468d8537bcb460be73d445026d74b41d6380a0b28f041bc43742081274ca
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.16.self_attn.k_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.16.self_attn.k_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..28565a4824337fe35abe0c8ddc6db6ef6b32af56
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.16.self_attn.k_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5748cff65e31c7e8757a2225997bc95c9136c3c8c87cbdccd704cc36d9f8e420
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.16.self_attn.k_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.16.self_attn.k_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ee59092c205d9b8da101fef2ea1264c1d93b0fa2
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.16.self_attn.k_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:04927ebfe95e4cd1c1c66dff1e21052f57aefbfdf6d374cae9b515510d88c984
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.16.self_attn.k_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.16.self_attn.k_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.16.self_attn.k_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.16.self_attn.k_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.16.self_attn.k_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..befef55c8fdbc879b45c61c6c679bedbb975bca8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.16.self_attn.k_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2003627183cc69ae247a798b32280d244cdbfdaa386090cbd1f78f7553bc35f5
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.16.self_attn.k_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.16.self_attn.k_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..78cea1483cd5576bfc4d61518a23959b420135ef
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.16.self_attn.k_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f31907e7da339825a1ad751b517f2423d3b60966ad7fdb374acbeca0b067bb38
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.16.self_attn.k_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.16.self_attn.k_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ae9e810ee493e4e48bca9ec96f4a7f0222666e3e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.16.self_attn.k_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:57e08593da21b2e8ea3fbf7c29df48486064ebca1450347b1b55907058e30e83
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.16.self_attn.k_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.16.self_attn.k_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.16.self_attn.k_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.16.self_attn.k_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.16.self_attn.k_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..666fafa2cffcfea2ecaa34e360ee7407797daec2
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.16.self_attn.k_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cd1c8fa4142f08458f59ce7b00b753315677c99be10cace7c87ee16c20fa91e9
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.16.self_attn.k_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.16.self_attn.k_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c73b5f375f72775bc372b70ebe88373dc0445d99
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.16.self_attn.k_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:af266d8264abe77f17f39bcad49609a48a0b07a37ce2baf53ab53fbcf3fd0d78
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.16.self_attn.k_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.16.self_attn.k_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..24f85bbd8d6a2c7de7a4f9e589627355c92e1fba
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.16.self_attn.k_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:646cca7a55e373ca1a472b507f2b0ed9e3df53d77d76a872b5920318d6d0cf33
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.16.self_attn.k_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.16.self_attn.k_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.16.self_attn.k_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.16.self_attn.o_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.16.self_attn.o_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d287102c88555ec8c0dd739444f00a871d35c504
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.16.self_attn.o_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ecc86a274b7499ce602c1548619aca8643494de8a86ac2d41d1beaef9ff95bc7
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.16.self_attn.o_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.16.self_attn.o_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0d6b384e6f0eab6ee78aa744a2de015cb85d52b1
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.16.self_attn.o_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:af068e801275808aed1a9f763791d8895a51e0122ef6287a42db5ebcd23671b4
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.16.self_attn.o_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.16.self_attn.o_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..39eb54548f8eb60ebd08249827c7f89c1df2c565
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.16.self_attn.o_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a5dbf9fcf639805e15fb2f74c12fe1870895a1548b63b49c9d525e44439a2e9a
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.16.self_attn.o_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.16.self_attn.o_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.16.self_attn.o_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.16.self_attn.o_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.16.self_attn.o_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..43ef4fced0bad9a181e963ade636144e7bc50a7d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.16.self_attn.o_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f0780cdc361aa493c9904a37fe6916d50ad597597e3be6cf2edfb3529f10e8c8
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.16.self_attn.o_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.16.self_attn.o_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4ab662bf1d668149701a2f5cec45ea7fb26de856
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.16.self_attn.o_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9feb6b85cbae8d5fbffdbbfbdb5f098c43af572091775a19abd197a7b2707a2f
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.16.self_attn.o_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.16.self_attn.o_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fea80781fbf4d240a25b06f94a1e30df0cf12471
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.16.self_attn.o_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0562628090ec7822d0b40d63f05ec85eaad4da36a6a75ab35c51ee7f1b1b127c
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.16.self_attn.o_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.16.self_attn.o_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.16.self_attn.o_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.16.self_attn.q_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.16.self_attn.q_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f849bac7ff135ddf5d7c3da2a95ee29bbf8e6a8b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.16.self_attn.q_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:98af980c0d9df113c39b391d306757b9620dcd8a8bbe9a2cce07a168c82f9ab3
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.16.self_attn.q_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.16.self_attn.q_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c6c4fbb2aa17b7037a040e6075fb32a72759317d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.16.self_attn.q_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a16f245b486f26c2c280153d253c79a44d948c1d841e71aeabe493538ff7e72a
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.16.self_attn.q_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.16.self_attn.q_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1419e9cfa0e45281fd837cec6b2eb1ac665cd1cc
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.16.self_attn.q_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef6f49327078bf659e9a4ae9322402f60526d9097042eab71ddbb42f30c412af
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.16.self_attn.q_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.16.self_attn.q_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.16.self_attn.q_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.16.self_attn.q_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.16.self_attn.q_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..746a4cbdcf71b4ef6634961abf0279f83698ccba
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.16.self_attn.q_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef3bd07815328423dd7a65584767e0b36f70a9d891a8e5d03d0df49e2c395bd1
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.16.self_attn.q_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.16.self_attn.q_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e0ac5ba8c671e00d68b3555f1ead33013c0342eb
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.16.self_attn.q_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9247912e9590520a88c4dc697fb3460ac7ce19439797d1c2fdd8fd3eba642722
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.16.self_attn.q_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.16.self_attn.q_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..af4d0e990df4a5fb932d881c35fec6f8a2abc306
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.16.self_attn.q_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fcb6dcc555d16dc689eb632cc2df9636ec524610871b5e283efc093fe21a70ab
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.16.self_attn.q_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.16.self_attn.q_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.16.self_attn.q_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.16.self_attn.q_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.16.self_attn.q_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c0f2802cde670d6f246bf32bbec58cf59802c791
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.16.self_attn.q_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:23aaf20ff84360652eddb9a53dd34974495234f363e9f749562d70b97340a002
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.16.self_attn.q_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.16.self_attn.q_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..473d191f37856852e7a844eba3d0cdbafd7c4d21
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.16.self_attn.q_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f6d5e07b4c9cc825e4e3a44ea90d325c7733e37889198a90e3ac5bebd16c63c1
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.16.self_attn.q_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.16.self_attn.q_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e0f48d8a404fa4631774677ad1b57cee2f8d1a3b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.16.self_attn.q_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:50946761fcc465b463f6495cc95ddc9f09c8740f381039a141c6cb06cc9696a1
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.16.self_attn.q_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.16.self_attn.q_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.16.self_attn.q_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.16.self_attn.v_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.16.self_attn.v_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e6f4011c9747c1040ea671394c0e5f1da4d28fb4
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.16.self_attn.v_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:16e189746d3f7f75c0a075780995a18bc8fa5eb35cd5546671c519900a031ef7
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.16.self_attn.v_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.16.self_attn.v_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5b097e4d13ae3866ce8f4cbe319b217c903e24c4
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.16.self_attn.v_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4687c3ada5c1ca6eb13c9f0af9ba1f88d2092ed6dc7a5aa4dc861fa73b257b31
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.16.self_attn.v_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.16.self_attn.v_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3c18a5ea4e0cb6caee5bf74e9df539bac0b8ef30
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.16.self_attn.v_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:18abafe63104b6f4be07571c1578622ecc78e3019e51c74fe99c36579a0957c3
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.16.self_attn.v_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.16.self_attn.v_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.16.self_attn.v_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.16.self_attn.v_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.16.self_attn.v_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d707ec6a06990f50df95166d850d0c5f2141ea43
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.16.self_attn.v_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:930fc628b644f4a93a2f9049246b336c02f5f2757f66d43daa7a9ce052aef557
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.16.self_attn.v_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.16.self_attn.v_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..88ba0254f64454347be2e65241e4816c63517e8c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.16.self_attn.v_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d1dab480abdfcdb2510a025986894203a7df54abaf308a3438430f6e1139c93d
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.16.self_attn.v_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.16.self_attn.v_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8afff6b4cd0d0a77f4bccb6db2b5b5864f7bc58f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.16.self_attn.v_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d721136d49ed6e1630577885825a2f8363da30c6286dc2f1948e9917b614eb6f
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.16.self_attn.v_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.16.self_attn.v_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.16.self_attn.v_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.16.self_attn.v_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.16.self_attn.v_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c38031d1ece5580953f1331cc70beb2e306ca6e6
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.16.self_attn.v_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:012316b024fa0681f3a5781b36d764e4bed1213bba34c695993d29a9493bc8cc
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.16.self_attn.v_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.16.self_attn.v_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fe44f289ecc1ba0cb32e5c776a6ef8d44172f502
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.16.self_attn.v_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e541db6f5f09a6185bb21423843110aa4cc64d19bbe09c09d9199f6d71188283
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.16.self_attn.v_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.16.self_attn.v_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f595d99a626002dc9b364ec4c7ed918bf2492435
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.16.self_attn.v_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5aa433ee8f8fa65eeb31645eca7809ee1336082538aafd54e0aa2ce779fbbbeb
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.16.self_attn.v_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.16.self_attn.v_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.16.self_attn.v_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.17.down_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.17.down_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f6e90a00087043ed7dcc82bbeab67e5b6f440546
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.17.down_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5332f76f3d0020fa3df11843f57530ff6c30a022f70aa2919179e3d6767b1b04
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.17.down_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.17.down_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0f01906f8e429b72699551ee5773332551d2dde6
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.17.down_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cb3c78efd8674b6872de667e9b132e85c4193ee9f0a90abafbb4144d638b2efc
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.17.down_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.17.down_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c079c8f54619493846db44b1693a65d352ed657f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.17.down_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f1f18fac48c03f20825bec3b06377397d5a447e4440ca24a891184965685b0c0
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.17.down_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.17.down_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.17.down_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.17.gate_up_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.17.gate_up_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..49b1cdad5fa9c45c78a3a19d4e7634032d99c9ec
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.17.gate_up_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5ba83567ac5b9f818ce75e45427255f0f61612789631e67f6d776c1d0148a17f
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.17.gate_up_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.17.gate_up_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..33f17626267697816b04d3005f47ed8e3539de14
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.17.gate_up_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6dd4fbccd1827b13bfbcf0b22151befb19d6e64c22a46565917605472fc987d1
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.17.gate_up_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.17.gate_up_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ccc563276bb7f53f7d76cc0f5e090175da66c5db
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.17.gate_up_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9b083d90363e803a42983014042dd7d4526967cc5bdc49a7fc8beb1366dde380
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.17.gate_up_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.17.gate_up_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.17.gate_up_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.17.input_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.17.input_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f79da24f0f6f7b6a2ad60a8c17d436095db60f63
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.17.input_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0c88a0a185e8c17df5c627ebe0530e7baaac3ba8679e416ff2a3da3da0f51d20
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.17.input_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.17.input_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4ff76b8fb94eef779d764241a79fc0f5ef408b7c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.17.input_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:86aef31f6052ebd6fd81161d84bcc8ddfd54375bcbc9c85fe6ada4ab077e446b
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.17.input_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.17.input_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4d5da5e70aff6696e06cb1c349a9cc3a7746ba5d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.17.input_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1e436325b86a704c283c82c4c3d739a531938a600eebf15c446f7a4cac2c53f9
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.17.input_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.17.input_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.17.input_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.17.input_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.17.input_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b4510d418c7343a89500d566d54f15e7af819df8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.17.input_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d5a8f0c99c2764a052e1ab10409bb00312f6161af14e157808bc3b7bb0fcdf7b
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.17.input_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.17.input_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5c1d99abd8d69313b22d7640284508ed62fe0998
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.17.input_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bbbcfa69f72ed0504a9ddf3fef3bcf12a068c58aca88b5bd7439428366c7bf98
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.17.input_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.17.input_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c6873a3da7a111af7d38e00e9ed0302bd50959d0
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.17.input_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a3974e6ef0bf0bbac21614920e1d9b4a078de49815ded5e884bddc689dd934a5
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.17.input_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.17.input_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.17.input_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.17.mlp.down_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.17.mlp.down_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9544a5bb043e979b9f253f02d9fb36c27ead77c4
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.17.mlp.down_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:31e960d8747a77b5e35d4631179cdb34976949f80df9b6db032f72c5fd864701
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.17.mlp.down_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.17.mlp.down_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..132b1a8bafbe7d9e549bc4ec922b646923adfc7d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.17.mlp.down_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aaab57d85a34c30a4049492c8b62a10cf231fe6f606d9583831524256fc04c2e
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.17.mlp.down_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.17.mlp.down_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6cd3745cbb98bbdd5d757591b3964bc7d6e7e378
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.17.mlp.down_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:589499c199cacead67c9c487f53960824f892a20e624caae0d4b2ecf9fb08a52
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.17.mlp.down_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.17.mlp.down_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.17.mlp.down_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.17.mlp.gate_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.17.mlp.gate_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..efadc3804a4c038ba67c6d7a890ce3125adc230b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.17.mlp.gate_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6cca2261be44f4c05d30d18faebd66f364f591f211f5738c010c81ab417c7f01
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.17.mlp.gate_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.17.mlp.gate_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..11be91dc3c8871f1b4d1b1c8dc29732e41e8c52e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.17.mlp.gate_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d7d7b3e36616b087906f24a60731da7e37b0b8e55d926b220c141c7457b69189
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.17.mlp.gate_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.17.mlp.gate_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4d50574c13cfbbdcaff6ebbad316fa467e71370e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.17.mlp.gate_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:025c5ba18b0fc649ecb69bef22461df5a86810177d58419897d6457090c2811b
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.17.mlp.gate_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.17.mlp.gate_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.17.mlp.gate_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.17.mlp.up_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.17.mlp.up_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0c1fc6491f191dc11a23925d5006f36ea8e0b0fd
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.17.mlp.up_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d78a3b398a3d2ee652cb475420d3448b8ba7553fa07d9bbd957b42e9104213dd
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.17.mlp.up_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.17.mlp.up_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1e8cbdf8dc718b235992b7a850e7b9a63ead1033
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.17.mlp.up_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:83ae27307f3a215d12e23d841f8a5e450544c92a03e0bcf685d952ecfddb25e3
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.17.mlp.up_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.17.mlp.up_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..49ddb94a4374cb6740c7cb530f5faeff781cef9d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.17.mlp.up_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef98dc4bbe74e592da086ec3a7b6360dcf35b4aa659f88fc9855b0bb882bf855
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.17.mlp.up_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.17.mlp.up_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.17.mlp.up_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.17.post_attention_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.17.post_attention_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7e187d5fb768f710af119df5ccb834017d840e86
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.17.post_attention_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:995ce07ee5e87aa9d41f801e7c8c7747686e957196704393202fb87b0bc9d818
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.17.post_attention_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.17.post_attention_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..00744b4f8255915bc40d94969c4b3f08af6feefb
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.17.post_attention_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:03a4a65ae147bf03f714f1bbbd7de81e2caf7076dbb9e2a4b6b2baeef0d39a32
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.17.post_attention_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.17.post_attention_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9f5293b8c8e09a5dc8e94132bfd475bfa4f14c3e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.17.post_attention_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:25602246f3782ff17620bfc3629f49791ee6d4014c45bf214f58d4897ebe561f
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.17.post_attention_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.17.post_attention_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.17.post_attention_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.17.post_attention_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.17.post_attention_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..64c3acc1e8a55c19c15108e1387e449d4673f0f5
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.17.post_attention_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3abc0c9d7275d502f768618f0a80150d6f52fa04dd0c89eeb3d826b9a4b5ded3
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.17.post_attention_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.17.post_attention_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..085dac0917341cfedada93f9131d69953851a9dc
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.17.post_attention_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:44331fa36333419e32ca2098f3a00cad1239f509fd905ed7116e7c3ae497d7f5
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.17.post_attention_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.17.post_attention_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2beb2644af38cbc388447b1005fd7052296511ec
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.17.post_attention_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1502bb53171926a5c33dd6e1cc2f307dfcf334be5904e81616b8d08c4a418ee9
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.17.post_attention_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.17.post_attention_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.17.post_attention_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.17.self_attn.k_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.17.self_attn.k_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5339b6c587161721fba3d22be8fc37379dc7e843
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.17.self_attn.k_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:36c787030bfbe8ac7924c6c1b3108e0f03cadb1b4abaa94d7a894b2130c00d0b
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.17.self_attn.k_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.17.self_attn.k_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ef90b6f69a1361bb8bfa6376d9c311890bcad182
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.17.self_attn.k_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e6bddb1ea75ad6652259bace4637644dbef6cae4c9aa8c73cfe73b73b1574f85
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.17.self_attn.k_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.17.self_attn.k_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..94e3535ed9291e05deb144af9c702c707580be78
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.17.self_attn.k_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:16488951f21371aaf5ee2a8791c12d2e71fef94a43deeacc2c649d9a1f7abbd6
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.17.self_attn.k_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.17.self_attn.k_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.17.self_attn.k_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.17.self_attn.k_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.17.self_attn.k_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cd800025f56ed0fa6af9c202fe3416c3d81539b5
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.17.self_attn.k_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a55afd00009909270fb491b1a26a668ca77ffd27b9e456ad2254d96b2d53afe2
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.17.self_attn.k_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.17.self_attn.k_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c1200c1fc0a539666ec6f7c61561d030b40074ea
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.17.self_attn.k_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:21d5664d3fa746f589bac77ef62d5fad5f68de74d19e7762e0e0943e440e7a9f
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.17.self_attn.k_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.17.self_attn.k_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ef9ec50569096eae492797f92af2a635a5d66d45
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.17.self_attn.k_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:036d2f29f261ffe6ec5232cfa006482ac509cc2c3fc5278268a4b90f0da98573
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.17.self_attn.k_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.17.self_attn.k_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.17.self_attn.k_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.17.self_attn.k_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.17.self_attn.k_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6da3bd74384ba530357458c197242eaafc29dff8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.17.self_attn.k_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4299255163333e73a3107a59d335717ea8dc9fe74d10213fe7a18305dfc569fe
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.17.self_attn.k_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.17.self_attn.k_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c0225ed87e2bea752a9199e29f5d49f934f9d9a5
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.17.self_attn.k_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f4046ea91b14ea55fba59ea5aa43d411624fcc13c4848416b6a5dc8ae3dfdba6
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.17.self_attn.k_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.17.self_attn.k_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e502f528177f9f656898883b0fbb92bf9836c210
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.17.self_attn.k_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0d96c53a6e877418bc1f965722950b8473283db6bdab0ab2c5694381342e94bc
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.17.self_attn.k_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.17.self_attn.k_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.17.self_attn.k_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.17.self_attn.o_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.17.self_attn.o_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..dfd885252a076dabbddb91b812afb69069b8b8ad
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.17.self_attn.o_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4d3c941f2cb84500a1d92866fa96a609ac6985af200805444a22200e817c1dce
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.17.self_attn.o_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.17.self_attn.o_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..38bfc2693d3e24d1c51d81e09e456ebefe542122
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.17.self_attn.o_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c1c7152a6c12ff43080120969c10d73c7ced22276d8c1041343466b94711c711
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.17.self_attn.o_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.17.self_attn.o_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1c7b056606346b814c9fcec82a96267d23e4539e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.17.self_attn.o_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3f3879f5d77703f78687d96d2fb0d40406c997e8a04c9b37c438ec3d604240aa
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.17.self_attn.o_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.17.self_attn.o_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.17.self_attn.o_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.17.self_attn.o_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.17.self_attn.o_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..790dcae07fa34fe3668cd7268bba2f901fc51aa5
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.17.self_attn.o_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f1337ded673d9e296f9c59aaca2c0e45750d54bb4e43f9e354301f7c5aadf658
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.17.self_attn.o_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.17.self_attn.o_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4f2b521d5c5e03163b4aedd9333002b3e3a8cc4c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.17.self_attn.o_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bd70501a8cc9c3df90dfb3152d0119cc0ff68ce42095126beae0818f16cf5d39
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.17.self_attn.o_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.17.self_attn.o_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c0236ef95f53b35efe71104e845bb5577465afc2
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.17.self_attn.o_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:895016f9a441f6bed77165a671904d5005223e2db61844827c509c7b29fbb50d
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.17.self_attn.o_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.17.self_attn.o_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.17.self_attn.o_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.17.self_attn.q_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.17.self_attn.q_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..dca4f5035ca77ce3a2c199a29b4512eef264295c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.17.self_attn.q_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3992867fc282c10e18cc412d528d63fd9c4be7b6fdb7fb901315da3f84483f2c
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.17.self_attn.q_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.17.self_attn.q_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c9bf5428620bd2e3a9d0ee218b6b15525dd5a96a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.17.self_attn.q_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9b5407409f2f41765e462824fb5d54b8d2d53d7214e96aab4b33c614e64e812e
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.17.self_attn.q_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.17.self_attn.q_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cffc77c17388f20a0fb336f64ebb49b226abe298
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.17.self_attn.q_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:11374cc0fe1e86960365dd63a9549b1012e8902aa8c4458d3ed003d635d1bd69
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.17.self_attn.q_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.17.self_attn.q_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.17.self_attn.q_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.17.self_attn.q_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.17.self_attn.q_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..bfd42086b495811f4d70948de143eac25493bedc
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.17.self_attn.q_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:af57c283b97d90729f0e83795c88bb343f486bf89e333efd8ac761ef2b893a23
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.17.self_attn.q_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.17.self_attn.q_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c5fa98fad7b71e13cdd2e5751af553a0d5cb791e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.17.self_attn.q_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:81c75215e9ea95d599cf745d1095c803214917bf1de3f8fafc0496e5f8d56150
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.17.self_attn.q_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.17.self_attn.q_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..beaa75fd0d8afd0ce264fc745cbf57bc63d7d437
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.17.self_attn.q_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b08050a0f47f3fe8a6aadd405fd1b56ed8e23d8a2230f8852d3b83977125a788
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.17.self_attn.q_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.17.self_attn.q_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.17.self_attn.q_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.17.self_attn.q_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.17.self_attn.q_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..bbc589783f7d6404f3d1a14fd250939dcb73515b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.17.self_attn.q_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9761cd57be1fbf00b234720fa355da263e228a921b38d24cc9823676efa61fdd
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.17.self_attn.q_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.17.self_attn.q_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..02a0ce9889292dbf318049ba2a73e156a3fe339e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.17.self_attn.q_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a67d48af7a633f63d3a80518128832203f8098a84773defe2d3a1249ea1d2a89
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.17.self_attn.q_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.17.self_attn.q_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e6525b4a27eed93777a0cd21bdffd4fb10e74e3c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.17.self_attn.q_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:41fccc8c7f7ad778cb1b510f44e1f1415d6c7d90a35bb818759db4314e3d45c6
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.17.self_attn.q_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.17.self_attn.q_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.17.self_attn.q_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.17.self_attn.v_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.17.self_attn.v_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..95b76c628d116c95ccaa5dcd4fb0708fe1d94a0e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.17.self_attn.v_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea2ee52445ad4fbb32b8431ed02c53a4b99a3e3779a65f8f245ef99af986f12b
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.17.self_attn.v_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.17.self_attn.v_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..32e72e2b3085b78fbca23a6b848d4e3cc1d21af5
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.17.self_attn.v_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e5d5a0232f7c3a32c8f78582abf0354495f4e09b759874c01714c73ac22593a2
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.17.self_attn.v_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.17.self_attn.v_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..346dfdef3ae95be9a5d3f41e8353601e666936b6
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.17.self_attn.v_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e242f32f0df973f91218a959dc7cda2a46c2fec2df2c21d9e336e353469b0042
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.17.self_attn.v_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.17.self_attn.v_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.17.self_attn.v_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.17.self_attn.v_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.17.self_attn.v_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f9a7be9f277754df9dfd9169f44e95d1f47b1a7d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.17.self_attn.v_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:903a9fefbae20189f37e71f2025e68763c94ccecbcf44504a7ec52ca118be946
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.17.self_attn.v_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.17.self_attn.v_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2afbd27cabc24bc56ab614fd28ede9326f67bac3
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.17.self_attn.v_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7a09e9e81c29b3d5a8fd65ee4252956da53bf30cfaa79801d90f645818257891
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.17.self_attn.v_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.17.self_attn.v_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0bfd96ca90841fe5be50c1bb3a5172054c2966cf
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.17.self_attn.v_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7eab7aa77e50737fdde9e2e9ff0c4879f14b9cb00ae21cef036fb2dd888a219c
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.17.self_attn.v_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.17.self_attn.v_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.17.self_attn.v_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.17.self_attn.v_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.17.self_attn.v_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..df37585e433c1d5ce9a77f60fec4f599feda93ab
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.17.self_attn.v_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f851df846ca6053b75ffdb8de31dea65184f57f02783f44df9da2e9b72781ba8
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.17.self_attn.v_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.17.self_attn.v_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d4d32b37bc668514dc9f7a8c67469f9affab5072
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.17.self_attn.v_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6784a52255bc7edb7b3dd83a5b82bc93241830280865d2c4c067fe548be2fa8d
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.17.self_attn.v_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.17.self_attn.v_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..594f2c772b86e8c09746cd163345ddd599141c25
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.17.self_attn.v_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:936207584d042e788283c77eeb6075347813b04ba092fa5988572b1a174a66a9
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.17.self_attn.v_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.17.self_attn.v_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.17.self_attn.v_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.18.down_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.18.down_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3498bbec161a1f09cb0baa2bc8847cba0a013f71
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.18.down_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4dd1cbce7a1fa6bc7ad27ec022a656c4a0629494d91c34be26c35776b5e5b2eb
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.18.down_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.18.down_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c00bb8e3d030ce769545a5de98f990bc0dfad603
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.18.down_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4fefa8064688f1898fb4676842752c4a92518aa44966dc4126aecf2b02d06d76
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.18.down_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.18.down_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1401b7c363657b2109463b3a2c16af36373a1ae4
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.18.down_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d51f43e768d6500ecd520a8e38fca9a3247f6fb832aa907c6f22a749691f86bb
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.18.down_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.18.down_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.18.down_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.18.gate_up_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.18.gate_up_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5af642fc236c08eb1a09cae9ddf3f1c9cb646b4d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.18.gate_up_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d6a6c7b7226e6ab9e114187827b891a0eb1eaeddc18692deb9d0175f53442c95
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.18.gate_up_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.18.gate_up_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d3c885b60ec3f4cfbcb95862021f2c185a515800
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.18.gate_up_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:269a88acb4b43a26728ba97a458e43448615f8ab14579b9cf23d021367c52b3f
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.18.gate_up_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.18.gate_up_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e04a07a431416492c0e5c1ef3b378927cd0b6090
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.18.gate_up_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d48dcd1ac4bf4587b2d8b00c002767b035e062e2d325d79a338010da94dccbe
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.18.gate_up_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.18.gate_up_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.18.gate_up_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.18.input_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.18.input_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b7fb89fc9574b5690218665aa31890e708b90b43
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.18.input_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6dca41b040220f72fab18a4c15ad7c81a3f8e0d9bae566ee363f2aeade31a6da
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.18.input_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.18.input_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..53e97ce39ccc839807283f0551bcd9f21d177437
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.18.input_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c5d81d883c67aa639ef90811523d4201a149a6e15b76c2bb8ecb451480583b60
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.18.input_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.18.input_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2e0468e499e221e9026438b6134fa0d9235a3b44
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.18.input_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:617687aa1fae16d605017752e339dac348347ed949e1690e9aaa9c2f3f7e4161
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.18.input_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.18.input_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.18.input_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.18.input_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.18.input_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7c660f5f773ec2707762f98f8b665d2faa8e44a2
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.18.input_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f987b1f5b5e313eb0f74722bdc5c0bb7bbe08db52d406df33302142a5fdb9d02
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.18.input_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.18.input_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..468bd0bfa08e35225e42a82d421a6566e98b3958
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.18.input_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:45a8fe8c207acc82fb0cb9e8235531cdb646f3c3b31f7c5b035904ea79797fe8
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.18.input_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.18.input_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ef3a42a19a34684a989847ea175354a6cdfa2152
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.18.input_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b478d08d86784a58f3ea72d2e6c5b4bb3db7abdef21210dc9e8ba2d24b61a9a1
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.18.input_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.18.input_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.18.input_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.18.mlp.down_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.18.mlp.down_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f9d72873c539edab00e72ee71ed04b4a5b4cd7e6
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.18.mlp.down_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3a3eb5ccc8865fc28ee422004ca6311e32fb8cf02252612a6efad938e4947dc8
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.18.mlp.down_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.18.mlp.down_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..627bedbb97c87ad3b1e1f3186e0379c4c810aa9c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.18.mlp.down_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ff3ae6b0b327b5d628558fe4ae0e99b835618624017c9a651982cc426bf8a690
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.18.mlp.down_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.18.mlp.down_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..80be36ba28c523da7efda5dcee1ffab5758ae3e7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.18.mlp.down_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f15aa56df747fa5e3e506f2bc26dd0115dda34298d136ea779b03018474defe4
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.18.mlp.down_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.18.mlp.down_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.18.mlp.down_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.18.mlp.gate_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.18.mlp.gate_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..796541ccfcfcf5f46bfe1c0b74b9bd173632e82e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.18.mlp.gate_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d2043e97d8a954c3e00a1f14b1b0f6d5eca5eab8d43d561dddad462b778a4698
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.18.mlp.gate_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.18.mlp.gate_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a9b7aef16f5fa6a97d6ea6b257a49884c3012925
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.18.mlp.gate_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0e6ca174bc78a1d9055c036eb7cdc7e4ec98c35b5005f497b653662f1f18d7b8
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.18.mlp.gate_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.18.mlp.gate_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d188d9b13382dee90da21fe12918016f4221f58b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.18.mlp.gate_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:486147f26d5327fa795077bd59378bf58994a48fde6d42fcf2c2d72b7cf0cf57
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.18.mlp.gate_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.18.mlp.gate_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.18.mlp.gate_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.18.mlp.up_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.18.mlp.up_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4d667e458e50b138ac60ffa6c7c82149d8821142
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.18.mlp.up_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:140a93b1c87ecced5d19fdfc8c591cea189754b80df214c3ebb24b52c8a27af0
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.18.mlp.up_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.18.mlp.up_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..438ec93a48ecbf2c1aae2c936273d1e7002ec1d1
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.18.mlp.up_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b11f89687f91b9d7b27812828069a37ac1c4d7f52f60b279909476b8dd46a541
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.18.mlp.up_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.18.mlp.up_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1c7e2ad5cc0022132dab4b0035748765e6d53b79
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.18.mlp.up_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f133053a20fa125ba65580b0c93b8c4a909381a71e44f2206e17956e04a44542
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.18.mlp.up_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.18.mlp.up_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.18.mlp.up_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.18.post_attention_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.18.post_attention_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..542bf4565ddadb609222357b5a68d13b7aa6e05e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.18.post_attention_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3e977660828a176e1bfed240d06d433d6ac7113365d2157e9057a780219a2723
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.18.post_attention_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.18.post_attention_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5b4e0b6ce2d2a1b6412a678ae59713975442263e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.18.post_attention_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9caff362aeadb62b59c5a9d45d72f87bd2ca37dddf5caddedc4e1b56ce076993
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.18.post_attention_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.18.post_attention_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..109a1f509bb02b151b9e127bbeb526016dd9cb9b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.18.post_attention_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9ab872207578984da589cc257c925f0d47c013afa32a1a6452b315a43445bfbb
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.18.post_attention_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.18.post_attention_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.18.post_attention_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.18.post_attention_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.18.post_attention_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0b75e142ab7f4f99b664f20a964bf37f0165568c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.18.post_attention_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:146f58cfa4bec3acea20e0cd76432e932cb378080dff7751242a215bf973e2b8
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.18.post_attention_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.18.post_attention_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..872fd811feb0e9240e1887f03cc91b70070bff5b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.18.post_attention_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:24e9f57cae5def885eeec3f2689a1f12a9ac5423e2e3bf83a725b4cf142f41fc
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.18.post_attention_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.18.post_attention_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7a7897627e987887d4b4ea97bc7bf0e833616b56
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.18.post_attention_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b87deedb0e026cb797dcb2a57e67398d2160c3a3745c79d0631f02b215d3ff43
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.18.post_attention_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.18.post_attention_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.18.post_attention_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.18.self_attn.k_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.18.self_attn.k_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1850157c836517126ea24dc80470a650e34eb52a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.18.self_attn.k_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c7de9b63e3f3c4cbf307008ffffafde2fdd42729ea722490e71065a31674d6ff
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.18.self_attn.k_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.18.self_attn.k_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..09a2a96e34a20bb187ae285bb79fa1ece40e6f85
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.18.self_attn.k_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a305aeb8e001e6ebe0b907c1124ff5971730e79eab0e0f7deb168c775917bb79
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.18.self_attn.k_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.18.self_attn.k_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8a1577f5cdc6bfdda63f67a8c5b2d83d654dac27
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.18.self_attn.k_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c71da02a5171994116aaaf040dbf53b0d5623cb85e5634fc0c4ec4d21304d2ac
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.18.self_attn.k_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.18.self_attn.k_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.18.self_attn.k_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.18.self_attn.k_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.18.self_attn.k_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7f3622c06410edc0dd89d4cf158462af08f21bdb
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.18.self_attn.k_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:14261d99ccde385a93aa77ecc459d7d9b45a123f40dd8cdf1899f61812845712
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.18.self_attn.k_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.18.self_attn.k_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d624dda71029ea2ab14e27ed053e9b6c1e7c17f1
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.18.self_attn.k_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ad36e3fe4b920fab95b65173b399db1374df6f91c983c0c3b5825f088d4f804
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.18.self_attn.k_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.18.self_attn.k_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7dcc43cec3a9caeaa45f2d399a74d327ac058c63
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.18.self_attn.k_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2af339510c486910cf877d722459e573e50aa1dace1316e4591e7f3d422420ae
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.18.self_attn.k_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.18.self_attn.k_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.18.self_attn.k_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.18.self_attn.k_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.18.self_attn.k_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..77dadc2a66ee2734e0bedc779f301cdc2d37b035
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.18.self_attn.k_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca52c5685b222f305b8a920c8070e5a209b5b29b8ef9b9e71cbb163ea3eb9aad
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.18.self_attn.k_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.18.self_attn.k_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..df264b40f1088f638f4df6bb01e1ef74577faa8f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.18.self_attn.k_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bfff7e279fdc1ba4ea8e44d67aebeb48a4cf6dfb3c28e036f872b03d51493643
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.18.self_attn.k_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.18.self_attn.k_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5204ea7607db35a2ce67bc44d637287fb658ce16
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.18.self_attn.k_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fffe63c0ef8b65f285644c62f7e7f35798416e3e44817a0e56d398abdcb3bbc5
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.18.self_attn.k_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.18.self_attn.k_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.18.self_attn.k_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.18.self_attn.o_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.18.self_attn.o_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8be86e5a75697b237262f5bb9bcee27e3ce21c1f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.18.self_attn.o_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:98c84a5258f065322a8cea03f012e2ecd5c7c3f0636cb3ff69b09e3cd47ce564
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.18.self_attn.o_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.18.self_attn.o_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a5b968d276a31b05615a1c601734b57ba07cfd61
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.18.self_attn.o_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:02db1833114bc7f42a80e627dc3e671d35f7bcae760a943707ea643ac9938e8c
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.18.self_attn.o_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.18.self_attn.o_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8a714b7b52080b03dadf253752712b577d5d9340
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.18.self_attn.o_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3ff480842e7250dee207fe2f106567966087e7dbba5260c5014e7eef76351f50
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.18.self_attn.o_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.18.self_attn.o_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.18.self_attn.o_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.18.self_attn.o_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.18.self_attn.o_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..937723168b7f91f09460f810f269b50aad178f76
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.18.self_attn.o_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d077eb3c02a904f0db2e2c98f8b19b0de6a9e38b64633b7501b64196f0f815d8
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.18.self_attn.o_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.18.self_attn.o_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1df98dc67b8c61ced5ff0ecd8856209e7ceffe5e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.18.self_attn.o_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0b4f8cf1c389616c76eca10962e9e738be1a2571146c5e7da08e104f35d5ff79
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.18.self_attn.o_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.18.self_attn.o_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..074c63dae0988457b208b824bd6f2dd0b49d7150
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.18.self_attn.o_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c6151b30216dc6aa542718ef89d7af6b17ce17e3a53f9412ae551a9e1f0837a0
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.18.self_attn.o_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.18.self_attn.o_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.18.self_attn.o_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.18.self_attn.q_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.18.self_attn.q_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2ba5596e6bd98bec84e87125808f8dd3e927318c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.18.self_attn.q_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:88b4dcf3129c1f88a7c2ae95d4f46a43e5f953a2a034486e99a32b1f346a509d
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.18.self_attn.q_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.18.self_attn.q_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a13ef229f6f218361435e24f2860f4c27162d599
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.18.self_attn.q_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ad9f92dc8edc31d12ee41db95ae0e12a0b7aa3ef92266f5c2d8e669fd6e60c7d
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.18.self_attn.q_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.18.self_attn.q_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7e1f17e2dc4e481f8f874e21e8bc90a8c697d48d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.18.self_attn.q_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d0b2c6776aedcfd4956a324eeaa2f693e2a351ea8493a93b078cabed6c53bb81
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.18.self_attn.q_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.18.self_attn.q_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.18.self_attn.q_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.18.self_attn.q_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.18.self_attn.q_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5660f5ca2b0974eee92a44e5fa0cb71c705102df
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.18.self_attn.q_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c6de66efffa6112202d0febe7f92721d44131e35e9eaecb6bcec0f24f7d2532b
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.18.self_attn.q_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.18.self_attn.q_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fdb84c64cce8a29cbf74a3ca056bdcb6f0c08998
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.18.self_attn.q_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7b8706a0fcabc597278f833276fcfb94fe63a0378b0ad738d4de9b5991a1624c
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.18.self_attn.q_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.18.self_attn.q_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9886e033a64c6d308d22a2967b2f038fc71af957
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.18.self_attn.q_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b92256116a8144cbab1f230e04daf3e70d926e41d8f9e2a207859d2f112d38dd
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.18.self_attn.q_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.18.self_attn.q_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.18.self_attn.q_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.18.self_attn.q_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.18.self_attn.q_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f6fb34fcd3c50ec486630e0eac92345ef00c70b5
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.18.self_attn.q_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:406ba0d35ab56a437767d1dbc3e3d24af589ff81cf50bd8f7dc72dcb1773b529
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.18.self_attn.q_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.18.self_attn.q_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c51a0a8473ccd7c4c10376991559a716f8ae2413
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.18.self_attn.q_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e7adf0f2827caf35ca0ebff2d9cb7f24c399330f90aacb3a635189a3c9b29d5b
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.18.self_attn.q_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.18.self_attn.q_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..40a1703138acde74a85d6df3b19fd38c82b2d11b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.18.self_attn.q_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:721a588cd97cda5ff886034f25e977b47ef99335373deca0012e7087f1e42df2
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.18.self_attn.q_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.18.self_attn.q_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.18.self_attn.q_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.18.self_attn.v_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.18.self_attn.v_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4b3c200320b36ab16dae6f97d0ad58a0f2201261
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.18.self_attn.v_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eae7d69b5ab146f0f84161724b8ef2bdd921c6b52ccf9526b417df8c851c068a
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.18.self_attn.v_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.18.self_attn.v_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c4b29398092c04f22a4596ee5b7c73a68306168d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.18.self_attn.v_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:56c9974e8822737f6955c02d2557fd091f1439ffe10e5dfc129ea0d05bbc1e0b
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.18.self_attn.v_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.18.self_attn.v_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..de8c577549b0775b093f671fc56e1bacd114719e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.18.self_attn.v_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:53d15cbc99a41ab7d256f67807bc0868c380f438c127d69dab376e7ed1f3571b
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.18.self_attn.v_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.18.self_attn.v_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.18.self_attn.v_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.18.self_attn.v_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.18.self_attn.v_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5281cd89d5c3a2abb7a3dae8e2dcce1324c32c6d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.18.self_attn.v_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cbe716618fffe4009104acb5d5a8606e5c4a3cc7af6432c9037f34d782ac3d6d
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.18.self_attn.v_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.18.self_attn.v_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..125702ae75aa9021ab426b302ce7e907f32288f5
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.18.self_attn.v_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe7e7808a2ab4f6d6056c154de0fbc64496924d2024d217eb27e546458f5d039
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.18.self_attn.v_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.18.self_attn.v_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a21d3a6437c3946b377b5af4216c9de578100963
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.18.self_attn.v_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3c3559f6ba6fba53182e03dac63f778d4d4470e4f6e2ffb3e0584b2ee46d0f3c
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.18.self_attn.v_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.18.self_attn.v_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.18.self_attn.v_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.18.self_attn.v_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.18.self_attn.v_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5c9da72a94749af0b63bf3f752a3191301bd1e40
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.18.self_attn.v_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5db980f9cf94842153fe63553d250df2d28eb1b698e0a834064c1c9e2e2f9352
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.18.self_attn.v_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.18.self_attn.v_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..df019a12e9cda9a9bcf4d8d5048dcc694b2477a3
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.18.self_attn.v_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:410aca4d9dfd098faf3b0d37e84a4d4a6eca44aac9901aac4d9579db834525c0
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.18.self_attn.v_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.18.self_attn.v_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0a1b62951d4ac23a5004bda785d66bdec376672c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.18.self_attn.v_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f969fded2b9ef116bc4acbfb62520a980546bfe087d438e580f1f3b12ed294e6
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.18.self_attn.v_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.18.self_attn.v_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.18.self_attn.v_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.19.down_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.19.down_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..19f085ecdc90959cd5873b820416559e235df80e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.19.down_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1bbb1f0e4be7905b602afcfd26ba4457399253ef9625182a2b4b62e8c61bf8c2
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.19.down_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.19.down_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8b431aa624c692fd3670df7af98e6a9c3846801a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.19.down_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b24a456c1d981af6581ed953cfc544ca60096005b469dd77d4bcb1a8bea21860
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.19.down_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.19.down_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fd3ba58c9974139349d8fb840b08fb1379ed14f7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.19.down_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:49f686c774e91c6d8c3f3398164ba42ea897fff2a674802fe35dc2c58d9333da
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.19.down_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.19.down_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.19.down_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.19.gate_up_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.19.gate_up_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8cfc968260d51c89bb0c05c70a25e44db3f5a604
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.19.gate_up_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:192df8f5e28ec477a34e3e90ae90cde9bef51d9618d2af8382f4e8d3d89dc25b
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.19.gate_up_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.19.gate_up_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..52cd4815bf370ca651390c4c18f78bd340531686
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.19.gate_up_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2f8a8f8606ada250f7e71466fde0a84367629d00485312170f33f910acdbe25f
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.19.gate_up_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.19.gate_up_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..40a7a55796e165353ec96d671a583ea2250927a7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.19.gate_up_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bcd85ba7adf373457c9964fe562b4b81a68342f9704050d53e3376643a99b25d
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.19.gate_up_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.19.gate_up_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.19.gate_up_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.19.input_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.19.input_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1637af1a15fe96f28ef52b8409f8ebc44d026397
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.19.input_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:39b424ffcd2db046345a28d45c63343be07c6cf34dc3a867d1692553b679f4bf
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.19.input_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.19.input_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0596ee5d6aee6c590a7d31b7aea01be256a0d723
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.19.input_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d8cdfe2a1432d3c833513bb389794dd61ce92884003462eb5d7d9e0e2d84fb13
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.19.input_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.19.input_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ecac9c78f696ec3e641bf5e3c6cf95b12f7487a2
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.19.input_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:80e42b687fd764d24a10579136f12a2ae5430ee23c875b10670882ecc0f613e2
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.19.input_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.19.input_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.19.input_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.19.input_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.19.input_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d5dd4258144d0f26e74723cb932aebb0c5257e8b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.19.input_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ec90d02f164d30ed5fa95b8aa0dd7d20bb4db292d41b3fea5b5b3655bed57c20
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.19.input_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.19.input_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0a3ae3f4020084e07c92d51a01c5214e0379b9af
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.19.input_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f0a882b484a5e38ffae47349d45165bbd6d7ce8c9a3eff2ab01eef4f3bc592d7
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.19.input_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.19.input_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ae2ca244e55791cde9ce294c7ae49dec18ad35d6
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.19.input_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:76a84a833e7c2fa39d563c57d2a445b1aa1c9187d050a6583fce5fe3dda248c6
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.19.input_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.19.input_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.19.input_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.19.mlp.down_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.19.mlp.down_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c6e12dbddf2f9e505c3480b0384eca1ffdf0af57
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.19.mlp.down_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7417c9eeda8c4d82d9edd72f19af1ed01c52a06092922945376a0c647e3458a2
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.19.mlp.down_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.19.mlp.down_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d4786374283ce9b5f6586dd4ca9d3bb16d2f71f5
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.19.mlp.down_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d94b89e564ae14beb5a89a632e2545f2980c550b9455f463185f4dd540478326
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.19.mlp.down_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.19.mlp.down_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d8935847a934e3b2655775c8bfa2fa01c8f7a0f2
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.19.mlp.down_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a82113b4e3174a183e7b6856c9eed6f941299837becf508f1f75079a087b963e
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.19.mlp.down_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.19.mlp.down_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.19.mlp.down_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.19.mlp.gate_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.19.mlp.gate_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9be3865bf1f5a52c7618fa5ff811c6a959c59a35
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.19.mlp.gate_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8fa6eaf178ba99ac5433cc8cc00ee877ff80e3303fec64a7ec4eaf8e633c138b
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.19.mlp.gate_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.19.mlp.gate_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..853e51245f65ee6376326dff3413e1a5eaa7a615
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.19.mlp.gate_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:883650ed38790f0e6c5a17b4443872325542800afc0c11337b0245cbc1469c79
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.19.mlp.gate_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.19.mlp.gate_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..473f1eae68f603e09a14337e5d6d2bc2573232d4
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.19.mlp.gate_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5354dadb7cb845506a31f3072eef94fd84417c527708454e7161eb9c4f621ca4
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.19.mlp.gate_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.19.mlp.gate_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.19.mlp.gate_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.19.mlp.up_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.19.mlp.up_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..47584dbc4319509c427198c24fbe5ca1ff67adad
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.19.mlp.up_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:de0217497084cf8806a0a4465c20c0ff69dfa396485a430f95f36dedbbc89ec5
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.19.mlp.up_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.19.mlp.up_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a77fa1739f174dd82e4399dfa8388ceeea05a1f2
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.19.mlp.up_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:db2a1b4997bb2a3232eb4448ef0240e7ef27f832d7a00d452569c323e9677dbf
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.19.mlp.up_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.19.mlp.up_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..dad62480c558cf7c2d0fb5a96d7ed0ec2c4e6e78
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.19.mlp.up_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d6a191bde2ee33a9deea4ad0f1733fed1ae72df6e4cbd8640a03cbe115e54a96
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.19.mlp.up_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.19.mlp.up_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.19.mlp.up_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.19.post_attention_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.19.post_attention_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d6c3dde480d7866242d0df767c6c7cd7a08d8f26
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.19.post_attention_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5e5fe82c80c70f1fc86b77d4f26f7bdcb87f5ab8312ebb82c463904eeef53968
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.19.post_attention_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.19.post_attention_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..24e65c27d1a122f859b0c8b0c84e6ec3e1d3a97f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.19.post_attention_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2c602d51d7c5b5e09a9b809634275d2e169722a0221a037d7bdf0f457e6e3169
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.19.post_attention_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.19.post_attention_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..59e72df6a4321fe6631d3ce3be1205a713f6157c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.19.post_attention_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6d0b96fabcfcdf4f5f3f3cb11ead98d134f9d92ac3748eb8dbcc173208f2733a
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.19.post_attention_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.19.post_attention_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.19.post_attention_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.19.post_attention_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.19.post_attention_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c289bad2dcd8d0acac14257dcba1afe842f6f7f9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.19.post_attention_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3a519ab859536fbe51f0afbb064b11c0e88af655b079c3269529359d7cc49e26
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.19.post_attention_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.19.post_attention_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..77721293625ec8c23a5581d4cfd012810224f667
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.19.post_attention_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f25d60d40b899a14ce6d4d0eac70fecf5a18a126b9dff9036327b847e9188129
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.19.post_attention_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.19.post_attention_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..316e97b16070b0a5fa17391247d4f0b6fe1de6b3
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.19.post_attention_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:39281d2e5e3ec150a87a031d51768e7eea59d239427a38ea6cb0172e02c422dc
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.19.post_attention_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.19.post_attention_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.19.post_attention_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.19.self_attn.k_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.19.self_attn.k_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ba2f175193c54a05f081957cad3ac57b3bb95e0c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.19.self_attn.k_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:35f813a7f47da11c4fe6b8b0851ec70574f8f64f34c0f02e6c27f2165659e355
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.19.self_attn.k_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.19.self_attn.k_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1c710936bee202132666153fd0fde5c0035d878a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.19.self_attn.k_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:af112b4bae1349cffb015e94b10064b9f3f7e8d83bf5f8d7f48d9cafcc8a07b9
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.19.self_attn.k_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.19.self_attn.k_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..86bfc54419173855d4e0c664411d091154d523f4
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.19.self_attn.k_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:38c92295dcdba985c8ebff209b048a91dd16d7bfdec224907675e1230e852bd6
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.19.self_attn.k_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.19.self_attn.k_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.19.self_attn.k_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.19.self_attn.k_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.19.self_attn.k_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..951d5dcefd8e1d1734f4f3f5421bbba9875e4b8f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.19.self_attn.k_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:94ff87e00ebc851b9a23f855da47d517f21abe05f6f79b54d93ead10df8ec93d
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.19.self_attn.k_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.19.self_attn.k_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..405ce70b58687b92777a1311edec74f3439d0841
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.19.self_attn.k_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a769970cf0d0dfd5fc00b6540c2381ae679c19db3289ba483b1f81d5400abdf1
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.19.self_attn.k_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.19.self_attn.k_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f9180d99dcc01f549d0e1766e7ea7d015ad52bc9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.19.self_attn.k_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3cf344484a6eecffa020d9daf930ece836473fece21e1119a725f2029e1f027e
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.19.self_attn.k_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.19.self_attn.k_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.19.self_attn.k_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.19.self_attn.k_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.19.self_attn.k_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..685af6f0193e93d9e6e4e9b6f3ea6836f4f47e16
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.19.self_attn.k_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a410d0532b98684fd3aea146d8a55bf596db969d43fc725799b997b81e7f89ee
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.19.self_attn.k_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.19.self_attn.k_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..be335fd8ba73b59fd26cb337dea16b6b8cc1c570
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.19.self_attn.k_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8efeee7d225ad510423385a4f399a9dd88693f9603da584e55e46ada2b1f084b
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.19.self_attn.k_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.19.self_attn.k_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6bb900ad5038f7df5dc00b97e4c5309b845c5a6b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.19.self_attn.k_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:67793e1e83e55bb883f57b64a1e772b0a5f8af1d88ee5b1b94e329320806d75c
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.19.self_attn.k_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.19.self_attn.k_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.19.self_attn.k_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.19.self_attn.o_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.19.self_attn.o_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b89a6c726d0a41b43b7a3e9e22d887baff3bc0dc
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.19.self_attn.o_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a01d828c43be157d1f5124293e110e1656b35a3ee112ad2b5e12835c60332a51
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.19.self_attn.o_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.19.self_attn.o_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..71e3dd609588b927d7bdff6ac179b6b8f4a18e46
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.19.self_attn.o_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d5f133686ed103ec7a6934b9a9ce7a2d99cf1ce43972675848370a0bd8e3a10
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.19.self_attn.o_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.19.self_attn.o_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6f278d58703e8c9d28b71f3ff220c69b799b8e86
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.19.self_attn.o_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e32f13b1a1f71cfaed6a12dad91c4ec1538161bacd90edde5e95435263d62f64
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.19.self_attn.o_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.19.self_attn.o_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.19.self_attn.o_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.19.self_attn.o_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.19.self_attn.o_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1e9b585c2ac8368b63bffe9f6a11775ffa411057
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.19.self_attn.o_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2f0df77532066b7bc5e74562a956d608b194211dac97abf12e93d39c1ee5a402
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.19.self_attn.o_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.19.self_attn.o_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..264f2fd1be16f6277be314f086f50c44e5b8037f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.19.self_attn.o_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c8bd0e396bf9a878ac592d43d980fae653d6bd9d254966ff19393d9ab6451578
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.19.self_attn.o_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.19.self_attn.o_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fc5fddf105559c8f50d0ae3a3582f81a1eebe83f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.19.self_attn.o_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f6072fc51dd678afefe78a0b159df4f54d37827a582038b2f7b0034f6ae41c10
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.19.self_attn.o_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.19.self_attn.o_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.19.self_attn.o_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.19.self_attn.q_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.19.self_attn.q_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d3a5456ff0b94c4ebf5b6b1ab1762fc25cabfd22
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.19.self_attn.q_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:31171ec8244c1a0267f3c649e1a8e46c05c6905bc733a88a300925d00199b0c9
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.19.self_attn.q_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.19.self_attn.q_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4df8d00714e592799bca671bc6a8aa4fd3c50eaf
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.19.self_attn.q_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1bb0a9e25eeaca0dd7b46b139a96aa3e9e089d4fb422e3604a05401337c62b0d
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.19.self_attn.q_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.19.self_attn.q_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2b886afc16b8e6296b2c08d0c076b048983eb47d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.19.self_attn.q_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b62ecea3c0c993c087054c80aedf9d05d37f12b8f5e75e7281576cedc900b291
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.19.self_attn.q_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.19.self_attn.q_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.19.self_attn.q_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.19.self_attn.q_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.19.self_attn.q_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0c2bafe86bdb2238324294fc38f2e69930245028
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.19.self_attn.q_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc10bb33255de6b739ec73127645041ee9daf6bd881d08ef2696ce0af725767a
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.19.self_attn.q_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.19.self_attn.q_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..66fe6b8b267f5d15d1777f59f7f583c5c61894d8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.19.self_attn.q_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:927d56180fac293e13f65e3242c8c99cdefdc501919ac68e4079fb158bfeaf47
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.19.self_attn.q_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.19.self_attn.q_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5420f7275b48dfcea629f8f2053c78656ed72354
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.19.self_attn.q_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6d1dac2188088191a303d67608a90f3d1594d2ffc68509031583ab38771ffe64
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.19.self_attn.q_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.19.self_attn.q_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.19.self_attn.q_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.19.self_attn.q_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.19.self_attn.q_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7307b1273887d62d8431fc860d8ed8154466138e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.19.self_attn.q_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f0bbcd46417abd376f223380d632d62388a4ea52a7ea2b2312f6e3b973d30226
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.19.self_attn.q_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.19.self_attn.q_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4937561ebb7c8e37803494a8d277d81f54b2f413
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.19.self_attn.q_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c01023efcf1787b442c15965b308ef8d67c67380019dca32e7625b7091418fa5
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.19.self_attn.q_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.19.self_attn.q_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8295d96fd98b59e695a1df9d8f98aa419e63c129
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.19.self_attn.q_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cf18791dd94885b2e7e1a3cf9208708b7bf1223f5df1362d4ee32c2d3ed8b3cb
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.19.self_attn.q_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.19.self_attn.q_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.19.self_attn.q_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.19.self_attn.v_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.19.self_attn.v_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6fddde98666fc73e6f34201c1a7c5a6cc52631ea
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.19.self_attn.v_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1cc03970f322a60bd55086f918dc65d607c9c85c4f346662ac22e3206701d02e
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.19.self_attn.v_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.19.self_attn.v_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..23486e20868e198c8d5380dea5dd0cf13c7ff135
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.19.self_attn.v_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4aea3a3e965598ef586be1f6f002c0b61b6ed5ec12d82e47045b55d09f2718e4
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.19.self_attn.v_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.19.self_attn.v_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e8c50584711019711a6dc19745a2f575c3c3cc17
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.19.self_attn.v_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:369e560e41fad177f40d54407d653fec87636f7eaddaf04699e0cfee6c518e6a
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.19.self_attn.v_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.19.self_attn.v_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.19.self_attn.v_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.19.self_attn.v_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.19.self_attn.v_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..26d0195d386a71cbee1be7c37097078a0dae23d0
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.19.self_attn.v_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e726f2ae0eab9ccd5c4187c4632c4368e3fa39e56d7e354e9441ee9f2e87de29
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.19.self_attn.v_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.19.self_attn.v_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f6c9072361c2a20e65285d29d9f404ea63c30e3c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.19.self_attn.v_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f5537f272b013271c5ef8c45de503be87019f03b3eb1b44abcd5c6cf81d783c4
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.19.self_attn.v_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.19.self_attn.v_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..67b0771b364b1ab487afe0fc71757d85b5e6b97b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.19.self_attn.v_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:25afb6d7a305dd40322c51bf8fd61dfae522d12940c61aa40d0fc845da20d3be
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.19.self_attn.v_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.19.self_attn.v_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.19.self_attn.v_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.19.self_attn.v_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.19.self_attn.v_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cf5282a921c12c83649affb98dc865ba03ef64c0
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.19.self_attn.v_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ce4878667342902b6620f5d42d165c0a58726772f4e2f4d6171b6b95213133a0
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.19.self_attn.v_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.19.self_attn.v_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0b04ab3d2fdb451807786f61eae3591bbb7dece8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.19.self_attn.v_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:19086529a0f8ca0fb823764b666440c9a6e931f75ea594c03007907b001451e1
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.19.self_attn.v_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.19.self_attn.v_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3a6eff9c49b2ec87692070d2320261f34e6df9d0
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.19.self_attn.v_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:774d65c58b3c2470d1a3cf00cc0abde883393a4c8fada79cb17de7977430ce9e
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.19.self_attn.v_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.19.self_attn.v_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.19.self_attn.v_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.2.down_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.2.down_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9e31ba0c38e4394d7b74fc77b307580e1ba7702c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.2.down_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d6c860d774b29f7114eae4379afe509564896980676b9c6dfbf0e4481edd4e4e
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.2.down_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.2.down_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..52ba0cee6205566d327b88745cbecf5922ae0730
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.2.down_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d93e0e695159076d22d52a1c4fd8d49b3bd7e4c794be2c0f20a9654e5f1601bc
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.2.down_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.2.down_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c2cb8402025357ed335e14751a7c9f2ff2d80e84
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.2.down_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cd0bc37068ef136ae913cd090cb14e5a8401f087db10524fb23a5203d40c7959
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.2.down_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.2.down_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.2.down_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.2.gate_up_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.2.gate_up_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e1ac723a0bbc7dae6325225044c19d47067d67a7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.2.gate_up_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:39b60d91633d195a67025495610372980a0155a01f28c49f3059a092d8fd5198
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.2.gate_up_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.2.gate_up_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5cd0a8e2d0570d5f40bfd4a71df5756804969c79
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.2.gate_up_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4fcc59585af0be63a9676fb10e6e11462e7fffaaeeab4a33912098a7e5d2c7c5
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.2.gate_up_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.2.gate_up_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d8e6760598bf5e90915463365bcd9581450cdbf6
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.2.gate_up_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1e74a304d3caa08ed7da23cdbdde1fe846d642438cd9016d9cae328e04d9456c
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.2.gate_up_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.2.gate_up_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.2.gate_up_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.2.input_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.2.input_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..12a492af3ae051c1a6bcc96bc87fdbb358d1ce9c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.2.input_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4050166bdf1833e9be26f325518b64612511a8190c3fcc525c2534cd21d4a893
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.2.input_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.2.input_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..80e5825af2d8776469446e2fcee77f2c2108375a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.2.input_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:96b8a6769dd1e1460a9e5c144008763d6a0e75c174237185416b270f3ca2f134
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.2.input_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.2.input_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6cf63aea5d23a88e1ac61a271c994a457dc2a282
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.2.input_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:89f598327644518599e871208d088e7b6369e363508666d6e26f615ccca6f271
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.2.input_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.2.input_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.2.input_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.2.input_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.2.input_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..30664170ae0c6226559dd571a2a57e512a25f2b5
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.2.input_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d96fc4a0a66c37eda5379cbae33566a919b53de4c17fc234ada86ef05f7dbfb
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.2.input_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.2.input_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..07c8a3fbc3db5b84b4447d1f1a624a65172f6a54
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.2.input_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:997b929cdb1f681109912816de5fb5a9bc36298235901a5a3e7a1be98d354ed2
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.2.input_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.2.input_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7a7ff1437b9fa7a2dead6c030f3ab272105b3fef
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.2.input_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ad3b121d978c3d8f7757b83b7eea0f5796c7b03591f96e9c85054d5c57848556
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.2.input_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.2.input_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.2.input_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.2.mlp.down_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.2.mlp.down_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..dacf62ef43d955709b8ff321571dc12deaf6ffe1
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.2.mlp.down_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:42bf2d239fd249e091370ae0f8705bf2927a3ddfa672358df983173e30dd71ac
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.2.mlp.down_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.2.mlp.down_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8db334dc4b3e4d5c642af1cc9766003dbda24fdb
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.2.mlp.down_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d3136c90adc50e5472b1f26f1cd33686b06c61cb5ae45687f9d1992a1b721ddf
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.2.mlp.down_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.2.mlp.down_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0158cd6bd70c59586f39c7d8cf291b59c96a80fd
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.2.mlp.down_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:27cd5e20256ae02de4703f6c4370f544a707fa717c2f7044b451ae3edbe3d85c
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.2.mlp.down_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.2.mlp.down_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.2.mlp.down_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.2.mlp.gate_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.2.mlp.gate_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5bf9cc92b54683939403e4536f5dfd4e51731d9b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.2.mlp.gate_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ccd5ae32b6ec01ae5a24c036e9a3f6ff26fa6e588334c56a66a26fe8400dbb0d
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.2.mlp.gate_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.2.mlp.gate_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..963c0d9176ee50482b2f6391af2aea48f4eb637d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.2.mlp.gate_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:745234d2e34db40fb02ee4671480e3c08eaf6df7ae043cf8ca853b856de26178
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.2.mlp.gate_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.2.mlp.gate_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e6ddfaeedc97228faa171f3b793b1f93cb524fb3
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.2.mlp.gate_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8d67e6c26b86dd38e9ce8c6fb101bb6f83e9d7a07adc679ae8b1bb77f34b7e29
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.2.mlp.gate_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.2.mlp.gate_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.2.mlp.gate_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.2.mlp.up_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.2.mlp.up_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c40bdbe1f7ee9d732cd0b6c016fca9dff7d95f46
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.2.mlp.up_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:43b9be1f709651e649e03017aecb37500b5569630da4cfa5999fb4489149fda6
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.2.mlp.up_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.2.mlp.up_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b46aa75a4c35a99d2ce21c42f8d2441a4d47be86
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.2.mlp.up_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:380667f9bd3fa81b6cb5b552d5efcaf1ca84804d431f0c25393136104496a0f9
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.2.mlp.up_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.2.mlp.up_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..bd4e8ecd51e3a0023c870fc8ab2d95edc4d9a142
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.2.mlp.up_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:17feb43d1e52a87563ab36de0b9c5666695722353a2526c62d8e72822103b5a1
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.2.mlp.up_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.2.mlp.up_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.2.mlp.up_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.2.post_attention_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.2.post_attention_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0a710e07958de8523284c07ea8d64f8eaebfe8e4
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.2.post_attention_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f087c9520696136a2a91a038ffc3a01d7c5d7e526e0f64b97c864f21a8047d05
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.2.post_attention_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.2.post_attention_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5a52ac06063d5520ca2d3e0b0d6d843729f97a8e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.2.post_attention_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e3eeff5fc5ae1aa8126396d079ad8e142ce87f1a15b02e15ec9cfb6b40e4ed50
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.2.post_attention_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.2.post_attention_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0107afed9d589721e8fdc237f45c7292ba0a96f4
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.2.post_attention_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:10361b0635c2c671433a8c8edd6f86eba32df8e74d7064a2c3d9438bbc2f0d52
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.2.post_attention_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.2.post_attention_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.2.post_attention_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.2.post_attention_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.2.post_attention_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7fa9b522c20419e7bcb9a7caeaff9179abb1031b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.2.post_attention_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4b27b320b0cfe8cd2d6ea0ff2d53efad5f4e736c82c9c0105e86a264cdab2222
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.2.post_attention_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.2.post_attention_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..dea50d251b29026881b6eac12bec57284a224141
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.2.post_attention_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:87033269b58b84785c49d3712583aff32256cd6db1f91f3ee5e5f065d6a1ec17
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.2.post_attention_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.2.post_attention_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..05dd68998a7420d99d4193cc83ec2afb60545f7b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.2.post_attention_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d430d6700fae07801fbfca03694e6665c7e0926c9e310b2120af6ee05b80ee4b
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.2.post_attention_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.2.post_attention_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.2.post_attention_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.2.self_attn.k_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.2.self_attn.k_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..eb331ec154598ec5a8588ff0a2b716b82a1498df
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.2.self_attn.k_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a78c75690f5d4821e6a02957c27669b94d15e20a3a6082a533daec874fc6c176
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.2.self_attn.k_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.2.self_attn.k_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..099a6903d26fa3c96541d7544569636aba836314
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.2.self_attn.k_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9f22ce51bc67e0ca10395d6a2b0094ad1d2d2be093bb9e44a9361c16fee50d11
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.2.self_attn.k_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.2.self_attn.k_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fd745209492661192bb79645f46cef86c051f1eb
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.2.self_attn.k_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:883ed92e4029f7414225fc40414d29472043feae376e4e45401f2ad827873d1e
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.2.self_attn.k_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.2.self_attn.k_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.2.self_attn.k_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.2.self_attn.k_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.2.self_attn.k_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..33abf88caaaced3c6019efc270599c453471b631
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.2.self_attn.k_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9faf7201fc4b8c0fa352f52304bfbd7219d61a997ce6bfcafcc99d3d422939fd
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.2.self_attn.k_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.2.self_attn.k_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..60519325d7a91a73caac396f7c80706e4e6c38d3
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.2.self_attn.k_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:654fefb02622891f7b6b736bb2a4bf620e91abb77443b71e5dcdece1b250920d
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.2.self_attn.k_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.2.self_attn.k_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..92a58962261b9e7be4833a9088e8b6776aad021e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.2.self_attn.k_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a82ce55c3c284b9ee1085164f718d1934cc4fc35e6f88376046ce0aa369432e1
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.2.self_attn.k_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.2.self_attn.k_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.2.self_attn.k_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.2.self_attn.k_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.2.self_attn.k_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..dc540ae5889fc406d8091f71d265a2866322ec77
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.2.self_attn.k_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dcce796ac5cf2296f8552159ecd6e22adfab8d2b6bb1583771a3579382abd456
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.2.self_attn.k_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.2.self_attn.k_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8eb670662a049bdbbf7007a960c15c1f7b05de3c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.2.self_attn.k_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:02fa2984b7ad55ff06e2edc9466468cfbee110418d0b1ff09018d2c16b7ff9a8
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.2.self_attn.k_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.2.self_attn.k_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2b2189525aefda25a6ec74a5f0f67edf21572b88
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.2.self_attn.k_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3808476fb56cd3583f4ce3f716ab9d3fec560136a0c8b58cf9edf942c3b8729c
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.2.self_attn.k_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.2.self_attn.k_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.2.self_attn.k_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.2.self_attn.o_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.2.self_attn.o_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0dce5364c30f66bb63c2f165b7d5ccb5d6631a65
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.2.self_attn.o_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:84b81e8150c4de7c153b7c27338a49ee9fc7cdf564009f6dec1e2b740d60d23e
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.2.self_attn.o_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.2.self_attn.o_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6e68be9a658028049171738b59afe1e33bc3a497
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.2.self_attn.o_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:14e74b9419ea19483a6f8f4e543c71f3ea90b1d9b6845545390ac4c99058381d
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.2.self_attn.o_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.2.self_attn.o_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..77eab295926c3bcec74cdf484b701cd2f486aa8e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.2.self_attn.o_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:945ae19e8ee8d03885b528195f471065c71e52471e3400c8937cb4f19d89b0e9
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.2.self_attn.o_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.2.self_attn.o_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.2.self_attn.o_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.2.self_attn.o_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.2.self_attn.o_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..58c4c96dd6a3afdf215df912ddab32167b97e365
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.2.self_attn.o_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:980bd76e058ec9610660bf25e35080dcce20e9122dd8104c58961b05da4738fb
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.2.self_attn.o_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.2.self_attn.o_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b8c00a7c17a00b9ce59b1de479cad65ede836d6a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.2.self_attn.o_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7c7fc68dc07fec1b19b0becfe036b917474c5c55cbe9a28234358c042d40b507
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.2.self_attn.o_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.2.self_attn.o_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..418b5ead9d1d1638faa8bc5b162d2eed12f61c94
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.2.self_attn.o_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:635e82d71f1295f5415dff73c768ab60ea3445905f1712de070878a787ff65ce
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.2.self_attn.o_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.2.self_attn.o_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.2.self_attn.o_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.2.self_attn.q_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.2.self_attn.q_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..613187dd08aa27c50472ad2861e8c7d1be900e7d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.2.self_attn.q_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3f38527392f01e7732b9e9ed44e3705a8a570595efaed17fdd2695e3087e78c0
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.2.self_attn.q_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.2.self_attn.q_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..091ce6b2b6b9a70882543801f738c06755ec62be
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.2.self_attn.q_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1c8c01ecb71181f4af1c7928d52efc5f8fb9dd6a3c62870625427fe777f93c24
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.2.self_attn.q_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.2.self_attn.q_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..17b532b6581e26c53ad55662df7ac0729d23166d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.2.self_attn.q_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5c55042de1588bb63eeff64e39ac4602f2596bbaea0bd60ef5cca266b5d18d5e
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.2.self_attn.q_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.2.self_attn.q_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.2.self_attn.q_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.2.self_attn.q_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.2.self_attn.q_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a11f98b91594664c474fa8cdc44589a1a7c4739f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.2.self_attn.q_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d0070af3c86d52d34d3558d674edfba8307ad861d5a896b903b3433c55655f46
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.2.self_attn.q_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.2.self_attn.q_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..70c2510f12e968a3912021fcb6f8f94ace3950f0
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.2.self_attn.q_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7b49b9f3a22b3f7b46eeae3db411a4aabe95e0064bd8707776fac9b8a5c91545
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.2.self_attn.q_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.2.self_attn.q_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..541676955d335cf7f76bc06f98cd2b85fffcb7c8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.2.self_attn.q_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a1143be2f5129efa94fa691c4ced44e5ada541b55177327b09d6d5beb8614e65
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.2.self_attn.q_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.2.self_attn.q_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.2.self_attn.q_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.2.self_attn.q_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.2.self_attn.q_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..50512140a607ad08fffeb2004337557addc01553
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.2.self_attn.q_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f24a6c6aa64e57bc864c99e1b93471c44c3f2321290202a7567e6762f4468029
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.2.self_attn.q_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.2.self_attn.q_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..279724fc7585696a700d7eb632e4a82bfbeef48b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.2.self_attn.q_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a8eaac6d8be8527acbbf989fd4e380ee37fb606767a0e7d0a1942f5001e8dd49
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.2.self_attn.q_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.2.self_attn.q_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5e7fb6b98393e1bcbe55069f385e685a20f27036
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.2.self_attn.q_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7fd641f5c919290854c9a6b4a8f845766dde05971ab3ba901ef8cd303e299857
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.2.self_attn.q_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.2.self_attn.q_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.2.self_attn.q_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.2.self_attn.v_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.2.self_attn.v_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b60d0de0fb65364afea7e571bce5d0f618d93629
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.2.self_attn.v_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1f7357f1bfa166ca9e5e18d596b5cf86b4f48ed96cba739c11d4e5a1c951b51f
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.2.self_attn.v_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.2.self_attn.v_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a90662ccf61f55a2acf1c79e1c282a414019750b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.2.self_attn.v_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:20c5ebc7f22ad856f7168461de9cc69627518c8fad8b828c0110b5aea8d5203f
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.2.self_attn.v_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.2.self_attn.v_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4fad171a9256e0a8a661f42893cdc243344098be
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.2.self_attn.v_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:987a02720f7a0587ae9643e2f2afbbff37a515d6e242224115819274d0e79c19
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.2.self_attn.v_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.2.self_attn.v_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.2.self_attn.v_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.2.self_attn.v_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.2.self_attn.v_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c320db2e15b80a9ac40375431f7cec6255157cb8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.2.self_attn.v_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b5be598543106108da677da39f17cab21aff1eacd8f45c32b28bcc45587b9591
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.2.self_attn.v_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.2.self_attn.v_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c45db73110c541dc2b948939294a35ec7358f076
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.2.self_attn.v_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:56533d6045c839329f1940d29b10cfb0db5df087a5bf46151e669c75815dec41
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.2.self_attn.v_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.2.self_attn.v_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..edb353d9f36e9c84a7d0bc0a2275010cdfae0f22
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.2.self_attn.v_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d7f6d48dd4e7ffdf8c891a6add50206595bcff3980f6e3a112ce0455dfacb61b
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.2.self_attn.v_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.2.self_attn.v_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.2.self_attn.v_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.2.self_attn.v_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.2.self_attn.v_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c484516784e6c623933439989589b56ae863e6cf
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.2.self_attn.v_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3bdc0e7eb9eb6afd48ae1bd276e32e397ac8f5b3e0e5d1479475d0c399ba4351
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.2.self_attn.v_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.2.self_attn.v_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..76ab062d21f8a134cafafff594e89c4c8e01c98a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.2.self_attn.v_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4f283b41e54fda4a0b856cea4bf8ad52239837323245b2298b754d1637cc604b
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.2.self_attn.v_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.2.self_attn.v_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..321c5ddcea9aa8fc234679165c960910a6f760b2
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.2.self_attn.v_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a472a65d95cf0f25b749571ac5e8fc3d331b8f558d82be1e3be36aa1b14f3a41
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.2.self_attn.v_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.2.self_attn.v_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.2.self_attn.v_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.20.down_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.20.down_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..033374018b732d6cf9e1ce7a87ebe5b73e2956c7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.20.down_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3c71ed63daac67df4eaa14f9bad0e06f5200839453f27d8495f631a8bcf93643
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.20.down_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.20.down_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9a9dd453ce3d8c30290941e1938df29008c9755a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.20.down_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fa3ecc930ead32c23297e2418e33a49ee94b1234788b8a9da54171204702bd4c
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.20.down_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.20.down_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d48bfd9ef1f3af61c6e67d8098bcc66022ae4e2f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.20.down_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:69c9d9693759817893e495216124c235ca8249abe0547e0c872210aa96364dbd
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.20.down_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.20.down_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.20.down_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.20.gate_up_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.20.gate_up_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..08639f29d7e3c3cd1ad95ded588551fc987b5784
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.20.gate_up_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:03698504288acb1954817bbfdcace7f9e3781e300ee4feb4d94832ec995921dc
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.20.gate_up_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.20.gate_up_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d776e2fa3e10d6679310b7f8b64669d4c754280a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.20.gate_up_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bbb0375db9943b95121da203468b9d9e5f734703cd23aed9f54889e379ca27e7
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.20.gate_up_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.20.gate_up_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f9b79903e9529d341e4ee8f6b2cbcd348db67035
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.20.gate_up_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:76380c8adf7c98591163678cbaefcb2a4884736caf02b96be5340291bdbdb30d
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.20.gate_up_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.20.gate_up_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.20.gate_up_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.20.input_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.20.input_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..284dc2501b961d0de95be590f6f65a87ff0045ce
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.20.input_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eaa751166e0327bb89eaf8e9623991a5001867bbcf4de58092bd2e93796d3243
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.20.input_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.20.input_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a0818b0c7afd62b7dd7af81ff3c5b2bda242e23b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.20.input_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:494a2eca2179e309ddc097749f56f9327bb00312d743932f8f2f571352370b7a
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.20.input_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.20.input_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0db34d43b2735823e4b7e2ef71563359177ecb92
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.20.input_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1106a1cc71b9806ad5041f2df387a9542e72fdb99f256e6bde9247bc0c8e9d24
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.20.input_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.20.input_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.20.input_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.20.input_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.20.input_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5671b320c33b9efc9acdc67ae45b511600be1ba8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.20.input_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7a72e4ce0b39ddb91df3a4d690600713feb4328b42151df8a463495b29e809da
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.20.input_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.20.input_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6ce47e11f410fe81b20ef2ef2ca5745116316792
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.20.input_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:99a887eb338059949955dbe6cbca9eb134d332f006b403a2d98316b84188a29f
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.20.input_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.20.input_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..09bc78eec841d6a4ffdb742e991f5071230bef41
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.20.input_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0759d3239118bb51d18e33c5cb1cd76f222ca7f752e620beda5e616e4f47fd2e
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.20.input_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.20.input_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.20.input_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.20.mlp.down_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.20.mlp.down_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..197df9d810e9813f98c81853009b38a532774ea3
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.20.mlp.down_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:42837c7bbcb3f57c6eeb1a01826f7cd7b06681d2b5f21d63c921495ec4b114b0
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.20.mlp.down_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.20.mlp.down_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e7beb2a4647ce8a97bb7af2cb0882659d40d0963
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.20.mlp.down_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8814bcf1974468c655829d2ffc83933fc933ae737bc041805a395a9c13f925ac
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.20.mlp.down_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.20.mlp.down_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2b77a2c1cd4761d77ff6ebbb8596b4a9e0e33535
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.20.mlp.down_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:58c5c880ddfd567227463cf18ed1b6647f6b9d95bf13cad20e4fe3bf04f1eaaf
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.20.mlp.down_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.20.mlp.down_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.20.mlp.down_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.20.mlp.gate_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.20.mlp.gate_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9382b60b63e4e12e9904cdeadca2bb9340883926
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.20.mlp.gate_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a267ed356316ae2b7b8eb2f6556d168d90596afc6e59713416e618aee7fcc952
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.20.mlp.gate_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.20.mlp.gate_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1c8c65e4bb447218c55850dd2f1242a6b821a1e0
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.20.mlp.gate_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:52d34e00ff3b5f47b7bf5b628307809968c4fd81cc1d21a4027760dfee90a39a
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.20.mlp.gate_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.20.mlp.gate_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b54891af799fffecf10b222d80147f8c085bddfb
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.20.mlp.gate_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ff44f7c5fa6eecd8b6af5da85b4b855ef2cb9075aa0f0e017889b37de197b4ff
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.20.mlp.gate_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.20.mlp.gate_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.20.mlp.gate_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.20.mlp.up_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.20.mlp.up_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..40814e58ac93c15792fde2ed84647c58a0803811
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.20.mlp.up_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8477df7ecea3e935ca841c66a19f0c8d9b000299060ed44c9ff4cba766488fce
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.20.mlp.up_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.20.mlp.up_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ada3a80595bbd5630310e2abb5b905f08ec934b3
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.20.mlp.up_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c742151a6d2c45d8f2f28dfd936d0caa7508059aaa5729945892ab6736fe4c55
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.20.mlp.up_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.20.mlp.up_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3c5b53311e110550329f41fa713e3c4f9102c039
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.20.mlp.up_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4b959154481514782a23894f46ef88daf3899e35a6d53db98337ab6b68a1b49e
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.20.mlp.up_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.20.mlp.up_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.20.mlp.up_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.20.post_attention_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.20.post_attention_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..16a7eea06028efa56ba21bd217882eaf3fbab2eb
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.20.post_attention_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3e6bcdd575d43d2905e4faa7426117274d491b007e6898a150a8c6f6b3e6c075
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.20.post_attention_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.20.post_attention_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3acf795d34daea88b7a844de6c673531906ba5b4
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.20.post_attention_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c663e9cebc24a82a6f5aaff3090e76745729ceccfc856197febfb0b664da08a0
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.20.post_attention_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.20.post_attention_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..54990764eebe2c264a73604063468532ff49c033
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.20.post_attention_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:65306f9d6b53e792a264cfba0f28c47adcf224410da363defafb0b0bc45e5afe
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.20.post_attention_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.20.post_attention_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.20.post_attention_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.20.post_attention_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.20.post_attention_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f6eb11b9eb8d916d1aed153e3e3237b13f4aee95
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.20.post_attention_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e616eb361a7809ba1a247b5fafe024aa7202b4bcd403dc2ad19872d62b4568de
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.20.post_attention_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.20.post_attention_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..635ce9f753c729efb12b36cb35d40c5eda68f627
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.20.post_attention_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bf325e0bc87851ac83c0bccaad04ff51d92e0afa85730f3a995ecf481f43752d
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.20.post_attention_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.20.post_attention_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..733a34f9052003e431397b607a54025aa4ccaa49
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.20.post_attention_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca9729d05d0441082e969fd495ff56080e7f8367ecef2ba734b2ddb1b92e0d5d
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.20.post_attention_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.20.post_attention_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.20.post_attention_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.20.self_attn.k_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.20.self_attn.k_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..460f7f34e87aea5eaffec4bbc92ae927f2eb0a74
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.20.self_attn.k_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2fd08757291fb1b8ab090f8aac69dd097e85bec27fa61951ec5bfef0f010e847
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.20.self_attn.k_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.20.self_attn.k_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e23873b645eaabc96638023c39d29f60deb791b0
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.20.self_attn.k_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9da7de3694d0ef4a1645fa97aeaff36d6ed8633769512869eda7d095a74ec098
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.20.self_attn.k_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.20.self_attn.k_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..50a44176e05778366dc50b0919480a4848b2cbae
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.20.self_attn.k_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:838954d88ff74d38a9e2ed19ffed57733dafb6b71ee161b16c222443908bd2d0
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.20.self_attn.k_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.20.self_attn.k_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.20.self_attn.k_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.20.self_attn.k_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.20.self_attn.k_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8ce182da753637cd616934e890a77256cd0fb34b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.20.self_attn.k_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d8fdcb6ef637b75ccf87029d5112697a76ed3e2b60fb8b7ea05a1e72a4681620
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.20.self_attn.k_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.20.self_attn.k_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..47ab51cea9bbd6f1db077b08c84d3ba52c87d617
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.20.self_attn.k_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d4a9c7e4cd391f234f64470a86a9cd569a89300dfc4a64b1d47684a336b3c8f9
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.20.self_attn.k_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.20.self_attn.k_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e8ef1139fc690d89b66135ac415406bc26aa0ba5
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.20.self_attn.k_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b25e07900fd97df6580b3da83e51c1bd7c7be2e755da9468106b1fb02a72fe1e
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.20.self_attn.k_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.20.self_attn.k_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.20.self_attn.k_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.20.self_attn.k_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.20.self_attn.k_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..dc3192e5ec429f51ba84d099572b78af1b2e8572
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.20.self_attn.k_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:da20902f6f7312e3b6951ae659bb8705785a93747028399b9b861a632449e07e
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.20.self_attn.k_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.20.self_attn.k_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..21e6d08e57e764fb75acd6d3eafb2f77cb8d6b48
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.20.self_attn.k_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5c6d3f976ae3fc5b8e135520ed249ee87c75329d8430c6fc534cb229bacbe053
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.20.self_attn.k_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.20.self_attn.k_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1d24910c3fb9837f9c81951d1d8c3628ce9d4b46
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.20.self_attn.k_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3235ab5e175e24e0738d8669eea9deaf01e2481571bee8bf51f88c43126aafd5
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.20.self_attn.k_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.20.self_attn.k_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.20.self_attn.k_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.20.self_attn.o_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.20.self_attn.o_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..808c4551e4b8e337a77586249898e8f57549972e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.20.self_attn.o_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d9a8fbfe329c0bf39f81089f75e72059f2dab8c9d7d3bfd2369e3240a209d6b7
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.20.self_attn.o_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.20.self_attn.o_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8ecdbf9b756ceb6128f709d9575c682452b4f80e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.20.self_attn.o_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1693a6db7c6efd60488234bc4432860daf11286a34ad570a52490821149eff50
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.20.self_attn.o_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.20.self_attn.o_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f3063d13ff46f78fa805365b3d75b06e5ec464cf
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.20.self_attn.o_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:03fbcc30f2952554309c163715b488f4f726d0896bdb1cbe0a93cd469d00c9b6
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.20.self_attn.o_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.20.self_attn.o_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.20.self_attn.o_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.20.self_attn.o_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.20.self_attn.o_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5f442b4c6e34b00d7e570dc680a15bf23b188732
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.20.self_attn.o_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:34427251dd18ecc3eebff8dadff945103aaec3d7d9ef44cf4ee3897f2e62e162
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.20.self_attn.o_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.20.self_attn.o_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..506780571c4c03363268e6adb7d7cd0e70acae0a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.20.self_attn.o_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ed1bb3fd09bffe5d3dd51ced5a458174027b3533c23aec536c213792661b1e2c
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.20.self_attn.o_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.20.self_attn.o_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..564c61fff6be7aae6a6cca40216ab8a5dbc3c548
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.20.self_attn.o_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a643b7bc59295aa0bb6fa5617275a74ec6fcf0c08090c8179e65e4026f121641
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.20.self_attn.o_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.20.self_attn.o_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.20.self_attn.o_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.20.self_attn.q_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.20.self_attn.q_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d4ed2ae4d71d01c9871204bc89a8e40700d98809
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.20.self_attn.q_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:70d29daa326928908fa0a477435f3655b52cb3366305ad4b0a4c4691becbb961
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.20.self_attn.q_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.20.self_attn.q_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1965da0de450b187d811250b9b2c763dc747d5e6
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.20.self_attn.q_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:570cfe57cd2227e0df9654367a622f0b7a5dfbad9d62c2752ef9ea11261bb1ff
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.20.self_attn.q_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.20.self_attn.q_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a44239bce44b86e53e275c98b98c008edca0ef52
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.20.self_attn.q_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:52bb9345b9c7f71606479c5386894e958ea2f5abfaafca5c3393e88c0ad4d461
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.20.self_attn.q_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.20.self_attn.q_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.20.self_attn.q_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.20.self_attn.q_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.20.self_attn.q_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c44d9f5ec16ddeaa045f922164a4278b7b713d7b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.20.self_attn.q_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:00d4b26b4297ccffad40eb19b24a8d3aa782cbac16f08e12b84256e1fa1ca673
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.20.self_attn.q_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.20.self_attn.q_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..69e889dacbda0fc14cfe43c412253fae1e91da79
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.20.self_attn.q_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:90eee49f968663090c2737e425bd9fe21b31b1b357be6098e3a5ebd742a5646f
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.20.self_attn.q_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.20.self_attn.q_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7138267a9f9dbfea659fcaf90da5f9d42040926e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.20.self_attn.q_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a39a0e8b687f6a22ab97a6bc17b37b41aa8f8424361881a2a6d058d0478bffc4
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.20.self_attn.q_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.20.self_attn.q_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.20.self_attn.q_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.20.self_attn.q_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.20.self_attn.q_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..bdbcbed018abb121c80b51063b65e099106ec877
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.20.self_attn.q_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9757ea1ef7156fdef2e23d97da298758182c01d37fb3f6d5826190f34aab8b1b
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.20.self_attn.q_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.20.self_attn.q_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f7575e843442a2fe0cb8b8a37e64a206c59301f8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.20.self_attn.q_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e0142b4b04084c359a83b75eec62c10f3168c8bac2ef432514fc903e99ed0073
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.20.self_attn.q_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.20.self_attn.q_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..de6ae81fcc25ed7679b8b35f9c874b7ff8193166
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.20.self_attn.q_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:48dced95eefb6c80995d3fe330b89dce79005567aa4253c47ac88191cc713f17
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.20.self_attn.q_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.20.self_attn.q_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.20.self_attn.q_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.20.self_attn.v_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.20.self_attn.v_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..00fd8f72b2d4b09346a0b4f128510f7438e062e0
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.20.self_attn.v_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0661df205ca325a926be876e2de097450d4c22c8aefd287afe8cdf88aac8c626
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.20.self_attn.v_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.20.self_attn.v_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cc50af8797c23cbb563df3f69f0ffa37db87df4c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.20.self_attn.v_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0de84cc1dcdf18cd8222ffc4a1d9439897207da212ef2550afcd90b2c4aee7a0
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.20.self_attn.v_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.20.self_attn.v_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..00a4b004793a57b6c86950f0d90c57d41cc80f61
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.20.self_attn.v_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3a48c1aecdc3f73a9babee8989f55364898a87fa51f33c6e4460bb590f7e2a69
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.20.self_attn.v_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.20.self_attn.v_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.20.self_attn.v_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.20.self_attn.v_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.20.self_attn.v_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..06a58bdd744cfb194b1d76823f0115983348dc14
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.20.self_attn.v_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:731ccfa8348f9f99b510dbae41961e12d6173e118c60df628478a93eac633444
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.20.self_attn.v_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.20.self_attn.v_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5cda75565d849573f7e13b0d0ce28b46bca72471
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.20.self_attn.v_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0685e3adb989e8ef995be4470d07aa3735860bd5db9808b80abe4580b0ef421c
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.20.self_attn.v_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.20.self_attn.v_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4829cb25a80e58d97c9b75d053fdc1d831cf9825
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.20.self_attn.v_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4ae451333aae99c9bcd915da6ed903259f621b8eb11459aa2c353ec2e0acd185
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.20.self_attn.v_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.20.self_attn.v_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.20.self_attn.v_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.20.self_attn.v_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.20.self_attn.v_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a0232269d93c647e6d21d0e2ef8ea9e1ea3a0885
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.20.self_attn.v_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9b48d329dd7699eb3caf9b1bdad8571d92294481556960f9a79d36c1e642c436
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.20.self_attn.v_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.20.self_attn.v_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e3e9373302add000ed4556aa564819f8a8c24bcf
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.20.self_attn.v_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc3d4a897c04355435d5021fa8796b0b3adacb3cce2f745a3445e072f03c24de
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.20.self_attn.v_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.20.self_attn.v_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..110a331a5bc73c3a176f13eb4572a3e170a8488b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.20.self_attn.v_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:642d331ff34d039d0121004bec02ace4de1a3b702d4b534cd5b210119aeedeac
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.20.self_attn.v_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.20.self_attn.v_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.20.self_attn.v_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.21.down_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.21.down_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..49cae60ef38da03525751aea9fb419c0922a92d9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.21.down_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e627ddbf709c20bca5f631c21c8a428d84cb9986784a1fdd781e3b98a2282486
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.21.down_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.21.down_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..177c1f019c25114652871e2ba5837a6d3d04c6f9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.21.down_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:87b509a3d9191da1b27c0c4d011440ec56d8c58fdf70aee63c129f1937f6ed8e
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.21.down_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.21.down_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2ebbaf6903feb5ddc8a13a08af042b96515d16b0
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.21.down_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:baab37118269778962d58794ff18b51b02423f42b34aeb535cf46278474316e8
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.21.down_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.21.down_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.21.down_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.21.gate_up_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.21.gate_up_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..584679d35c67f82e2e2f04c9a62b4124d7c72061
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.21.gate_up_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4917564f522479fecbd4ae5e1d61832095040bc17793d3a8100a70a38531ccd9
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.21.gate_up_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.21.gate_up_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b2f5039da7e3093e838bd0dce6033bd2018830bf
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.21.gate_up_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:45fd44b652d98ea159e2264ff394bb8c58789d0e5007415c4ccfc4fcada4b202
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.21.gate_up_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.21.gate_up_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5ba342389d860ca414b9bcc0e9b9a2149ee6f256
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.21.gate_up_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3461af88214185e5f51ac4a958ea52505f4e8c663c6f57fd8b6474767ea1d2bb
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.21.gate_up_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.21.gate_up_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.21.gate_up_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.21.input_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.21.input_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ce5a8c0a78bb38c8f4f4c06ada5e71d8b167a94d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.21.input_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:68111897eddc3250a1bac84cafe3f0d8743d5b90dc840a4e2a446df4a4f02f79
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.21.input_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.21.input_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f75cd01e13a6c08e37128c6e74f51e6a33887726
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.21.input_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:77d7e78855be47f2c44e4fa856d796a0fce7379fa985a5105a87119a1d77f386
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.21.input_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.21.input_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..44abf95577ba4a2a64fe975996ccbf6e2427784a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.21.input_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:020093727ba73ccf81b219926ddfcec72e6bd21d0afa00c59da8ce865497b90e
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.21.input_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.21.input_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.21.input_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.21.input_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.21.input_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c693f0241cb3ed56d40035cf0031056356124ac0
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.21.input_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cd398158558f875f2ae18126ea6890148765221cb31d9b3a552d85b32dce7c5e
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.21.input_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.21.input_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ea4c136e7788770ceb5610fb3f4adb450aa81f8e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.21.input_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8c368c6a3b98d136eca917b9da5733f861aa5e363ac805e0a91466c280b3fa39
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.21.input_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.21.input_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..09e72e294c11449dfcb55efe14d13d334afed0c9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.21.input_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ec14b6592203c3702176494bcc1c8758ee65243fe742d7d6ec6eb0ebe6cff4a2
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.21.input_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.21.input_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.21.input_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.21.mlp.down_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.21.mlp.down_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..83d1d2159d6bb4a5d4b5e27665221690fdbf2187
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.21.mlp.down_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:30316daa39fa401e1f9357846faf5d7c4ade6ce2881b500d40275bc3ac47dd4d
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.21.mlp.down_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.21.mlp.down_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..15cd14845c3350f39b61fbf1560474d39d9b8e5b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.21.mlp.down_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:11fa9f23136c94bb46317762c29af8bd5abcdef6ecbaced8732f81b442b25a26
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.21.mlp.down_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.21.mlp.down_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7a1aea90805dcfa3037183ec18e1576f9d04df8d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.21.mlp.down_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:65057fc61693f7e7ea4cf30e5f9375f426a971d2647bbba2ae0185d7604d47fe
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.21.mlp.down_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.21.mlp.down_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.21.mlp.down_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.21.mlp.gate_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.21.mlp.gate_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0f7a284ae62b587285b1b768aeb7fee555d730da
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.21.mlp.gate_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:613e93047bd575df837d117159d92d19e2a9d78a4d0013c689c5da18ec8d4671
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.21.mlp.gate_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.21.mlp.gate_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6e5172925ae1257cdb4fb7d7f7b214c095386f4e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.21.mlp.gate_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e9555b17fd742b675ff9257c163272f6e84abfc926bdc4dfa3a45237e5b20d88
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.21.mlp.gate_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.21.mlp.gate_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8e6abfb26c071340e5676a4543ae01dbbe0433b6
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.21.mlp.gate_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:59442f0598af04cb18cf48107ed139f5b02b52b2e910fd9e54ac33e1d6067dc7
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.21.mlp.gate_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.21.mlp.gate_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.21.mlp.gate_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.21.mlp.up_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.21.mlp.up_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..df5b40c9c9360778e57d1aad95839d1f97753f4d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.21.mlp.up_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dc8757c9b083abb1353d49d21df29b69ea6b36134bd676b4c84effb8eaa2af14
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.21.mlp.up_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.21.mlp.up_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..06a0bb1396d641603e199b3ee57426f592abcdcc
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.21.mlp.up_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d003be6e1151f6e17442cca6b95fb76dca77594877ecbc0955a13006c8163ac5
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.21.mlp.up_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.21.mlp.up_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..824b6e61942ad5b7d1fc4a6db0a043aaf97c0c58
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.21.mlp.up_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:48a9fb356a58855b101fece128b1268c7569ed102dd19e7171e07a704d6f3101
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.21.mlp.up_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.21.mlp.up_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.21.mlp.up_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.21.post_attention_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.21.post_attention_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8c6d7167e64638634dffdc01fedcf3b22a8868e3
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.21.post_attention_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0d9c1860257bfcb997bd31edc9afe95f4cd80416600b770b2acc802ad34360a2
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.21.post_attention_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.21.post_attention_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..44c421f5f4fccab7faa966829b3e7ddaeeba46d9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.21.post_attention_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ee55fe3141f37c4b98d3bb96e20f2e28795e25729f4ced6c35f3e7114b389a0c
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.21.post_attention_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.21.post_attention_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8d5f3e5dda0a245ad1cc39fc488d95c5ff529912
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.21.post_attention_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7fcef9add411e7f3c6746d542ca706eb4b8003fb30e14cf899c449e75fe6f5f5
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.21.post_attention_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.21.post_attention_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.21.post_attention_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.21.post_attention_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.21.post_attention_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..eab0196fc00bdd62c666a4218397eaf3ccb195ff
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.21.post_attention_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:62e54fd55c11646f197ce4e3d2fb08c771baf9926fae8b018b446fe95ddfcb4b
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.21.post_attention_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.21.post_attention_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8686245cee427054c7d99feae4ac1b858ce2d851
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.21.post_attention_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:39768a7b21bd4deee8eb5c60de0c2af114a517d33d3961e4a9b98e43fe32cd9e
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.21.post_attention_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.21.post_attention_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6a2685ef2ea3fe586fb5dee3b5341265792fb05b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.21.post_attention_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e904be20a74d4b419a6a8a17812d3f845b2a2ca537651bb938195c4230e170e8
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.21.post_attention_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.21.post_attention_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.21.post_attention_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.21.self_attn.k_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.21.self_attn.k_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..02da40ed43cec8cd4d09e44ebb26fe271b21a302
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.21.self_attn.k_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:524252bd39f46bd446ef8238c2dae3bc309dfdd9f48138c4b134754481728cca
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.21.self_attn.k_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.21.self_attn.k_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..08083202095683ce445299843384912c17692a6d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.21.self_attn.k_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:11cab976c09615a42c5d2958bb7ad6097ceb7d084c8b81c5d7c2d9cae026e68f
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.21.self_attn.k_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.21.self_attn.k_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..425443b04ce4778ee264d762e71b3bd7f0c75edf
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.21.self_attn.k_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8fe5e813ce8b519745438fe6ebe033612879742f4c7a9d20f3162f33b6dfd6f3
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.21.self_attn.k_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.21.self_attn.k_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.21.self_attn.k_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.21.self_attn.k_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.21.self_attn.k_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..31bfed94bfdee2f3be28f14d2d48c5ffd4a72840
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.21.self_attn.k_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f0cb9df81b2322cc7da923bb0e1df1fe3b62ea2dd724fe2bab9cb2d9b0b8eee3
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.21.self_attn.k_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.21.self_attn.k_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..081cb6a058072b5dac858ff05329796cbdd30929
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.21.self_attn.k_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:13f3765c3ddd73c87c589394345148123692f4d87026173e4f4ebbcbb5eaff7f
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.21.self_attn.k_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.21.self_attn.k_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c3bdd75edd8b5ccd433e1abbef1baf438a750b3c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.21.self_attn.k_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7b7b27b810b3ed52757d92a67550942a4ee0cc22bb8797d0d45009cdf876d70e
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.21.self_attn.k_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.21.self_attn.k_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.21.self_attn.k_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.21.self_attn.k_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.21.self_attn.k_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d41027e7b232b937ea920c33bd4feac5645dbf32
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.21.self_attn.k_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fa5be65373154ea760ff0edc26342499f7c9e6131dab3c301a61b3b64cdd59c3
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.21.self_attn.k_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.21.self_attn.k_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..03eaa4b3a4a27871a0d2860f90454e696db094f8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.21.self_attn.k_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1b0fa9944dc0bc6d2c1f4ab9601e3ad3fe55ffdc883480ccb264893c5d1dd30b
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.21.self_attn.k_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.21.self_attn.k_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f972c5cc198ebba725ee226e03e848ac987f8abc
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.21.self_attn.k_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0f745f078be9881680cd3d7550d5ae8fc3ba9d5732e09697ce080a3da4d070b7
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.21.self_attn.k_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.21.self_attn.k_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.21.self_attn.k_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.21.self_attn.o_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.21.self_attn.o_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..08996c3f6b8ff6056e787ba6843312dae038e0ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.21.self_attn.o_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:76e30786697ebf8e16d6c1416d17c5a45703a76ca686b7ffbf7b167f5dffacbe
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.21.self_attn.o_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.21.self_attn.o_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..83b93cac1cebad9ea399a211eae7641b762d9994
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.21.self_attn.o_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b09cb98571e67315f044948674bc980eb4b0e7f1335b2fa0d1dfd3b2d97eaab1
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.21.self_attn.o_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.21.self_attn.o_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f8dffceac81cf50daee0c8e294c51dd02ebaf993
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.21.self_attn.o_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fb31afea1e255dced6133a5cfa9d9c648ab405d1921926cd95bef7aef8b88ad2
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.21.self_attn.o_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.21.self_attn.o_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.21.self_attn.o_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.21.self_attn.o_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.21.self_attn.o_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4ab0c9fbb5d46b3fdba7a49e14d05ebbf41e9832
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.21.self_attn.o_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:978a1027b5e1d15e5cd70b146562a97d3096104fb1eeb1c2e6bbb85e1c5be5f6
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.21.self_attn.o_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.21.self_attn.o_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..708dbbf1a3fd258392804ea8a2fb012f9df6bdc6
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.21.self_attn.o_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4b8f81b73abe1b4df4e7774d2e246ec1bcb6e36bdf9131afcdc27268bb7aedbc
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.21.self_attn.o_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.21.self_attn.o_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f58c51e53c132e4af6c30f137848985564b2bfd3
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.21.self_attn.o_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:130421786dd6c6b1a9b565a7387c08bf0c980a11e6c066063f513a26bab51e13
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.21.self_attn.o_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.21.self_attn.o_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.21.self_attn.o_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.21.self_attn.q_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.21.self_attn.q_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..08cc1d96676cbe18f7db037665e517398ea7492f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.21.self_attn.q_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8f45371c5fbc3de816c64d9986674b138015198956fbfb2bb7e06fe4e09ea510
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.21.self_attn.q_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.21.self_attn.q_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..300f4d54a4bad3044902473e63d94279f04a5643
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.21.self_attn.q_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f8b85748ab2a04cc42239448e37dcf71e4037593473bb969c4511f36ac95f0fe
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.21.self_attn.q_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.21.self_attn.q_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..59e9d30f8059cca24868291a1c72fdecfebbbe0b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.21.self_attn.q_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cb5dc240cd6078d197b7f92dc4b3476019bf881434c720ac7928b4931b1e419f
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.21.self_attn.q_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.21.self_attn.q_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.21.self_attn.q_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.21.self_attn.q_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.21.self_attn.q_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a541fec5528fe9d3457416140f9e50f6e62eb20b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.21.self_attn.q_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3d82c3734bb4d318d24746e09ca592c4d6b3657979b0199c6ee470fb009cb600
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.21.self_attn.q_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.21.self_attn.q_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a5bffc1ad6ee94c415963732f8cf80cb516e9828
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.21.self_attn.q_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ede52a4416ca4cf53ff0189f6a2a26c03321c9476e2bb6ad0c559feaca62b73a
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.21.self_attn.q_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.21.self_attn.q_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e7d5f6a27c4620401652e5351b437e21362d2ef8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.21.self_attn.q_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:428fbbd3f2606ac225040c1b371787c304c05c39632ca6ec0615ac2c08d5a8c7
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.21.self_attn.q_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.21.self_attn.q_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.21.self_attn.q_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.21.self_attn.q_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.21.self_attn.q_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..76f5bc94b417eae949714ad79505edb37ce778e7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.21.self_attn.q_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4fbe9f030e0b0a391110b76d67f5fe662bfd7097edcccc6601a36ee3f214a874
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.21.self_attn.q_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.21.self_attn.q_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5b4788b95234113ee12d8a9dbe0dcdd182fa601a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.21.self_attn.q_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d8706970219ce107ed96a3660f8292f8eac3eb2e481e6c23d1e34316d9ccc97f
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.21.self_attn.q_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.21.self_attn.q_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c37da21999d7aab0b09cdc7cb0eb59617a98709c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.21.self_attn.q_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ff715947b6a0e24ff2aad1e51b16b44b330374a7bfd389dd9c445b0574a42b0
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.21.self_attn.q_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.21.self_attn.q_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.21.self_attn.q_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.21.self_attn.v_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.21.self_attn.v_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3534e3194206076a0e248bd58cbf47cbbcde2eae
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.21.self_attn.v_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:44161d01632e995d94b22eabeb9e71697a362b1e944275b6aabf30115d4ed6d4
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.21.self_attn.v_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.21.self_attn.v_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a93ed19473b9e8034e04cd86d76910647c50a02a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.21.self_attn.v_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:54362761b8871aa87fd9ae8d5b60b4c46d9950fd0a0a7f1a81a6b4de2a704d75
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.21.self_attn.v_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.21.self_attn.v_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0930cad09a2e092faf8d8528fa03f8104f637774
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.21.self_attn.v_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5e3c24b30873423125818c5045d41883cac0f69909367d74fb9ddd10993130e5
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.21.self_attn.v_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.21.self_attn.v_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.21.self_attn.v_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.21.self_attn.v_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.21.self_attn.v_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..08eda35860f47838abec188e26db8a8280ecc8eb
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.21.self_attn.v_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d2365c1b4de393bb36d08a09ae122e7d91ce9a2ecdd97c9fd186e75861848f22
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.21.self_attn.v_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.21.self_attn.v_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c43902f07d8063341a9ecb4327eeace8aceef28d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.21.self_attn.v_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:486765da7f6afaa9e3eb08cc36888fd020db124dbcb92d75b05e1b0a02b559b5
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.21.self_attn.v_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.21.self_attn.v_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3398f435c756d3276e7ec8c7a84429268e55452f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.21.self_attn.v_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:79daa647f8443eb95ea490cf405334081f8e8bec3232e98bab9ce17206801924
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.21.self_attn.v_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.21.self_attn.v_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.21.self_attn.v_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.21.self_attn.v_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.21.self_attn.v_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3568c224252b86f8410dde3b340e7470b0fb419a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.21.self_attn.v_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca057ab9ab95918fb1dd93e5e3386c69767c64fd21ba2392c3249ae20a6ba071
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.21.self_attn.v_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.21.self_attn.v_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cb27be113df709aad5e368cdbfc92d70656b1802
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.21.self_attn.v_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fa98b2be6232c33fc057d0087d8fd02398115f500c69712f63a805926b257697
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.21.self_attn.v_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.21.self_attn.v_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..139bdc52512353ba60009ec52d00f1729a3413eb
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.21.self_attn.v_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a320a97b4a75204812ad4ac143180137fc9d8dae6f2c5c1404c335bf4af61652
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.21.self_attn.v_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.21.self_attn.v_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.21.self_attn.v_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.22.down_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.22.down_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0d6111011ec0206633065de3875a6a07bf1197dc
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.22.down_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1f05e118befd579680a001ab9877878799ac5a8d0a94a3d5472e84b22852850f
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.22.down_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.22.down_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..67b43495d3b1d2ddf7533ff1842829edb7c83b9e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.22.down_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4554b75157bba3c67b5399e8081d7605352c822c5ddd17350ec163f8ac1eec7
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.22.down_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.22.down_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ccedcce5ff90b7a2c7d1a57776fc20659125ff0b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.22.down_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f5b88c23ae46ec349f62964cd3ba7106b151a49b4952a1f94589639d13bfcb87
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.22.down_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.22.down_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.22.down_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.22.gate_up_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.22.gate_up_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..18e7b142ae5be95da668e1f94513117abaa692d6
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.22.gate_up_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f1375bdf28192141908b9f9be32def936099bad468e706a83659f9e4b3c76ae8
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.22.gate_up_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.22.gate_up_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..00957a5a82630f86e0af4e233ca2e679c3f85701
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.22.gate_up_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:196f574c1fdf93a2ebbe1df14bd0f3857a87ae56531dece0b57a551c24615d6f
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.22.gate_up_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.22.gate_up_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..061312fbd152110662bdeb0c6db22acbb57d7bfc
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.22.gate_up_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:da281139eff43ab0064b36020b6c3c1ad29446534e04d36f7a8ae89cfb603784
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.22.gate_up_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.22.gate_up_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.22.gate_up_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.22.input_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.22.input_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..96c9d99f32b0f57da770b4a9fcbfd6e0b48d9e2a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.22.input_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:37a7a02bb113086727df1c04bbb4f10098e1f11fb733ea98cd188327f32431ed
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.22.input_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.22.input_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..067cd25becd4a17eaad9a6ba86b69baab51478c8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.22.input_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6a46e5e582e97a1e853abdb9ebe92e8ccdb792f9ef7f9bd5f1810def5072943c
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.22.input_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.22.input_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b4a0b2bcb2316303c34fab82fd19b2cff1f5cb39
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.22.input_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6d33feefb2799711da0acbc40a2d2d11e5174d582266a5a47af32079d8d771ef
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.22.input_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.22.input_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.22.input_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.22.input_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.22.input_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..99d0466369760a759b4379c0a9297126107344df
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.22.input_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b22da8218997872b5df41caf6fe4656c655fbd88a84e05ff3d3d704153f808ce
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.22.input_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.22.input_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cabd1d79727770b73072d67278aa11106461b336
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.22.input_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1d2b56178dc539e3870e3dfe4b81e43abd7437c6543276589151a92095b19069
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.22.input_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.22.input_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0bbf9c2f001ebcabc8b66fe116f1ce9fa4963abd
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.22.input_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:698ab66fbf2bf10d730e16d1e177853e5e2cc0601dca3448148cad25d52e3b71
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.22.input_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.22.input_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.22.input_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.22.mlp.down_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.22.mlp.down_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..56a1864d9a83543644088e4f5fb2f1919fad2cfa
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.22.mlp.down_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e757894bd88032c32442a9ad41cd2bfd80553a1016abe2c295abeb91764e63af
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.22.mlp.down_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.22.mlp.down_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a1fc6a4c015b0c6330c905d4f49a8481d899e38e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.22.mlp.down_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:926471ea144d8481d99f93569c4ac8eccde3c825d1bcafa21368941982a820b5
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.22.mlp.down_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.22.mlp.down_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..72aade2e0fc4e0a10f0fc0e0550261dbfb9806e1
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.22.mlp.down_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0956141d879d883fbce97bb4bc61649aae7afdfccb0c652ffe1c3c6c216c3609
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.22.mlp.down_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.22.mlp.down_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.22.mlp.down_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.22.mlp.gate_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.22.mlp.gate_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f0a701def822e3ab2ad3c167c3266b5111364dd9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.22.mlp.gate_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca5dcf8df58152fb88b3669ebf03b55233dde64dccf82af620ffe2772ae5cbea
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.22.mlp.gate_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.22.mlp.gate_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4a09e2c8a4bac611feeea5fd97bf3f60424f4439
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.22.mlp.gate_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2937053de69a3893c5396093968f6aeb11732b6512d50630c72e2fd3313f0300
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.22.mlp.gate_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.22.mlp.gate_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7bb85d6128e1c6f53a1e73528ee0f0ec585846c2
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.22.mlp.gate_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8a0f7077ee8bd638cf886770e734d3ed6cf2d71f088e2951881a3c041ddca155
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.22.mlp.gate_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.22.mlp.gate_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.22.mlp.gate_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.22.mlp.up_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.22.mlp.up_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0953c2588f750eb57975955c1ac80da3f84962bc
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.22.mlp.up_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:628bd447ff14c78c1016e8c75420dfbcede25326f79b4ebaf143cb4ad0fa3d74
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.22.mlp.up_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.22.mlp.up_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c2cb0fe5bbd6e8e858a032295089dca0c67fc37a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.22.mlp.up_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3b6f39b9037e8301e5290227169013b87521371ae5668cb3f8026757d5a903e8
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.22.mlp.up_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.22.mlp.up_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d6265a7f83b03a61be0ca3c35a5e1bd95e571c54
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.22.mlp.up_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a15eca9bbbe24dc0cfe58067e2839648af940430f21296c5f987b7d00ea457a3
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.22.mlp.up_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.22.mlp.up_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.22.mlp.up_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.22.post_attention_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.22.post_attention_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..068c4e044b990bb72bdace3d3ee8b4befc9cb970
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.22.post_attention_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4eb85aa0e65153cd6f99d268e12c30c0980b7824eab073611d6d8d9bd8a19012
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.22.post_attention_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.22.post_attention_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8bf3e09b91b7b91a10cac104c999e31360b3fa50
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.22.post_attention_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:04364355ca01d2a1f7df9e8534c5ce49cb242083a3b3e301ac7bea4105e83224
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.22.post_attention_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.22.post_attention_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b632c428d7fbd64085c3a928886d99af37cb1265
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.22.post_attention_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5eb011a5a09da51a5a0f3cec77a01768881d76d2795807622219c89c26cba7a7
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.22.post_attention_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.22.post_attention_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.22.post_attention_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.22.post_attention_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.22.post_attention_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..19e18f0d2a74dd7cff4a0a3d4f574c8c54d9bfb4
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.22.post_attention_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e5b500fd96f896b04550e2755abcab3747d21b3fea433921c8137fcc59806dec
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.22.post_attention_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.22.post_attention_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c17eafe5f90452a4059b8bccab5971c002b69be5
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.22.post_attention_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:710b7f229ef377b640d25055ff89d0c8fe69ea46c823d6706f559ef13c391533
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.22.post_attention_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.22.post_attention_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..965ee828425b4affb3b42997d687d17826654854
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.22.post_attention_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a64710833b0ea65566c73824b9c4f437739d1767c03e9e83b0df84b69f5a8bbc
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.22.post_attention_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.22.post_attention_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.22.post_attention_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.22.self_attn.k_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.22.self_attn.k_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e06ede2ff50296dc31d7f56ddb5d6ae643b24a82
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.22.self_attn.k_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca206facb7902e846954edebe3e3a46f2f22f8382177cdaf95f6ec36cff807a5
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.22.self_attn.k_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.22.self_attn.k_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..795a1df57c812daed3092d0e273e7048a5e0a04a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.22.self_attn.k_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9ff95d6654575ceb97e88d9d744e7f956f9975641c4fbc175bb42c61b7a16caf
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.22.self_attn.k_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.22.self_attn.k_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..86ea0e194c5c050df9d031d24764f0d3b7b8c292
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.22.self_attn.k_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:da807484f541608f56bf985ed84339eb1ac60c46850b722c0e00a17dcc6df383
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.22.self_attn.k_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.22.self_attn.k_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.22.self_attn.k_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.22.self_attn.k_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.22.self_attn.k_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f49630a40b165000b13b3bfe02c046bfae36f77a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.22.self_attn.k_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5b145d75ec7ad76c10e2324b03bc0643fe871160403b6604f8ebe7a65440d511
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.22.self_attn.k_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.22.self_attn.k_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..62a2e555e42362d67d080812d4223167bab553ba
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.22.self_attn.k_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9abce611f6fe93dec1e1c59a513cd1efdeab9f5fd2b14c6d60157c9d2d5f6aa3
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.22.self_attn.k_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.22.self_attn.k_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1ae7bc0b6f311d83c8d555feb59ce96763b7d89d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.22.self_attn.k_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2d0f1aebacc5e540763d276e794dfa5d8e77d93b31148086942bb4311614eace
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.22.self_attn.k_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.22.self_attn.k_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.22.self_attn.k_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.22.self_attn.k_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.22.self_attn.k_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..85ba9cc7ef83ed0c5b57531b689269072b856e11
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.22.self_attn.k_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:374b8506cf57e25e0e27230076b62fb83868092a23b7210f5ca52af4a17b76d2
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.22.self_attn.k_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.22.self_attn.k_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..05ce302e2a72557c485a6c595c4f978de53d2788
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.22.self_attn.k_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7139bccec561b24baa7e5e9f44f221fb9058ba2f2bcb8f69567e94a280c187ba
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.22.self_attn.k_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.22.self_attn.k_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..be1b9471bc1dd0e45ccd8f8c036d97771a224f5d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.22.self_attn.k_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9cb188736ff0118ace8eb5b0751b37c7cb5b653394665fd9128703a09617384f
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.22.self_attn.k_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.22.self_attn.k_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.22.self_attn.k_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.22.self_attn.o_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.22.self_attn.o_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0ada10d3c6d3228ab8d96187ec751fa2459c391c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.22.self_attn.o_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:62fe7ba92c605ef1d6f2bffaab00f7fbd32cfb9fbd67d6b5e6679fb12ba28a56
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.22.self_attn.o_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.22.self_attn.o_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f6c40da46e78dada1c0fe0ec6cc0ab49046b9a0e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.22.self_attn.o_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cd3f664a80aed0d93effc0984eba7456d9e1161e4a44104123e3d8b40373a1df
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.22.self_attn.o_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.22.self_attn.o_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d4ba851dd85dfd4f5bd0ac383500c2b641dc50e5
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.22.self_attn.o_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8076c5f7ae2f5d2a83bcde662e4bb199c08570b5e0d87099333ed4d1c8ec8281
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.22.self_attn.o_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.22.self_attn.o_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.22.self_attn.o_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.22.self_attn.o_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.22.self_attn.o_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a3b4ba1677e099f533a0a49da44c39473a814b28
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.22.self_attn.o_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fb53d23440ab9fe997cddcde161d1e323f4f3eedda38043490d613350a3ad9b2
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.22.self_attn.o_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.22.self_attn.o_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b02de7c30d3c67a8ff464392344d03519ac027d7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.22.self_attn.o_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8c7a9410d5fc2c53ecdee0274d526693a10a22e6aa50e6ec209590f84e2182fb
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.22.self_attn.o_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.22.self_attn.o_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7881d366b940a3d7c6c4bc48c7a1e69ee2a5014c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.22.self_attn.o_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:63bde29867ff20a64bfeaba05cb77856c14f1e36c5f03ccd3c758e9da5c3cd06
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.22.self_attn.o_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.22.self_attn.o_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.22.self_attn.o_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.22.self_attn.q_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.22.self_attn.q_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..963b18de109008882d7f39f1e78a99ff6440472a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.22.self_attn.q_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a5e4c07a2183139871ec3ac2200c3a89a758fdcfdeab78de8dd19a1ebc301728
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.22.self_attn.q_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.22.self_attn.q_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..46f118b8bf865ef9720399d48b6cf195994948fc
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.22.self_attn.q_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:604de8c9980c151fcfb4c8edca2160e4a66ec415265bee92f767cc5ffe5294af
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.22.self_attn.q_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.22.self_attn.q_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..864b8fc4484e7ad14dee687be3d0e4e50c732ab7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.22.self_attn.q_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4db9dfff851e99a56ed92ecae8789e8f2b938898951c3f15dc801b0d04ccb3ad
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.22.self_attn.q_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.22.self_attn.q_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.22.self_attn.q_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.22.self_attn.q_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.22.self_attn.q_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7bffcaa3f6bec2cbd1ad014f8d5548cf912578e5
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.22.self_attn.q_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bb0a9c99eb4cd770e0925b90f299741f618156f31b7113bec3e197bf6217b450
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.22.self_attn.q_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.22.self_attn.q_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8eb5fc32b563e9c25f303eae9d091180d6813663
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.22.self_attn.q_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5203727f7b3147cc0d812d3fb60f082209ede2afd1433261ba65029f0c6dc423
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.22.self_attn.q_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.22.self_attn.q_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b3912ba2887127cd7917c89aa82b75ae505481e7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.22.self_attn.q_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:621828c16721b69fb5d681ae371caaf758be0925242ca5d4f192c4d5ad7ef895
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.22.self_attn.q_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.22.self_attn.q_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.22.self_attn.q_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.22.self_attn.q_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.22.self_attn.q_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..df30c84c83b634753c6a913675aff67d9b21cbb9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.22.self_attn.q_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:089ff6e804386e7bd25a46e082f0d7d64ba4e984200db98c361c69b3cc81ee6b
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.22.self_attn.q_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.22.self_attn.q_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1f4f5dcdaf76af01b5c84f42e502c6d94935f719
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.22.self_attn.q_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d5deb7883e416c72028a9aa3061e655af9e7b8289b9c09d692039a80df620e4c
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.22.self_attn.q_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.22.self_attn.q_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7ecadc416dea68be23b411023fd7c2bf620033c8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.22.self_attn.q_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b62587a15f4c60732ba356ca9a1e5b327fa2478f560cd9ffbef68d5a2ccca527
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.22.self_attn.q_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.22.self_attn.q_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.22.self_attn.q_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.22.self_attn.v_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.22.self_attn.v_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..295bc887cf5fc2960c50ec0cb18c32b0af0ff840
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.22.self_attn.v_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3fabe458dd9cd92c05386e3b00b297380f40d16d31511e593398475ec200b42e
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.22.self_attn.v_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.22.self_attn.v_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2b793e73eebcb05d2ecc163a55ddda3e4f09c453
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.22.self_attn.v_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d0f25fe47d88c23fc7cb33d395d3355f6f1563df6c5a7e72c6c1d024d8281bc5
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.22.self_attn.v_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.22.self_attn.v_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8a8329bf7396038b13e4391d833558ac5f56ec3c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.22.self_attn.v_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1f2c00a48ec5826a7bec53399b579e7c9b5c8ae7d1e08bdc6897ba985ad34698
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.22.self_attn.v_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.22.self_attn.v_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.22.self_attn.v_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.22.self_attn.v_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.22.self_attn.v_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e83aa04158086991e8d7dfe985ada785e1e45216
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.22.self_attn.v_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6a4705f4920935191df5da48ba669688fd7f0713e4d185c9cc26f2f8155a3abd
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.22.self_attn.v_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.22.self_attn.v_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7cf05b8c1a509d0e7d005e22de34df8474472fb1
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.22.self_attn.v_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bfced5cd4edfb69cfaa0a44d485872ae884a849b180eb7186affe53c2b604eeb
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.22.self_attn.v_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.22.self_attn.v_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7d05c0b1fff81de61c1783a6a962994bd6272195
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.22.self_attn.v_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bf01df6538b4f9d8598bf3e7e7d6c54f35a68b3e738cd369174e47cf8829e4bf
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.22.self_attn.v_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.22.self_attn.v_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.22.self_attn.v_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.22.self_attn.v_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.22.self_attn.v_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..041c6efb13be42433ef2e35b0b2c5801503398ad
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.22.self_attn.v_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bd914b784a29146e708edc757cd5ae30bad73429eee45598c8b39624c4a1b923
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.22.self_attn.v_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.22.self_attn.v_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..10b59f74c95618f02f7c8ba1246dc4b26ac43212
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.22.self_attn.v_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5d1b4ec5f9fda6971e4b248f31350626cc2a0f7ac39763bf389770e22ef406d7
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.22.self_attn.v_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.22.self_attn.v_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a3b77f0bb971b0d01606c093ca03902b3f0860da
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.22.self_attn.v_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7b80b7fe810a8c2912d013ae7c3e720c385b1371b284aeb4c636634ddd855553
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.22.self_attn.v_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.22.self_attn.v_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.22.self_attn.v_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.23.down_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.23.down_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e29f8d62f045f9f4e43b48c5e99e8130acbdc089
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.23.down_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:928d5279072aa30ff6979e1d41218d935f7d84fc65d34211af51632217873df4
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.23.down_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.23.down_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2f8d2f7760b9382b9da042fc057b366bbfdf0c7a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.23.down_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4985fb71fd39421b305cb41d1ebbb4ed15c9bec9c4030e7ddc4d8522934d8752
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.23.down_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.23.down_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d727f3e5e1499efcdecb71878e3305db073f1dfa
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.23.down_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b105a60ea5fc397030f007b6a9895958c0c398efba5688a7c46aa005fc250aab
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.23.down_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.23.down_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.23.down_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.23.gate_up_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.23.gate_up_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..02680218ae66397832acf694a21d30231e34da29
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.23.gate_up_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8401c83391fa7dce536f5fbc3d7b13fbec510ffed0aba6c0fa050d2431c48a5f
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.23.gate_up_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.23.gate_up_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..14e26ed0c53f16b7d255a2f1d0ade75e25a526ea
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.23.gate_up_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7d265e2b0db55dfe9a85fdb648e738f8c4d75ca004a68422f066baddd6e121a4
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.23.gate_up_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.23.gate_up_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d4fdeb4be880ea7212909fba082e7ac33ccfcac6
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.23.gate_up_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b1b88ec3abae007025178c3a4fde6575d902befc3ede5c63993dfc4c3e64723d
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.23.gate_up_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.23.gate_up_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.23.gate_up_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.23.input_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.23.input_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9cf17f2877af1523e8d2a3ee9812c66a3400849f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.23.input_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:85cd65d86ae22016766d38b382680118ebafb659aa217d980e1605be1fdb16e5
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.23.input_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.23.input_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..85e1ff078c8d801efa05562e128c31a406b14863
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.23.input_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:31fc8204c2f91bfceaf11fb73369d65f8d17d2eddc8300f3f73ac21ade26dd0f
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.23.input_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.23.input_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8f7924af72c608c5d9a243c6eed07c1a11585fd3
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.23.input_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:76e79cdaaeb7588f42c6c9e5d617ebbab0a471bdd72d96eb378e960dff3e9856
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.23.input_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.23.input_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.23.input_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.23.input_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.23.input_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4c43b137f730a1a165456d3568e16e3faef20aba
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.23.input_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4da3cfdf1d3f5f3654abc51dfb37f053c1e0caa1771066aead1978e87479af77
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.23.input_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.23.input_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..30bfacb59d4eff23105bc6a16724264c1c77bb38
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.23.input_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:75a00c8d79498aec8482c8d03207b0648951382ebe2d80d0ed11ee6090b1ec0c
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.23.input_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.23.input_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5e91c1ab5c6194e4b449c00f547531f1ae4d201c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.23.input_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:849f1eb6be6a24bbf99fb3359bed60d44742d25deaa3640d3d71c47817a1c860
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.23.input_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.23.input_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.23.input_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.23.mlp.down_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.23.mlp.down_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..13d16e0a56ba1aab0c8bc019b263763108ba5e6b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.23.mlp.down_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e7802ff47657ef1bb07f72e7b319a84c5139c9fbdad55d02a3eef5c3e75dda8
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.23.mlp.down_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.23.mlp.down_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c0c3b3c0f7cee479ea4633c3817ab1cc714a6288
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.23.mlp.down_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9fd8a2cff5557224b2a9211424febad09a15dcfb609b7b8689126e18a41e63cb
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.23.mlp.down_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.23.mlp.down_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..be352c79a1f6fa7104e281773fdd8a226045b40b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.23.mlp.down_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6fb6ad307d515fafc9d2b59d5c501845ec97739dd3cb6dde87666bf86352025b
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.23.mlp.down_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.23.mlp.down_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.23.mlp.down_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.23.mlp.gate_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.23.mlp.gate_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fb3d680258ac6a0b0adecbb6e19635cceed77c44
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.23.mlp.gate_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5ef8c608cccbfd1e5648b38e490b50666515397ef5bf787c88b2c4538ab63cc6
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.23.mlp.gate_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.23.mlp.gate_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d30007075e6df8af4e89c945db76845b86cb9e61
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.23.mlp.gate_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d013a8a9c06604245ce24c7cc7d118f059a63d7f54eebb841b235cadd49039e9
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.23.mlp.gate_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.23.mlp.gate_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1b916460991b18904c782e5881d07c56e5bb759f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.23.mlp.gate_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f95ac07be5b02ab33ce10cecfaacb24a9ae1fe2b1e8da960a7874a6a7635b771
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.23.mlp.gate_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.23.mlp.gate_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.23.mlp.gate_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.23.mlp.up_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.23.mlp.up_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..633ad4926706da3015fb60c1215c7e01a82a126d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.23.mlp.up_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f5a82a736296dcd62d684558011c37535ea0191484fdf5bce9c5ac5525f718f9
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.23.mlp.up_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.23.mlp.up_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..26b71c27c8ca019c4a742ecfe0c6c50fb69b2c5e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.23.mlp.up_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:03f1bbe5c5cea6d5e32a47f3958b6cc324858b71cbde6983efdb8feb4753f2b1
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.23.mlp.up_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.23.mlp.up_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..337a9cc302095e12ca421ef5d7808784fe55cdd7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.23.mlp.up_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4430a9f4513e68dac62fd0a3e2fca114840d9f9b6f033e2f0c7907e5bfd49e94
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.23.mlp.up_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.23.mlp.up_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.23.mlp.up_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.23.post_attention_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.23.post_attention_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2c9a962b14f46578543707c8b0f9f8fd3ae3ccf6
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.23.post_attention_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1a9d8ab74ac6445df67ef8e0ce4d495feadf9870f9711a8af926cd1675fe98bc
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.23.post_attention_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.23.post_attention_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8ffe9639fb6504167cd161fbe32379eb64db062c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.23.post_attention_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5fd51b863cbbef6ae7ccb31b3ee82bb1943d1a8957027a00354c510a89ad3659
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.23.post_attention_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.23.post_attention_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8d99b0b199e649a2793841b8be46e6fd3b250f4c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.23.post_attention_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c88b5ddc82b017715251467fa213b9e99849ba63b59e6b4851ea5999cd02f5de
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.23.post_attention_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.23.post_attention_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.23.post_attention_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.23.post_attention_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.23.post_attention_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ba9cc67157f2011641473c3674f1dd16517104e8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.23.post_attention_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4392528195d003c7488d399d5f89392dc71c10ac5cb053e85d22e675dd682f32
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.23.post_attention_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.23.post_attention_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e32eae5002d08a24e8d0dec264df05af183859c6
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.23.post_attention_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:65ab71a8e8900fc175c0b2897c5ac7330ff8b4bc051ce8e31145b50766e4f4da
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.23.post_attention_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.23.post_attention_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a542eb8bfb6e159d23c9334e127880872ba9d421
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.23.post_attention_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a102a5722a6bdbf8bafa23e3f2dba56306a1f3c05affa49428cdc78b94da8ca6
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.23.post_attention_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.23.post_attention_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.23.post_attention_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.23.self_attn.k_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.23.self_attn.k_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a71bdd3af1bea705adb250a1fcac12c7b9e9190c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.23.self_attn.k_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aee5c70892ab2b99a38d66080467e7cf9e8149339cf14d60c23dd496f6aa1591
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.23.self_attn.k_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.23.self_attn.k_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..775077c7aea8d729b013876332b36d3fe88fe45a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.23.self_attn.k_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aeaf77779906ef333bc395224c0bc9db39dd14d21af163048a651333078e6f4e
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.23.self_attn.k_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.23.self_attn.k_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cacdf2671b8e4dc93a42e9415ae562485e0fe050
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.23.self_attn.k_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ed480e91b2abd6faf8ec87c86caa14267511d34c11ecf7e5f6fc2c5b64238751
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.23.self_attn.k_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.23.self_attn.k_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.23.self_attn.k_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.23.self_attn.k_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.23.self_attn.k_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6ecd6a3170c4cadb461937e823a58387d148468e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.23.self_attn.k_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc83a0356b015c9ea3020862e52c6e09a509462837d14fa3820e34936ddb82b5
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.23.self_attn.k_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.23.self_attn.k_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a11dffd4b58a6d6d50766add52cec516f06d72a8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.23.self_attn.k_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c877ccf916648628920d4f45ac3476744a14a87ef52b5f8dd7b7ddc987fe3b93
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.23.self_attn.k_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.23.self_attn.k_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3448e8bba2be14454bdd92cb0b329778676386b6
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.23.self_attn.k_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:792c57237cd5295294e225ef7f9e13730e0797b615bc3f11f9626f520e68b24a
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.23.self_attn.k_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.23.self_attn.k_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.23.self_attn.k_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.23.self_attn.k_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.23.self_attn.k_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ae5d26e34c60a0d1ab893261b072ec857b0d3aed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.23.self_attn.k_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:66dc81f8800ee20b34eb9c3f11d5168511f304ac73918b447971ca7b5df2ff24
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.23.self_attn.k_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.23.self_attn.k_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..85d2ba12f62920872f7970cbc26dab3f98122baf
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.23.self_attn.k_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2cc23e55c75048e30e8d9399079cb70148a77af5e60dac2ec4116a6d883e1d20
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.23.self_attn.k_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.23.self_attn.k_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..14a9309b9d40d07dd6f3792184c79a13abf5d872
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.23.self_attn.k_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2304adfc5f684dca1cd536731aedf5da6535668249d75e7c797e23541c6d1905
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.23.self_attn.k_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.23.self_attn.k_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.23.self_attn.k_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.23.self_attn.o_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.23.self_attn.o_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f48d3a50d1eada140d2d9b62d2b16fafb6211d02
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.23.self_attn.o_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:505aecafe0ba9f53c4f8fd0518eb9816c7a868dc31e0ce3803fcdd277ece05b2
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.23.self_attn.o_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.23.self_attn.o_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..186abfcf2220808cc95753cde0934d0e8cb3cd10
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.23.self_attn.o_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:81b5b3e148205d37cd66c23f14ffd66c53f0dfbcc35f71a4443cf80bf9b26082
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.23.self_attn.o_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.23.self_attn.o_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fe4aeb580c5f920b57a9d7d0b997c55feb248214
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.23.self_attn.o_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:76a0cbfa7be90edea2c22a3cfca6af433d490717b005189d557e398821fd439b
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.23.self_attn.o_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.23.self_attn.o_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.23.self_attn.o_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.23.self_attn.o_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.23.self_attn.o_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4aa912d0e5be1ee7a8d8e2ae0730269af431e545
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.23.self_attn.o_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7d3f631ac40e8eb0152efd9810021f040ca5bacab010e5117087e89412b95160
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.23.self_attn.o_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.23.self_attn.o_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..262367c39b0f45cee9289f187eeb0b6fc6ffd939
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.23.self_attn.o_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0df2e9e943625d5617826381b163ddf2afc68f616b52bd004a638b37e0c20641
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.23.self_attn.o_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.23.self_attn.o_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..35312764cf0456b262b46c602f8a79637bfe3030
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.23.self_attn.o_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a8988a02a188dd730e6692c7e6532a132acac6b57e01ce452a06dbf349c6aa5d
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.23.self_attn.o_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.23.self_attn.o_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.23.self_attn.o_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.23.self_attn.q_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.23.self_attn.q_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..51703eac33ae1f70dce4a955e5845551ee002a02
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.23.self_attn.q_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a4ed1b0fed2ba8adffa09681921b60ef7752a649b1aa96d6757a7b7a94960fce
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.23.self_attn.q_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.23.self_attn.q_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f88548779c589525ad269210f1d6129cfa34f771
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.23.self_attn.q_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1f707dea3086c045445156441eb5784b2c84c0ab76fe8b4f78d40669bf9592f4
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.23.self_attn.q_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.23.self_attn.q_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8b51b15ce5c1fd193671bdbe14fcc5217090c7f4
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.23.self_attn.q_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ab18ba08d4975230d7c61a750f47b89a348fdd1e3dff8a2e54a1d2f710a50af5
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.23.self_attn.q_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.23.self_attn.q_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.23.self_attn.q_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.23.self_attn.q_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.23.self_attn.q_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f3338e503d1d0abe29d6c79375dd81410a49384a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.23.self_attn.q_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:354e45ddca83d08f1ea317e42090e55b1028d4194a2a4ea1536d9520ecf97a97
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.23.self_attn.q_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.23.self_attn.q_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7e847fa41c737d15e90dd976ebf02a82b4d56f48
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.23.self_attn.q_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1fd51da05daeafc23efb571666e262b06daea7590a5a89a250643fd950f00679
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.23.self_attn.q_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.23.self_attn.q_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d2abc1c9a9096afafa30ce7ec1591d194181a713
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.23.self_attn.q_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:50e6e473a846a56868205f07f418baa74c8247a180ec0eb62f0581f093378756
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.23.self_attn.q_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.23.self_attn.q_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.23.self_attn.q_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.23.self_attn.q_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.23.self_attn.q_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8c64ab8b7a50d101cd0a005732b271c6ef6be1ee
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.23.self_attn.q_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7fd068a97b96396a66d3604268aabd7b0cd76fb8d550f660324687a920020675
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.23.self_attn.q_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.23.self_attn.q_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5901e10a4a196648489ce184aacb2651c61f99b5
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.23.self_attn.q_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe939a16ee7a88a2f41b50e6de8d3ea14b27f94795b8ca2b1c9a473c177ce0b3
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.23.self_attn.q_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.23.self_attn.q_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7739ccf9a2836f8d052a40c77e1b4f2d366f2eef
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.23.self_attn.q_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc9b98398d69d935a11158d047c0ff944848514ccdf6407e1be366c5f4128059
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.23.self_attn.q_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.23.self_attn.q_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.23.self_attn.q_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.23.self_attn.v_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.23.self_attn.v_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6d90c1c42bc9e6e0c7ea7d5dada449e182bd53ff
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.23.self_attn.v_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3410d457677c609cea3de5ccb0f5e80aea8cca817eb096f19f374f1e84d3e4a4
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.23.self_attn.v_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.23.self_attn.v_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5798c1faa54bc2d5be1b25d6c229a8581c350523
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.23.self_attn.v_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5874d0b28c20591225cf939cfac10ba2733cc73b8433122336c8181202cc1558
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.23.self_attn.v_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.23.self_attn.v_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..141ac2fcdac65c87395f67a4e50dd4c5dbe9dd90
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.23.self_attn.v_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:936d8807449c3c3bbba9ac6c058981f72c11f3c10137338bbff7b608d3f62da4
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.23.self_attn.v_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.23.self_attn.v_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.23.self_attn.v_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.23.self_attn.v_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.23.self_attn.v_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..dc6a6edd71a4369769a19912b7c173c007c3a567
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.23.self_attn.v_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5e99a1b76a666b868f3375de90c3c66e092efe977061fa497c0e517886557cb9
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.23.self_attn.v_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.23.self_attn.v_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a0eb566ddaf4f37e637e783ad67fb3a7e959000e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.23.self_attn.v_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:518c84e38fbf12d048fe37344f75c9727feb9a4134b056193a1aa522adac3edd
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.23.self_attn.v_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.23.self_attn.v_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..611b3550cccf041ddc65ba38514989635fe40436
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.23.self_attn.v_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f0a085e928d459de83850dd6f9dadac24bb629ad9223e28ac60050dbc8185ede
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.23.self_attn.v_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.23.self_attn.v_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.23.self_attn.v_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.23.self_attn.v_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.23.self_attn.v_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cd687a70d2e1248136f1fcd1845040ecc140e623
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.23.self_attn.v_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ef9824aec7b279670c068ac880cced8ced487407de59f4c6ac57404a23e3f05
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.23.self_attn.v_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.23.self_attn.v_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9b25208c2278c6bd9f2b6aab9645ec78eb3ed722
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.23.self_attn.v_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a2c79d5f0d3cfaec50576e0638c58ed754dff4b7109b24a0b8c6426ac1e6d29f
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.23.self_attn.v_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.23.self_attn.v_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3cf7ba9438a21ba9578d56903b3d422be64287a1
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.23.self_attn.v_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5b0f3f1e37f3d14cc03187e5e33d42a39f60c6d62a7759e3d1545583528c9d9e
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.23.self_attn.v_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.23.self_attn.v_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.23.self_attn.v_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.24.down_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.24.down_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..29e085d98b57a8cce33a3b07e0b4ba60c71d2ba1
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.24.down_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:573ed4f31ee4535be76e1d6d4998accf2ba7c8ec1cb6f78628fd7f1ddf583206
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.24.down_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.24.down_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..84f667be86d9b28161ba37b2170348de19c5c207
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.24.down_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e49245406a0d249da0f8e06b30f9c495ce90e514823804bcab68bf821e9128ce
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.24.down_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.24.down_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f9e2e76fbc76ffa03e2f56a9ce0160351bbce967
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.24.down_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:44d425ee60b7d6792e1a167a1707c865fcc887f10390f4f14b8cd4ed1b0759cd
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.24.down_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.24.down_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.24.down_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.24.gate_up_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.24.gate_up_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cd07566e8df06d1e13d107d5023e9156c9c712f1
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.24.gate_up_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:415ccb0ebdb15f9e1d67bd56dbe1cb6746a639b800ac05ec70627fa294eb6be9
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.24.gate_up_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.24.gate_up_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b94f8fdf1949ce58e55161ed1db57d6f33a084fb
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.24.gate_up_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:56017dd6d2839ed2e6ee52147035b491d8e47a110962353b06a40ee6db24fb63
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.24.gate_up_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.24.gate_up_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d8969b027811da17d16de061a93d957f730f326b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.24.gate_up_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f34ce9fe11e06afa59e1db4d80d751d44fd4a18daca16c35b769503075c365ff
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.24.gate_up_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.24.gate_up_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.24.gate_up_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.24.input_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.24.input_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..42ad828ae1f28b20969532e87f978901052ad042
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.24.input_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5377de34d75f75c8d54f85a4a18071dc107c7b4ef1e1798440cf89947c601dbd
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.24.input_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.24.input_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..15caf5f531ac06bb6246778cd44c43c4f17fbe92
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.24.input_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:77049dbc79e6ec9a3fe60c47fa8f34dcb9ea332fd6fc22da279750c5cc38c723
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.24.input_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.24.input_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..804653cab9f5ec9849623a075768e0709ad3c295
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.24.input_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f027435c0d6b9e34f17ef33fb252ca866671f7681927a5edb15b7b229b89f602
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.24.input_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.24.input_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.24.input_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.24.input_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.24.input_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2f0f212d6c8b66be9b2862935b0b3cba3520d8f1
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.24.input_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e948f6cd26dba55177bd274c6ae09dab64a8011e3ecbb89eab4a9379fbd070f3
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.24.input_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.24.input_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5367ceb6f79eae1f75677a34fdac0246ff8cdad3
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.24.input_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f9a85908fecd2afba71f0aceab51bbe3613d9ef79ac9449d11ac67292f1930f7
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.24.input_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.24.input_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..558c8f65eb80996d2252ee3f6db98e80ddd5c1b2
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.24.input_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0718de2f4472eba3d9cdd63278efed3df5ed1f6f7d63084d997fa0f794ec6fa8
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.24.input_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.24.input_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.24.input_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.24.mlp.down_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.24.mlp.down_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7dcc4bb92ad9ce8fc5044d0f48e66088db3d6940
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.24.mlp.down_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e462fc8c06abbd499e772cdceb6e7bd6e1953f9f1534ee1d666c8dcb57d3a99c
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.24.mlp.down_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.24.mlp.down_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d8ab5648518c2149aebcb482f412feaa08daca6e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.24.mlp.down_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:05836d6ee45b862ac0c48903b34005d13ff6a25114adb747b8fd9ea1d5069995
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.24.mlp.down_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.24.mlp.down_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3da801eabb1986ba3462cad3c28a48658be98b34
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.24.mlp.down_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:16e650ece19e9bbd3b5f104106ac5c3db865397effba596487d4d51ddeafd20c
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.24.mlp.down_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.24.mlp.down_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.24.mlp.down_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.24.mlp.gate_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.24.mlp.gate_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..76fb359b26a970d43924eaded42716ef65cfdf2b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.24.mlp.gate_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f5aa2cb64bf44c83d39953e7f8ed2ff4ae55ae64dc9598f4a43c130bd5c0c4e1
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.24.mlp.gate_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.24.mlp.gate_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a274a9e624668dc8ca6fe64b36ae279e8a902cd3
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.24.mlp.gate_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b33727aa0bd69e4eb8bda3d12e08aa641342a50065be7d1255302b01beb24ecf
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.24.mlp.gate_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.24.mlp.gate_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..68b8cc4624f468b08937bd0c024dd4454fed616b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.24.mlp.gate_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0be22a4564375f0ea1b2422e8246ea6fcc5ae096e8ff1b3ca661cf1a19324d14
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.24.mlp.gate_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.24.mlp.gate_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.24.mlp.gate_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.24.mlp.up_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.24.mlp.up_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..86255bc247a09c9ae31852a1ec4fa23d1a29b5ce
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.24.mlp.up_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a952a5e2515147f02d45bae554ca892b1dba1fbe8d03836b2392b9f014055b4d
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.24.mlp.up_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.24.mlp.up_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6cc9f1d7433a9bec6089c66da926f95b03e22bd6
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.24.mlp.up_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:776b95f1f3ecd5f467158f694c3aad13af8efad135196386c04f78d7ae12be08
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.24.mlp.up_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.24.mlp.up_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..834c02bb4f36e86cb7c2d188897c15565715e6d1
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.24.mlp.up_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f3ac57a0217d5c00315c22b830d943553310a00725379e19c744b4952f4b34bd
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.24.mlp.up_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.24.mlp.up_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.24.mlp.up_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.24.post_attention_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.24.post_attention_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e23105d066a38a0169174c19a265db7a2db5ea54
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.24.post_attention_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c65a7a369e4527ecf4471154de1e807746253c99aeff74a75f03fa727db1596d
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.24.post_attention_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.24.post_attention_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1167c714edc8abcefa867b64b224ca73f2f48636
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.24.post_attention_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e8461e99ffee1ec201b56a9589a9256931bf0578888368a083307c17f9fb8f09
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.24.post_attention_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.24.post_attention_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1fb1b55e1ccc264e84e026626e3a5ca9e0c59d52
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.24.post_attention_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:df423c54d2a09ad6eed2ec61ed705c08e3a3cec6673aba17512778ac132b40cb
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.24.post_attention_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.24.post_attention_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.24.post_attention_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.24.post_attention_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.24.post_attention_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6540d9f5874bde20d7aaff66d519dd5ee941ccac
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.24.post_attention_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:da0917b8605b468dfabda3973f4ad1ad980ede0b99dfa281464bebc1c65f87eb
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.24.post_attention_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.24.post_attention_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..110b6eb9725f14b655cfa987fb1048b19d39af3b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.24.post_attention_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8b93f20ab690f8c012653db558b175a044777b97b51ca79d5b454a99124a4e4a
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.24.post_attention_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.24.post_attention_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..67922d4a7519ea3594c08695b3fdae99ec9ba2ae
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.24.post_attention_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:de4698fd250200e463c0435cd21b5e8d1fb07e4179b541e1e831a03f7b24cd47
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.24.post_attention_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.24.post_attention_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.24.post_attention_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.24.self_attn.k_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.24.self_attn.k_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d4ee4c5c27499204d2669cf7c314cd3b1563d454
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.24.self_attn.k_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f7a70ce88443d90c2f97a9f2c37078c78cf5a4133ce54f3aec529979bef6974f
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.24.self_attn.k_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.24.self_attn.k_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e64df0815d44c8ac4df906fc57d5544de07e07f2
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.24.self_attn.k_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:59acba25ebcb434f9eddd5e4c71c056b6f8ede0d21254bcf515c537eeca34bf7
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.24.self_attn.k_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.24.self_attn.k_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2e93e5ed7e91019bf442d71777fd57dca6cdda40
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.24.self_attn.k_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ee664b51e04f534302a0aa6e851f5b9b1ca9ca48009d5c82267c084ec3544581
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.24.self_attn.k_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.24.self_attn.k_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.24.self_attn.k_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.24.self_attn.k_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.24.self_attn.k_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..90fdfbad4974f45750c87cde39f97958ce5f8bcf
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.24.self_attn.k_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f04c111c20abf7b88b33519cd6ad42f1808e1e13b921ef1aaf481ff97a93fcc2
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.24.self_attn.k_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.24.self_attn.k_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..146674993740667b131f8a1547651bdfd43aeb0d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.24.self_attn.k_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b92a2aab02a0b8824657599c48737b4117411690da4c3c8309f92e0ad1315a72
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.24.self_attn.k_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.24.self_attn.k_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7a373012da99d8eabb3d3073f0200cb7be6a45ba
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.24.self_attn.k_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e97e1fb8cc0367223d61cadc8edd81bd3948bed73fd190cfbfc17eeb60957e88
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.24.self_attn.k_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.24.self_attn.k_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.24.self_attn.k_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.24.self_attn.k_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.24.self_attn.k_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ede171c228c73d7da62fbb99517d86a776fb9e49
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.24.self_attn.k_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:383e3cb86c14ef48c146bac734ee1bcb4052b4265d98661251f69048cdf0dfdc
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.24.self_attn.k_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.24.self_attn.k_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4feb40349aa442d5eefa5d314b5abf735f9e7d5d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.24.self_attn.k_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:17638caba9ba0a6d1dba3543bba761854919a9a97f756225ef57af723a8c7c76
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.24.self_attn.k_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.24.self_attn.k_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..eb9129d8ba15e5381d0110be7562e20d9876e8c6
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.24.self_attn.k_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5b2505e72565464865a2f13ea6d557046ee7685ec55c9ad1e205341ee0216d08
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.24.self_attn.k_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.24.self_attn.k_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.24.self_attn.k_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.24.self_attn.o_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.24.self_attn.o_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c67ce6d84baabea37cdd6e6fd1b49c69817bfa6c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.24.self_attn.o_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b2959a47329ad12beab9b167d4380980e56cfe95277b29e7e1e2cd1d3e169751
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.24.self_attn.o_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.24.self_attn.o_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..68fb916ee6847d4fd2f0a673d6123ea7e41772a2
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.24.self_attn.o_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3c1bef42c0652ca786389c15bc61ac9a6507e0178d66c984b761e3419fc3fef7
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.24.self_attn.o_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.24.self_attn.o_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7a24a165086f2210eb6d3b01796405f699a53c9c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.24.self_attn.o_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:566d05c9b6bafb7356e849ba21f66cf8972f40d7952b771addfb7fe6030498d2
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.24.self_attn.o_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.24.self_attn.o_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.24.self_attn.o_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.24.self_attn.o_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.24.self_attn.o_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..78b2b7adfff09b6482c35232f3ba80c74f3a3c27
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.24.self_attn.o_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29a1e0d8492f2bbc3bbd8f6e3afcde072443ba513d88ef2b28b1a6382b2487e4
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.24.self_attn.o_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.24.self_attn.o_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..da7d5d4c6ce133639c1cc2d0b47d576f26baeacd
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.24.self_attn.o_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:862685ff1e1721b37e0cd1aacd707876b076bdc7de9937f5051b6d11ba65f875
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.24.self_attn.o_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.24.self_attn.o_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2db2cf536334322acc88afe2233884c27de6df76
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.24.self_attn.o_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:db528ea1b24b6486228e6a868fc06122f965be87b5bbf35bf4a17fd20f8a943d
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.24.self_attn.o_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.24.self_attn.o_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.24.self_attn.o_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.24.self_attn.q_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.24.self_attn.q_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..003a21173d52acbcad1e6b7f9def221eead19885
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.24.self_attn.q_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d6e098da0e9bf0f6f310d4ef48e0463b5b58183939bc8165c87193175a260040
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.24.self_attn.q_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.24.self_attn.q_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..92291e64b77d2e680917147130b028069899e41f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.24.self_attn.q_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:46e292b3281f7c6ec510843817f28f07fbf525b66ed93f67c332172ec0d3526f
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.24.self_attn.q_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.24.self_attn.q_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8aa00b93d465faeb461f74e83e0211f22ba7c296
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.24.self_attn.q_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:139fb6c77c119c1492926c8120bc94d77d0d2498fbb868ed008a5a48023d7b2d
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.24.self_attn.q_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.24.self_attn.q_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.24.self_attn.q_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.24.self_attn.q_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.24.self_attn.q_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..72159848e4c15879e06b8ff264e2b39576f72396
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.24.self_attn.q_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f88cb07f3577e53ade6cc5a38317e132124d96fd384a17f9119401b2425c7e57
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.24.self_attn.q_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.24.self_attn.q_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c5933292975ed3137cd03a5232002d5866a5a6d7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.24.self_attn.q_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5e62fd8e2517417d0895b69b5258bbed61a16f58084c314ee14a48539ab59212
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.24.self_attn.q_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.24.self_attn.q_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..79dc278294e344816f4a26891bfde19387f2d40b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.24.self_attn.q_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:31e97996a64d0b1e2e2c3ab8616022e6962db5bb9d4c8e5e46aadb75c281cadc
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.24.self_attn.q_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.24.self_attn.q_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.24.self_attn.q_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.24.self_attn.q_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.24.self_attn.q_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a00e976cc7230a7f627c055afddf87911bca05f4
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.24.self_attn.q_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:41b01f30fadc510ebc8fef661fc2334d9ef08196ecd6a41391aaf33ed1562ed3
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.24.self_attn.q_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.24.self_attn.q_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..524d12a85378312b189d59abf41db12f9ad28634
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.24.self_attn.q_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3625eac9ea4e46e71e2b1c6f8c0b570e567894a2f28600f6a9032cffec449014
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.24.self_attn.q_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.24.self_attn.q_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e82bf0ad88ad8a04f66d447bf5376454db989710
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.24.self_attn.q_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d1bc5fc492ede4ed8f8800559826e8c800a83e25e7dd6bef44d6ec6ef62bdb44
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.24.self_attn.q_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.24.self_attn.q_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.24.self_attn.q_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.24.self_attn.v_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.24.self_attn.v_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5640bf9572f25adddca6be3fe2e5878aad018c0e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.24.self_attn.v_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8672537ba6f9d2704acd7f7793bb0881ebc379a49e53fad6b9f29a226f206e18
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.24.self_attn.v_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.24.self_attn.v_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..563fa4153d9f9d8efecfaed223654b0e9dbee673
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.24.self_attn.v_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:523f96ca8c7744ab83c437a8a305c17614264b8db1fc6df42034032596abd79c
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.24.self_attn.v_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.24.self_attn.v_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3a34d324c8a1ca1251837e81d2dc80cb43751a95
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.24.self_attn.v_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7c52d0a50b822c207b84392e072c82082a41987497ce50454df96a274b646859
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.24.self_attn.v_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.24.self_attn.v_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.24.self_attn.v_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.24.self_attn.v_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.24.self_attn.v_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c0944b066bfc86c2467eeae7187337ef300507e4
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.24.self_attn.v_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:41b11fa0a55deb29725e3c424415bddeafd04e20dd04bc0febbc772cc2dd9d23
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.24.self_attn.v_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.24.self_attn.v_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..bf7f9cada9e856a6ee84b05781cb94e185f1fc29
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.24.self_attn.v_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:69dd1efc0996fd407e368924346bd58ae00fe966db91af7b37468fc38c698db9
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.24.self_attn.v_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.24.self_attn.v_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a48c562106155db13f235a39317917f256d65b92
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.24.self_attn.v_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4d618ea64a62b976461627d49284a45bfe11b5edac4d85fc6240633946891614
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.24.self_attn.v_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.24.self_attn.v_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.24.self_attn.v_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.24.self_attn.v_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.24.self_attn.v_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..636c992872416311a733df7c088ed9b88b014715
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.24.self_attn.v_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a8deb88d559ac9b469b7187ab16c6d08708ca59618ecfc6749d92481b7ec939e
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.24.self_attn.v_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.24.self_attn.v_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c65d135421b2d87c0689408b92bb59968c87ece7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.24.self_attn.v_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8195b060484cca33d27b581aebc0e322f21223e59d9ddb4d6e6af8791179f89c
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.24.self_attn.v_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.24.self_attn.v_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3c70b5f8d583626c23f6b0db53d2aa4cca581c0f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.24.self_attn.v_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e22610168b81efff2f4880d503943ededd490b0cbd1f6ec8cb7041a97e50287b
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.24.self_attn.v_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.24.self_attn.v_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.24.self_attn.v_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.25.down_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.25.down_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..838df4f766cbc895e7d10f2889e5277c710fca12
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.25.down_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bbd3c81a3d16c428357afbdebf88e68a3098a98315d84797fcafae831e5de976
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.25.down_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.25.down_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9781f7f0624a1bd3aa1a78e1b604ac0f33a57fef
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.25.down_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be94b8934e43f8f2b9ceaa03c2ab218d0ae47a8051502d30d9b8b1a887c3fa72
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.25.down_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.25.down_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..953de692c9f319e4826c0e9c86b92e4d52ec78ff
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.25.down_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:68bf05d6d2a16799120e58074dc459127f3b0a9077b1d2e4fda50269630d3adc
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.25.down_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.25.down_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.25.down_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.25.gate_up_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.25.gate_up_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..46596ede2cd3a781b90a18cefced4aa401a8c579
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.25.gate_up_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:091b7895254b6625a328d42db6697f4bc43e53f223d54809231e296aec0831df
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.25.gate_up_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.25.gate_up_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8fdeabe79768b321bfe91bdc0c08869bf19c8def
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.25.gate_up_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5e7ce878c10504787a3461a77b87c5d58d0406b825d302c3c9d1e654390b7b3c
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.25.gate_up_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.25.gate_up_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ba9204096fdaf6e6154b95f6a3a77dd53f3f15aa
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.25.gate_up_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f2c2085249725ebb7acb1434b1cf007bc2fea11de73839cb4835d9b11c0c883c
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.25.gate_up_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.25.gate_up_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.25.gate_up_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.25.input_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.25.input_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b6f18ed779309940826b2434adee8cbdfeb2fbe7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.25.input_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9ff9259d4e0b13611bb3143e7cd403d3c3bb90f9822c296b5c8bbe3d765a8570
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.25.input_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.25.input_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..00e2cc87abf3e8c63515dc45ee8e68be2e98b94b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.25.input_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ab5f57339139f205381cbbf4557e435d343800e455be4ac46a242667b543d2f4
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.25.input_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.25.input_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8a0f9e87231a6fb8535db2b6b6ae59a4b747f703
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.25.input_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:469b56cdb0e7022cad737ffb4b98f3b3101038c7d06398470f6f58b0d0b54946
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.25.input_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.25.input_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.25.input_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.25.input_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.25.input_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..47d8a53a86bca09c96fdd79e0329ee79a53b6555
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.25.input_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2594f8dd2fd4f6a9137a654a08cf3394a853a22bcdf71a9bb60fa6edb57f7df0
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.25.input_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.25.input_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..af3d4096c97ae0ae55cdaa1d997034c4e1689d3e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.25.input_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd07bf0083e44d9f17fe90fffc5b9c778a48004769bb25deda890bcc63140bbd
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.25.input_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.25.input_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ae6e9e0f756878c764930748525dca412594d4bd
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.25.input_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a90053fca150448affa157916c0d29661f9c857d4dcb712f62c8197cada0af65
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.25.input_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.25.input_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.25.input_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.25.mlp.down_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.25.mlp.down_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ba3db977188493d09c63a09795f8f779473ebfd0
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.25.mlp.down_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b35afec00713b2c95cce8e18f1871dc146d08fa8bcb5765e954ef1674e5a1ad3
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.25.mlp.down_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.25.mlp.down_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..858910e3678545203daa0e31d209fb40fc45112d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.25.mlp.down_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e5069d012e642762b9b1819ccf89664c907d17b408486290b5b85e2abe920394
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.25.mlp.down_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.25.mlp.down_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..75ff5bd72f8b6ff1fd56ce9bb95935cd6302faa2
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.25.mlp.down_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cde2d08199b5a42185af24b5c89a3b99ab1cb0aa266432ec5dbcc0bf34c8ae9b
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.25.mlp.down_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.25.mlp.down_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.25.mlp.down_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.25.mlp.gate_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.25.mlp.gate_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..97f0bedc781e227df958f5358bbf3b4c2ee4d46e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.25.mlp.gate_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a14f054c157c901f94fb84af57d575d175e5c30483e9875eab22290e719138b7
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.25.mlp.gate_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.25.mlp.gate_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..773bad086f12f01624d7a1539e1e1b0a8f23a019
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.25.mlp.gate_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:398e9bd7d9d1a9605ecf0eded603c2583abedbc796c38607a54acbe663be1d2b
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.25.mlp.gate_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.25.mlp.gate_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..59dd4ae62cfd44504a6c5b93b541cf8e0f7071f2
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.25.mlp.gate_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:afa591774ac1a073432a034c96bd9792861b8f3e8eddee1d9f45633c65cf3eb8
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.25.mlp.gate_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.25.mlp.gate_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.25.mlp.gate_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.25.mlp.up_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.25.mlp.up_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..56bc46902d36562d0d191871a7fdb551a5ce1b21
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.25.mlp.up_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5d07e5b779fc2dbd26d54c2499a1ad17fdb2097c9712f6d345efa80c05d4e899
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.25.mlp.up_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.25.mlp.up_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..68a32f5f726b4f86f7600f058abe0a1e6b2a5a8a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.25.mlp.up_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:805f195757413a26e84ada71c3c02ee641fcb46943389dbedc37c44dae2abaf5
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.25.mlp.up_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.25.mlp.up_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..97872b8d95c11738f9fd24fa82e7bb745450e853
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.25.mlp.up_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:de0cf1fda462c935640069cfb2cf51746fbd70f31e90d8786569a625ea661ed0
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.25.mlp.up_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.25.mlp.up_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.25.mlp.up_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.25.post_attention_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.25.post_attention_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3280653af7ae5dc9517135e528bd6705233327ce
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.25.post_attention_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d2864fa6d56e5fcbc8d168ea552385758bd80575c40efe961dd36a657ce9a983
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.25.post_attention_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.25.post_attention_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..893fe2737a7daa41831b0bb965501d0658b1c8c6
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.25.post_attention_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1755823885ad4f9e23bfe037cb6a4dd9d13122471936521c5234f709e49004b1
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.25.post_attention_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.25.post_attention_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e8a1668e8fa5e7cf8938ef06aa7a7b378e3f367c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.25.post_attention_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c232d11a9a21115962e65eec5a14b3354e6434cbd8ee8099f08cca3de2b898ce
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.25.post_attention_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.25.post_attention_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.25.post_attention_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.25.post_attention_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.25.post_attention_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a32f216666ed445c2e47d2f2e76ae8c28c2c8509
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.25.post_attention_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:35f1cf4897910936f7411e85f0294239e185b765fb40c9b0d7447c8ea611883a
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.25.post_attention_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.25.post_attention_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fb2c2701f47f6a1a321614b9502be5e102d2136f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.25.post_attention_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c7e304d8fa94b74f25e7a8e236ff04b9bfd28e7051b0192355363824f0f89893
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.25.post_attention_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.25.post_attention_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..222c0b04a412b705b922b1045f9d9767b68908e6
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.25.post_attention_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c5f382747c71eeecff6b13843755ac801238f54324a7bb326acfc0fdec6c7d18
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.25.post_attention_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.25.post_attention_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.25.post_attention_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.25.self_attn.k_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.25.self_attn.k_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2a1041f7b54f908467d18bf8a2a1cd845bd7c26f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.25.self_attn.k_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0d6670aa6895e270b2dbded9cf08cc95ef4c287b9c9f0db3ad4e594be289b015
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.25.self_attn.k_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.25.self_attn.k_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..db364be47c17bb54b7d1452c83d391f771ecb36b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.25.self_attn.k_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:430be54cfb14fae950bf92ec388998098859b9f8744e768fe051387c9eb65cbf
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.25.self_attn.k_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.25.self_attn.k_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4f7e4ec8310d249061fc0d39833bd2ff49ae81b8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.25.self_attn.k_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:67362b29d1a967d18c9cfac268b6972b10f178c3dd321de7314a9111427ac0ac
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.25.self_attn.k_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.25.self_attn.k_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.25.self_attn.k_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.25.self_attn.k_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.25.self_attn.k_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c790111c9cc7693192604326c7278b1ae789614f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.25.self_attn.k_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e6f34274b4f3e9b645979aff01501309dbed4d483c89cca36bb4cb21bfd0db84
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.25.self_attn.k_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.25.self_attn.k_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ccdee27604346bd9f428f284f9154e2f0c93c5b4
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.25.self_attn.k_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a8b8f402d23d20436ac44c18ae50030f55d9394b9e42b579baaf3a4165fa43d4
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.25.self_attn.k_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.25.self_attn.k_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..54112ee98b4534bc16401c67458604060a2feb94
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.25.self_attn.k_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0fb6533dc92b6043aead0f503127a215cec0735b0761b2f3998262a063c22e99
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.25.self_attn.k_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.25.self_attn.k_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.25.self_attn.k_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.25.self_attn.k_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.25.self_attn.k_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..77bf628ef94c85b1030a51b9bac0e126c87cc1fb
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.25.self_attn.k_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0978a4cf8908f5c6e81c0d0a094c68160aaa6b231f901d1ac554af424a740ce3
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.25.self_attn.k_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.25.self_attn.k_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1fca7f9cf4dab9ea2e30e2fb370c4bfe99414375
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.25.self_attn.k_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:32b0136f79e063322d4a65e7622051a1d0511337fecc9dad92f95bed782774a2
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.25.self_attn.k_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.25.self_attn.k_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1a19bf7b26a412e93029391ddd15c758c2503cf3
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.25.self_attn.k_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:40d4adb36b4c2ff10fc6d94f3a67e67f83512a146e6daf9298d9f1e61dbb19d5
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.25.self_attn.k_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.25.self_attn.k_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.25.self_attn.k_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.25.self_attn.o_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.25.self_attn.o_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8714957b9d6c76673bc12d3574f305237fa8ec7a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.25.self_attn.o_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7a4186f32debc6da6e958eaec655e7009cb473d40c779dc7618dbf48461d8ea4
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.25.self_attn.o_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.25.self_attn.o_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fe4e44f63802debf915eb03cc32aa1e3e75bbc7d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.25.self_attn.o_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2907eab57797af733ba6bc4904a74bcffb3b43ab180e3f9a8fe5486cf138ba36
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.25.self_attn.o_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.25.self_attn.o_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..96f7422f6ef2b395b2cad56a98c54ec25f5da188
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.25.self_attn.o_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:158788a7d74f38db1d4f28e09956bda17a462f8e9599cfa29c86076bae0b8af3
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.25.self_attn.o_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.25.self_attn.o_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.25.self_attn.o_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.25.self_attn.o_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.25.self_attn.o_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a18dc2e1fe99043dacbf7ef19df493be22932e53
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.25.self_attn.o_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8d2ecddf1b02fde5cd0848076e494fd5d2add9d64853762e898dfeb55f1bd07f
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.25.self_attn.o_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.25.self_attn.o_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0e40f1253bf7428a2d123dcc7ea85ac35474035f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.25.self_attn.o_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f4ea1b1f5a491f94e62404b5eebe280d706a385bce2f5dda4f49fadee473bc88
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.25.self_attn.o_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.25.self_attn.o_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c7c695df80f5e042f46bd62419e61688251eaabe
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.25.self_attn.o_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f7f82455cbd29255a51d6bff324f8d64a2a4814c1811ad752337d7ca2a402bcf
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.25.self_attn.o_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.25.self_attn.o_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.25.self_attn.o_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.25.self_attn.q_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.25.self_attn.q_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5e2b819983c21660eb6c03268cdebb2e76628fb1
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.25.self_attn.q_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3d18f7e825cace95ed09e75f4093ce56a4f3c340b1ae19a75b0898d5457b332e
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.25.self_attn.q_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.25.self_attn.q_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4d69d4c66d017ab8e2b3ec377bdb06a770f4a7d1
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.25.self_attn.q_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:86e113e80418ad9156aa887cc1a646e5678f4b6fd1b5b0adb73db06b4b240a2b
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.25.self_attn.q_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.25.self_attn.q_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..37b2386cd832eff1b458f42d5445665c6013739d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.25.self_attn.q_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1b29264fd5e647275afe9cdf54a848590eedda405fce14751cecf6e6fb7989f7
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.25.self_attn.q_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.25.self_attn.q_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.25.self_attn.q_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.25.self_attn.q_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.25.self_attn.q_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6902942a3e2e562f08cca1418bca7a3ac039733e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.25.self_attn.q_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4121b011634c608783afb37394ce8225d1971f800ae2131a4d1d894d5da83f12
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.25.self_attn.q_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.25.self_attn.q_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..bbca712f8d0e73dfff0473e64f67b788f01e19c6
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.25.self_attn.q_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b445a6e0266ab1e322eb890ed1a97f00a7068d1ab7dbaa0212fc7e81769849cf
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.25.self_attn.q_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.25.self_attn.q_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..83a54e6c79538ccd2036ed85090437c896ceadd2
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.25.self_attn.q_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:347c80350d00db935169c17217f9c9b9d904899933350bd0f4a13a70dd0fbb4b
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.25.self_attn.q_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.25.self_attn.q_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.25.self_attn.q_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.25.self_attn.q_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.25.self_attn.q_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..74eb31cd48412239f70ebb3e02c56095085859eb
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.25.self_attn.q_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4e46ed88d38f1f301ddefd3b19dbb5454a3d52ce8ae73294132205315e62db94
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.25.self_attn.q_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.25.self_attn.q_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b16e406213c45f5fc5871b8ebb230a7e5a27afbb
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.25.self_attn.q_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ed4b60483ad3792c797c52e8c9f3e1aaf58ee7597364255f85fc5f906a6b675c
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.25.self_attn.q_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.25.self_attn.q_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8836c901470947a3e52ca9ca1760c934fdf7e4e0
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.25.self_attn.q_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aa6c4200aaadb3c2e5faf3c8e7668f1339dd5d2f95b95c5206d663ab529c1afd
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.25.self_attn.q_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.25.self_attn.q_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.25.self_attn.q_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.25.self_attn.v_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.25.self_attn.v_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8319ec246cd6a0ab6abb2518c853e72888a23150
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.25.self_attn.v_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:77a0a5aa84894525e0271e04015f29047efa00731b9d9635a619743b6f080713
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.25.self_attn.v_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.25.self_attn.v_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..544cf1c646a1773eddfdbe0daa1a1d2221e1c259
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.25.self_attn.v_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f1c85ee5947844d755773ada649ecaba36a1e60d3698f148d8d4c01cb8f40d4b
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.25.self_attn.v_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.25.self_attn.v_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a807264ee132b95416add63dbe6e0172bda7da5d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.25.self_attn.v_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:528f45264dd1e4f2bd379a7a87c946f7ca856aa26d0819074a43a21eb74da0eb
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.25.self_attn.v_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.25.self_attn.v_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.25.self_attn.v_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.25.self_attn.v_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.25.self_attn.v_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..86f14e538cbc059ee16547fa9d623fd4e59ac607
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.25.self_attn.v_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f81bb884c728c0ecb065a23ba38e159aee4e076ddc41b78684b539bc2ed9a009
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.25.self_attn.v_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.25.self_attn.v_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0faad0a6067b82c1305108a330e89d7ea940d07d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.25.self_attn.v_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a4d38bf6600c78b91be647fcf091be272c58fe100deec5ea40b600c05221a730
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.25.self_attn.v_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.25.self_attn.v_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c3c28bb8e38e173b4a632a8a17cb248f5f0736fb
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.25.self_attn.v_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:222369f2d8c21ac0433b732a88243adc9ef3495b2d1e2961e57cb236d0e9456a
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.25.self_attn.v_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.25.self_attn.v_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.25.self_attn.v_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.25.self_attn.v_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.25.self_attn.v_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..dfcdb00d185af6390fc6fad0c6b81346a36e9000
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.25.self_attn.v_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2e99faebc84f740be24a4641c0098c6d07b1655db021382dc05ad5240effe280
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.25.self_attn.v_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.25.self_attn.v_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a35e94e434f4d2be36b4de44d87cb79bdf1050c8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.25.self_attn.v_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e3203b6f4da14c1f5fe63da944bcc09bf3eed4cc231d65bfd6ea39ff1fa84ccc
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.25.self_attn.v_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.25.self_attn.v_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..94456a3f11a8c15d5e9c68e1fd277641fe853be7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.25.self_attn.v_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c56ca29f3006917f99c9502fdb1c15818f9fca1dc23bf9159683a9b2b17d6308
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.25.self_attn.v_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.25.self_attn.v_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.25.self_attn.v_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.26.down_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.26.down_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..271500aca8f0c35a559b9f6df6ee14089ba2eec0
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.26.down_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:889ad6789038f2c597aae06c09f683065dfeeb80013bd22f2c5247e3a2de5932
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.26.down_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.26.down_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7e571c54e4495fca0d83308f9e8120cb9351de69
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.26.down_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3989086bb97b84eb842251ba644af527b1f77daf43eebe9574deaa15e53c5a49
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.26.down_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.26.down_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..03c6e7a9dbf72dd3e0431f8ad18439e2f26c7fc8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.26.down_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a420eae5b5063233fc1a7b8370842712e8c7a9709b9af037b3c231b167ff5abe
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.26.down_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.26.down_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.26.down_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.26.gate_up_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.26.gate_up_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d9c229fc16ae4e898eaccb0b5d70ee401ff14778
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.26.gate_up_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1572dc53aaf09d488356331a5fe3af4cb4a9ac0203fc7c2418db04f495ae2c98
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.26.gate_up_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.26.gate_up_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5098de7138e1be7403b21ec1109dcbaea49f187e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.26.gate_up_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3ad533d34132fe1c62fd2f229f9f7544a7047c8779a22e689748ea7446790340
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.26.gate_up_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.26.gate_up_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8f6f81f064ca32e8e64e9e292d5147f406d597ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.26.gate_up_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eb4adb13da84f270232d23b42971d9951515b2b0517c9900e5826d2d88e584a7
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.26.gate_up_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.26.gate_up_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.26.gate_up_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.26.input_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.26.input_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..72161ddd705ed3dcef047c24a1f681f9eb7ae1e1
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.26.input_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a7448b84e74cd8c1c5b442185b8dfafda61dba2c16647819d0b6e57ea54fd719
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.26.input_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.26.input_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..466951bdc77c90e79d9d90a5cd0cff5bdd3ad1a9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.26.input_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5349a39e8f470555974020ea125429a342fa5defcdb99cee51f64da2fc6a411b
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.26.input_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.26.input_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6681c8bd428499850039ff27f52254c45ff081df
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.26.input_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c62e2eb68419193416c3600eecd7f91702dc16b4c4a04221cd380e17835f9c6
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.26.input_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.26.input_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.26.input_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.26.input_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.26.input_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e6f73eab7deb2f4db847ab269aa6ba6ffe167d14
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.26.input_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:39d51fcf41750a22d0de2254ebf4a3e026b00d46a823afbaf60b7c7859512f35
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.26.input_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.26.input_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..83b5665964f27daeaff73c590e86816bf01876e5
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.26.input_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:52e97b8ef6c812fddcc0242f56a1e710941dd15f3a6eb94a43b0f48b4bf80b9d
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.26.input_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.26.input_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e68a37ddb66e6babe1fd57485c6f6a4e660b344f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.26.input_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d881012e4eda2bee93c240ac3c2e2921b688892617aa718e60cec12ecbf5f4dd
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.26.input_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.26.input_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.26.input_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.26.mlp.down_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.26.mlp.down_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..81e358bd097fca723f7194095f824fc167833076
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.26.mlp.down_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a99cd6ea78b896d4d06441835bb6084c3d12081f6e40bec616c726d3e8561581
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.26.mlp.down_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.26.mlp.down_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b599d4cfe147431b3a91167554be55d689e94d96
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.26.mlp.down_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:70a07555f28f66b60bb2fa13ef3639f75dadab036254bdb2dd56f20b64baa3a1
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.26.mlp.down_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.26.mlp.down_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..095a0a17bc0c535ba69cfd4bb4e81bd1daa344cc
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.26.mlp.down_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:15ee2ad721891d2e0e54b1e479600b0977cee30f0f89b9586f70f5d350e7dce5
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.26.mlp.down_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.26.mlp.down_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.26.mlp.down_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.26.mlp.gate_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.26.mlp.gate_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a2842ad18b5857351275bf0bb83f0d9d79936903
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.26.mlp.gate_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b6878df06180fed71cf91231218b25950e353f4e89a00e8608fa8650b7e6c1c0
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.26.mlp.gate_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.26.mlp.gate_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0ec7b407149027f596b673bbb3ac4f98cef2a7c3
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.26.mlp.gate_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a2eb3a516ed9da666a2732953a2eb6f67558df2c0081b2aa5251e6965203c92
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.26.mlp.gate_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.26.mlp.gate_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7e95f42b2fe2cd21fff443ead6c19bf6c181f56f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.26.mlp.gate_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:69c87b00e14a92232e12139c66a48188dec3bfe39639eb9dd8e6f434c0cd8393
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.26.mlp.gate_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.26.mlp.gate_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.26.mlp.gate_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.26.mlp.up_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.26.mlp.up_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f0737b9249c5f86630e1b012ddd69960d083f6cd
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.26.mlp.up_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b07faf57f9ee4cff203bd383858d8008b11dc8e47cb01292d2b96aca44661f0d
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.26.mlp.up_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.26.mlp.up_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3cd316b7bc997eac5b1258ed838ad162abafed46
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.26.mlp.up_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cbb4fe6622a22005a64af20af0936f92f5cd8514be42efe2ae4f6614ca134e1a
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.26.mlp.up_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.26.mlp.up_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..651a98b5045f077a46704455725ecbb6aaf436b4
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.26.mlp.up_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:91d4ef7d02f7ec541e127e5f50995e9fa1ab747a117a6f88da289dc08461a7ab
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.26.mlp.up_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.26.mlp.up_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.26.mlp.up_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.26.post_attention_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.26.post_attention_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c8d7a56b40c93d3c4353804a8bffb7ef1c6206f9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.26.post_attention_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ded066d52fa5f950035e2abdd9e42dfe8f5e8ba42ad459717939aff30d530a48
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.26.post_attention_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.26.post_attention_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b8b87cecfc4aae1ac64934fe090f22dc08b69fa6
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.26.post_attention_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:206b7a67bcd68e6c1fc6733543c5385554b2776ca7672716123fc7c0e79cf2eb
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.26.post_attention_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.26.post_attention_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f2fcb4f2bbf77518fa5859e81056c9ab0d81ca28
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.26.post_attention_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6691d99587576c877b29e765a8f9f7d0de754f388e91b605e8618e67809003ac
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.26.post_attention_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.26.post_attention_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.26.post_attention_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.26.post_attention_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.26.post_attention_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..abf95dd774b232a2dc4148965e64aab00204a252
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.26.post_attention_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:68e4dd5bf3e97b51c5b126746d850cc8dc12ecd936a48e60e57e3921a1d0dfcc
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.26.post_attention_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.26.post_attention_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ba5ae686bfede5d772217928813022c6bf584797
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.26.post_attention_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:76e60fb4b9e80c5ec0ee5bfa9d83dd97715538355412a5682eb937a9baeaecc9
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.26.post_attention_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.26.post_attention_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5582290a18e03eed4254a3950bf466686e8baf21
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.26.post_attention_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9138d75c9b4793669dc8e2ec8a56582699b13e4a10c33df44b5520e257cfcab1
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.26.post_attention_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.26.post_attention_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.26.post_attention_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.26.self_attn.k_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.26.self_attn.k_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b390c1a51e5377c2c88605636b6466e0b236636b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.26.self_attn.k_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7d233961c5088ba7fd7acc7ce91a5e3714ac15c7dfba6551b255656d825b9393
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.26.self_attn.k_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.26.self_attn.k_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..bfa991047888177aededd3321f3bb5ca022a3dd8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.26.self_attn.k_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:13fdf865ba6c5228f24b3f44ce22ccb209cf64d7f6abed213ad2c2abed377562
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.26.self_attn.k_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.26.self_attn.k_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..00978773a887fc763b4d60fff2f77dafcb3ce2f3
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.26.self_attn.k_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9da6528d814abfb1f5edc66ce43293d64e879ea3541c43a8a658d7e6e9532abe
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.26.self_attn.k_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.26.self_attn.k_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.26.self_attn.k_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.26.self_attn.k_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.26.self_attn.k_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6c61ff92a7038a64f06d4fe8adeae3291b968913
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.26.self_attn.k_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9b9cc68eeab3ee9fb5e9af47917731b79be9032cd9f730fb3629bca78359ba8e
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.26.self_attn.k_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.26.self_attn.k_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..725d9068efa851847cc6cc7de86ad04a71ce2ba5
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.26.self_attn.k_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d832e7c7510bfab09c7ed14e4534006a2c34e5df4f4bdec2272f0a4729f54eb5
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.26.self_attn.k_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.26.self_attn.k_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8af2914b9f6b06c1bd6b879c6f075b306ea94a7b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.26.self_attn.k_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ecf3a561616a0edcc69075be8d6d4b5d600d3e05b64ccfba6d52b104d7ecc390
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.26.self_attn.k_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.26.self_attn.k_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.26.self_attn.k_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.26.self_attn.k_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.26.self_attn.k_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1984a24ddb2f6cf178f43b542c543b73f4914090
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.26.self_attn.k_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:abdab0714794e880c775f6ccf60f5019110808e514f23940436c29b7b3829bec
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.26.self_attn.k_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.26.self_attn.k_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a85089817be98647480f7e6a1d440bf974fef81d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.26.self_attn.k_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:22c67f1fd4e82ec91d96bbf5457bbee84994156159ed48c6307f3a5fd6937a30
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.26.self_attn.k_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.26.self_attn.k_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..eec6047e2781105803c64bfe5394c6f48aa49636
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.26.self_attn.k_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c80254ce7d07f807326d2b0309ba9e49da95cb185278596e5881db7f5bfb3cb3
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.26.self_attn.k_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.26.self_attn.k_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.26.self_attn.k_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.26.self_attn.o_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.26.self_attn.o_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..93f0ce466d621975a67ccacb315aa5d910fb9e73
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.26.self_attn.o_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7517078de8ca2e641bcd9718264ad082676fc4949c2e977189c9d31168890778
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.26.self_attn.o_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.26.self_attn.o_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5e6a823248b6c4790a2aa99f649fc51723b57d3c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.26.self_attn.o_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:37d4306b99b0944a977137b219cb10bbd32cc3accdae70fdfde6dfb70a0e716c
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.26.self_attn.o_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.26.self_attn.o_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..18c9fb6cc592508af20fbf9579234f460873f4eb
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.26.self_attn.o_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:615d0aa1fb65979bf15bf37601fae4887a8e123950c750cf75ff3a5e37eed0fd
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.26.self_attn.o_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.26.self_attn.o_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.26.self_attn.o_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.26.self_attn.o_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.26.self_attn.o_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d8e914786cd308e3a9fec2ec7bcb43ef6a5b3ec9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.26.self_attn.o_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0fde05c3036930d53a5f989d68bf5ee8f63f049393ba37392bac53794757caf0
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.26.self_attn.o_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.26.self_attn.o_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..abfa5ef8d4da1c1ebe76a1e05bd449226e4c12ba
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.26.self_attn.o_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f073f991e995bf27658798dd1fbd71a2218a3af632175fd033720c0386166ad7
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.26.self_attn.o_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.26.self_attn.o_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f563efc192b75ee3eb09522cd793dbe17aeaeafd
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.26.self_attn.o_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a2c0b845dc678c95dca34f96be0fe6aa5c457f841ef07b58f5feadbc2d73fa25
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.26.self_attn.o_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.26.self_attn.o_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.26.self_attn.o_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.26.self_attn.q_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.26.self_attn.q_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3f447b8564b61465049e41cac380f25517e8a55e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.26.self_attn.q_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:48ff48ebe20095866b391aecf1f08cafde6ec0eba9671b253dd904e6c7538fba
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.26.self_attn.q_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.26.self_attn.q_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a399f43ffb9cd18c20ebe5270a2496bc6769e3fa
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.26.self_attn.q_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6216e671c3f9da658cda5c3ea1d2a4f68b9a4bd78a10a8a73706df7a33d944af
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.26.self_attn.q_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.26.self_attn.q_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6369c35b1ac1e022a9965863098029ec4e030fed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.26.self_attn.q_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8fae00f85e5754e88b964bb9f5909df959d6151ea8a6805a9fcc783b91b17b30
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.26.self_attn.q_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.26.self_attn.q_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.26.self_attn.q_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.26.self_attn.q_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.26.self_attn.q_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a3820cfbda01b1aee6c8ea5cd4edefc49c88dd8c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.26.self_attn.q_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0475b2683d764d26d356b6d7dbf225bfbd187f2938b44a42fabe6c92d427b297
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.26.self_attn.q_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.26.self_attn.q_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a78b479d93d9c98e518e490af74fee42c53fd957
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.26.self_attn.q_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f6c7adc2c52fc1db05b4f45a823b1234faf8565af22c8fbb4fc7f849f5852806
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.26.self_attn.q_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.26.self_attn.q_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3573fc53c0f70393749a04d09ac557d5ee767b8d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.26.self_attn.q_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b5549f726fb39a156f4e45ef175b35e80af4059165b36c0b013f6eadb1b34259
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.26.self_attn.q_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.26.self_attn.q_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.26.self_attn.q_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.26.self_attn.q_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.26.self_attn.q_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6d20d2073c326c0e5f730efa686428c20522c8af
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.26.self_attn.q_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:70b7af7d9aa3597b807c47d5d438686122dde67b2453262a78d57881a6029198
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.26.self_attn.q_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.26.self_attn.q_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b29b4e1f173806fc935d5c91e05b81b64987f27e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.26.self_attn.q_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:37ce3740580d7dce7a365ec5e4c4922fa5a68863409e0033a9d777adcd9ff2ac
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.26.self_attn.q_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.26.self_attn.q_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b68e1a0f04d444ee7ef181a7456b3245ac41af56
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.26.self_attn.q_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2d2d3011a4d0f0d61dc3da984ea7235d129b14747c207aed04610d9f86969e47
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.26.self_attn.q_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.26.self_attn.q_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.26.self_attn.q_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.26.self_attn.v_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.26.self_attn.v_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..75c190c26b5009f987640347e1d50fc6e5fa6de9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.26.self_attn.v_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:86397324e09c31e81fe0100cecc5d14296cf5718b1563babc4a8776611849165
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.26.self_attn.v_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.26.self_attn.v_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e3323669ed237b24c6aa50d45b1041c874dfe1c4
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.26.self_attn.v_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ab0ae2ac99801cbfc9e7534ef06a7bb587f05aa8cf94b524cc9461c6267f7120
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.26.self_attn.v_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.26.self_attn.v_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..295240cbf1f9be32a37f91592b5e879f42083c25
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.26.self_attn.v_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:739b3364716c10e28dfdbb9d53f3e6fffb59c5c6aa42d7e85c43fddd4c26538d
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.26.self_attn.v_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.26.self_attn.v_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.26.self_attn.v_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.26.self_attn.v_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.26.self_attn.v_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4c64398136a7094965cb0b634aa1dcd568e766d0
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.26.self_attn.v_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3a83276f45774ba15b107e0bf313fab5df091b76495a2f656816ce6100fd9a33
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.26.self_attn.v_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.26.self_attn.v_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..717013970d292c63d6e8230a30054ffbddb8e1f2
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.26.self_attn.v_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aea2c9ae47b90475f3acf47dedb89475f1c04a61a10c087622831934a8d0713a
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.26.self_attn.v_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.26.self_attn.v_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b84801991102334e12607cc2c1244533ee8de52f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.26.self_attn.v_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c7e52c9deffb4b7cf412f8a04aa18f67b4dc40bb9be180f4add24ae3ba1d6a55
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.26.self_attn.v_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.26.self_attn.v_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.26.self_attn.v_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.26.self_attn.v_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.26.self_attn.v_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..598de0cf8afc4e720bf96b041213749a0577db90
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.26.self_attn.v_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc08622341c0c33a66a3258c2fdb047692a5684b735d43d693ddb380c19a65dc
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.26.self_attn.v_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.26.self_attn.v_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1e5f2db55b374c2f52d5cc9d3c45a6e65af2ebc6
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.26.self_attn.v_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c1087855f4313e6b48ba441534c10a5509fa56b1d0313e7d07adba9366c0f633
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.26.self_attn.v_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.26.self_attn.v_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cc371dcf1fc1bad4837cda022ceed90e11028c60
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.26.self_attn.v_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:09352f66a6f81b0275c7c2dc88927ecee83db8b5c9e180094aa6a73e257ae231
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.26.self_attn.v_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.26.self_attn.v_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.26.self_attn.v_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.27.down_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.27.down_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f4f6d92c0f0125083268b50f1952369ad20cbd6c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.27.down_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6bfe9b401da275d7ae87ecbb1751e9dcd6c7eef9ef2bda80e7736630c4168e6e
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.27.down_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.27.down_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..02a229890db0634be47eae01cd3f063ee3368307
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.27.down_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b262dbe05126dc223bc15cda3a68772202d8ef27736e184e86d020172b7aa0d9
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.27.down_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.27.down_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..491397ad15fb6368b09df08d577e7ee36505f755
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.27.down_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:13013b23fcad295cfeb0e6d5547ec8b22e0852d4f073d27dbc1c72a68d7c6b2a
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.27.down_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.27.down_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.27.down_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.27.gate_up_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.27.gate_up_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..99058714910f3277179675cdbb35e584fd2b7a60
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.27.gate_up_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f2f945882dc98cf430659b40981369757446b7c0d620b30e5491385658b44be3
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.27.gate_up_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.27.gate_up_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..02691d649412069784920cf6e8926c2e83744d43
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.27.gate_up_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:762bb42bb5958558a09d35dd9b8d57281478d49ea3165876f72c48f1b074b3c8
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.27.gate_up_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.27.gate_up_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..12965604e5cca516360808ce83dc9eb04525dd24
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.27.gate_up_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5b8f70eb5edf5b4314af0691bc28209d398b0c7ade0016087d41710320578740
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.27.gate_up_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.27.gate_up_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.27.gate_up_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.27.input_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.27.input_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4a36905f589eecab3e11e444965383a5f7d7cbed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.27.input_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1f52f73daf09294835061d51d7036de9857b927f828b31ffb3aa75ffb236b231
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.27.input_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.27.input_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b9609d956d1fbbc300048dc0f9698ccda1467051
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.27.input_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:21b8f629393612ce63f585e52cad70c7f229ccd02bdeebf4ea02cb3a6d2d8c22
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.27.input_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.27.input_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..740e6d32326faede6246e84d806f92a71026d429
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.27.input_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:059ec5ca591020dd0222376430270bdd9c91c7d1a35e058001ddfec82df44e82
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.27.input_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.27.input_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.27.input_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.27.input_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.27.input_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..29b4740c57154715e0a12bd7d22aeca42801a148
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.27.input_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:21ffa62d7437be0c27ca82cfa45551f348e683e339cd622d061fe4b49cefdfa4
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.27.input_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.27.input_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f6f1488f0206e6a5745347f6c62ebc58115a82d1
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.27.input_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2dd3d0c4dc148aa26980cac75f62f49702d3a34a02151bace35c324de27729b4
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.27.input_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.27.input_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c1d5bfc488cdc516b8d80d69f272858be65ceb60
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.27.input_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f43ce8e0b53c1b7a42717a3029fe97916ff133ba6fe05a703c4e638dc1c562b8
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.27.input_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.27.input_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.27.input_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.27.mlp.down_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.27.mlp.down_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7bd4eb2422e6224f04c0ad990d6c57273ecd1bd6
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.27.mlp.down_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:160490601540d575613e2627c9de19ddc2037a471c4a06ad5807ef08d4ac6459
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.27.mlp.down_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.27.mlp.down_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0fb60134f3656c4ba509f02bb8eaba3674c173a8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.27.mlp.down_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d720be469e75391c831e1efc07dbb10e45121d14a7047ee3ab9d3829ac68b63
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.27.mlp.down_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.27.mlp.down_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b53616c75142c3ea7469a9f26780fb80d80c0f6e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.27.mlp.down_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0425fcc24d8d39f155b7985839cec1b6e593fa9ec71962564ff333254fc7c1c8
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.27.mlp.down_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.27.mlp.down_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.27.mlp.down_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.27.mlp.gate_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.27.mlp.gate_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..29eea747a6e98f6dd3970302d38f646314f370d3
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.27.mlp.gate_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8fd6ba2868bde0318eea9e2e3662f7f59ec510bb2b42f3a3e9bd379de81c28d9
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.27.mlp.gate_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.27.mlp.gate_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6eb25aff9dc683502f32ef87ce6bc236ba90e28d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.27.mlp.gate_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:73704c56a88634d5471807cc13e756b5a973ce97fc532841ec197d8b7c5675f2
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.27.mlp.gate_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.27.mlp.gate_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0896850d6158bf1525a2e1f540d770a7c6b2c875
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.27.mlp.gate_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0bff801027562d95581f73c979552c07fb86bb8bd6b2dc1dbc77cd66f86cde1f
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.27.mlp.gate_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.27.mlp.gate_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.27.mlp.gate_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.27.mlp.up_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.27.mlp.up_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..288964c53e3054e790e990d89c29f961e9ec504a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.27.mlp.up_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:55b2d051d245141de25274f33ec8ea4ef6a5123ad8ec0aef337e6bf1af70684b
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.27.mlp.up_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.27.mlp.up_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2b52783eb373c8535b2fa378c628e9cf3221e7fa
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.27.mlp.up_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dddc93ee21cbe6d051b0569926ee67f8e12ba3dcb3b83d8d9be92f634c7cdbc8
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.27.mlp.up_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.27.mlp.up_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..deb6539bd9ef53cfad5e0ae471cf60711d124257
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.27.mlp.up_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:240b28d0d3e78dcad6227db2edc373143e9db6de8e90e8f40964b5ab1a14a12e
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.27.mlp.up_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.27.mlp.up_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.27.mlp.up_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.27.post_attention_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.27.post_attention_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c12726deb3487b3565941ca84756cb1589759385
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.27.post_attention_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:65ca209349e8bbc8b282b7614dd06d94955ff6c90208cb25581d518fe9b6e830
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.27.post_attention_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.27.post_attention_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d8e1aa59973b3e84cae5a732fc780189b73465e2
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.27.post_attention_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aa0e7b9974a5b802d9c24a61b5b6f5fed9ba8d9862309702a3ba6b8b567b6bce
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.27.post_attention_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.27.post_attention_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..75b7fc9f90d9e1b76fd4f6463193823460745f6d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.27.post_attention_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:83b0f878e225bacaf5240c8702c14331d68d20506225b4a68c2ca80b932dfcc1
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.27.post_attention_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.27.post_attention_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.27.post_attention_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.27.post_attention_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.27.post_attention_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2a1b84bd1e21902bca44f2f7a56e617945e075da
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.27.post_attention_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3daace03215638602aec404c9e5f41c96a7ef4f9677ed2273dedaf67b6cee881
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.27.post_attention_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.27.post_attention_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a23d25090a5aab3e861f1579d87b52ec0d6c14d9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.27.post_attention_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ec0cf760e2f22d19fd61933f2d437dcd90bfaba6d1d46fa55544183d4d3daf14
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.27.post_attention_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.27.post_attention_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0106c59f10ea92d027c4cf5824500e3d62022caf
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.27.post_attention_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a7ba1486f95daeb9f0372d0e006d4de2662101fa5da2ae1f3b5bb4d9131e3e7e
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.27.post_attention_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.27.post_attention_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.27.post_attention_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.27.self_attn.k_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.27.self_attn.k_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8839c2bd0736392aea49e19e1d294920b5684b34
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.27.self_attn.k_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:287d5b411284d148cabff5c15eb8779a521486ff1eb0c1b123618b7b47bb07cd
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.27.self_attn.k_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.27.self_attn.k_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..006bf50da95247617422e010fb04e4efed09e6b7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.27.self_attn.k_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dc2506874dcca439dc87d5fd7785151158c95eecc1602072309c9c92b9168f38
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.27.self_attn.k_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.27.self_attn.k_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..449e5bfa1f505d706b2b94cd84cf1e53377af794
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.27.self_attn.k_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3fbc3cbcf4b717774e67ef615bff24bb6c4d0082d4a99ff002694c9037662d1d
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.27.self_attn.k_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.27.self_attn.k_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.27.self_attn.k_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.27.self_attn.k_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.27.self_attn.k_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d572bcd9ecf3e3dda62458688be059cf2d9273d1
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.27.self_attn.k_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c22ab5605d6791bd8ab127470c983dc4cebd0ac8d0b7f873148b491560f3cdf3
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.27.self_attn.k_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.27.self_attn.k_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3b5c12709a841dcd6cc798dbf7a2b975b11cfab7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.27.self_attn.k_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:88a4d59c2f35510633f4941b2c41d1ee8a399f78688f81c5014b655fba40172e
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.27.self_attn.k_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.27.self_attn.k_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..53d68a12c1f0178ec57d931f189b186d9a124325
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.27.self_attn.k_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5c081c155307afb31c17996e4384b62d8e81ec16fc264dacf93180688ef2a8a6
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.27.self_attn.k_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.27.self_attn.k_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.27.self_attn.k_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.27.self_attn.k_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.27.self_attn.k_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fbccfa034845946e4d0dd2f1bfb8b909aad207f9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.27.self_attn.k_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:04edff7111eb9eea07a807480ed2c8846f9a40e4d205dca6fca0e143ee16cb91
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.27.self_attn.k_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.27.self_attn.k_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b3351ba200afe5081d3978d3e8a6e18e489b1ca0
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.27.self_attn.k_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eba99f08cc4bd78d72a8f4861464d88307ddeccf8b5d26978dc52dcfeaf0ced8
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.27.self_attn.k_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.27.self_attn.k_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fb7353681a9a12257d0f8fe81c044da646792ced
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.27.self_attn.k_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7895971ca951f2913dc0c56e8e97c16f0509538caadf388249dea76120855f29
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.27.self_attn.k_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.27.self_attn.k_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.27.self_attn.k_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.27.self_attn.o_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.27.self_attn.o_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e5a6b28f9fa5f30f9ceac731e189a22cfe455fe2
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.27.self_attn.o_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fac056f87a204c5a9c0df4171add1def1fd7519534b92318398548bc1377c8b3
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.27.self_attn.o_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.27.self_attn.o_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a67a02e9a29fc2bcccbebfbfbb94f82daabd722a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.27.self_attn.o_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0ec6c76e227eefd1768205b3ce7565ba9d92dfe7b7f3259867864f9cb93516d8
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.27.self_attn.o_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.27.self_attn.o_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..aa6aba4d6a0a7b32d5b98458d3ba23484603c32d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.27.self_attn.o_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f7b7673f24f7b8dbe136cb2de300e31f407dac5e4582ef89362aa81f87789efd
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.27.self_attn.o_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.27.self_attn.o_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.27.self_attn.o_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.27.self_attn.o_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.27.self_attn.o_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b0abdf98b39b091188b49d0a14e33d71932378df
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.27.self_attn.o_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6cf4265d811af0dc975a7f4999534ed5dc24c599ba68d83383424a3295893f84
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.27.self_attn.o_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.27.self_attn.o_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d21fc0517df9deae1e076695e9e238e5af928f13
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.27.self_attn.o_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:118479d092bfd1fb79ff5e4dcf354bcf8a5a24761e23ac466ae3bdd20f56f881
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.27.self_attn.o_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.27.self_attn.o_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3593ec58e8c9ee65ee5b344154000ab4caf0abdc
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.27.self_attn.o_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:911ea275c6d332a9a1c9d346f7193731a002af63bf4e2d7223d23bd286cd9824
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.27.self_attn.o_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.27.self_attn.o_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.27.self_attn.o_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.27.self_attn.q_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.27.self_attn.q_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e2aacf895b937315b2eacfcd6c1d5eccc4e231d8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.27.self_attn.q_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:baff516bafc514e94ead9e5ed1d3d928f924d0cd93dd85a42560f4e2ab2c0bbc
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.27.self_attn.q_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.27.self_attn.q_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..37f6e883aef530fa32a72291259ba2d785deade5
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.27.self_attn.q_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1b59e073a0dec9c2ce0d66f81285da13b858c478e139d7096b49decf3ca59d55
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.27.self_attn.q_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.27.self_attn.q_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d60608bf4ddf1e566953073253ff18bbef7dac7b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.27.self_attn.q_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3ffb28c7a5a621ffb7b930c803faf6b36502e508375c568a7a9c9e43191b65fb
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.27.self_attn.q_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.27.self_attn.q_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.27.self_attn.q_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.27.self_attn.q_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.27.self_attn.q_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b8aa409276999ae942e575832c78867b59c05458
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.27.self_attn.q_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a31e88ce2f6edda93d2bb65708342b419e39826f3c7475e387b55e415542c6a
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.27.self_attn.q_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.27.self_attn.q_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..24ad4d2225ce866ba60d70409017b94a46a4f1f2
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.27.self_attn.q_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fd7957102d462ffd00020f155ed4d4b14f12f0dafc9da6e61e4200dec7b037bd
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.27.self_attn.q_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.27.self_attn.q_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d0af613e0b1abd60f16d6ccdcc5933132710ac3a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.27.self_attn.q_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d13fe201cbc913c9d003b2764358158e585eff1aed1f61d8cd4959355a6c5380
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.27.self_attn.q_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.27.self_attn.q_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.27.self_attn.q_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.27.self_attn.q_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.27.self_attn.q_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d47c3e5d297fe702f15ba3eaa0e7cf469ed81f29
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.27.self_attn.q_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f9f878037c6e8fc49463ee090aa2ed54a513543227f1323b0bf1914f0cdfb524
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.27.self_attn.q_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.27.self_attn.q_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..24bcb3bf6c1686a0f0450e4a897b284c80efd822
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.27.self_attn.q_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0eae0c4cc463dd95b3e5c8903b46b4f26d01fe6e3717ff43a13bbe2625cf659d
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.27.self_attn.q_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.27.self_attn.q_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..71ea90344c86c3bc9b63ed854b7c3f6aceafc132
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.27.self_attn.q_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe81fff820b215aca5cfc3ffedf68745246f0c1a2ceeb7ba7ea333f22965831e
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.27.self_attn.q_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.27.self_attn.q_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.27.self_attn.q_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.27.self_attn.v_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.27.self_attn.v_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9b0e613149aa84cdb16340421389ba4830208067
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.27.self_attn.v_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c2b6c0ab56863a177421405a27b6de0ecada934fd759c68e1008aacfff052ea6
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.27.self_attn.v_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.27.self_attn.v_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c75fbb8f48c8bc527d5854ceb5c8171f02a456bf
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.27.self_attn.v_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:30d461bac77a6c3d45dc2957c49d713d0e393abcfb9dd6d9a1e2b22bb9756f3f
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.27.self_attn.v_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.27.self_attn.v_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..543aa38457d74c508d3e48645aff32191d7280f6
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.27.self_attn.v_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:437e66ad87d77f4cae497c1c35c80b0c96a50871510727c46f484c95240ba40a
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.27.self_attn.v_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.27.self_attn.v_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.27.self_attn.v_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.27.self_attn.v_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.27.self_attn.v_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ef9746b9da82a3bd7a46997a3a16c0a9d47678de
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.27.self_attn.v_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2adb6c8aecc8715456dc31c213b48ea157e894ff26469de81bac5fcd06732028
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.27.self_attn.v_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.27.self_attn.v_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..665a5ff93ddd9aac81e3ea2f112e4a55f6d3d3bd
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.27.self_attn.v_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2c6f5c4cd208b014ff8eccb8f9b22d1abf6557f2217cef7115cceab40bea7c2a
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.27.self_attn.v_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.27.self_attn.v_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..84639c970e877f56a0699ae10ea1ae594d0e481d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.27.self_attn.v_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:265ea1ffcad671e68706ff73361b9e6d47c108a2720a94410abe5009f994b2bd
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.27.self_attn.v_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.27.self_attn.v_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.27.self_attn.v_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.27.self_attn.v_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.27.self_attn.v_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d97e5bc3b7d5c6ef7b934c22e968fea14700d147
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.27.self_attn.v_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4dc816ea2a7d2de1870a57d21e99da6cfb7d8957699a21724e479b4545aee755
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.27.self_attn.v_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.27.self_attn.v_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f7714e37636c459408cde4ae3fab3ce91f3827b9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.27.self_attn.v_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:49da9d4633c3225f70387d76b4c238bf39bf7863382b2c3274c38d95e0002244
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.27.self_attn.v_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.27.self_attn.v_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6f5e5c6dd5239530dd6fdff926a4aa06455a00e7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.27.self_attn.v_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6385f0d1db51d1c98e31c985c0cae80afdd218430d374114823aa0a5e26437fd
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.27.self_attn.v_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.27.self_attn.v_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.27.self_attn.v_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.28.down_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.28.down_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..26715325d4d8d53d9672da01edd733c16595f159
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.28.down_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c7d785f62e1a5edd62621ba9af56b8ae8a7432ad1b9729aff8d96901d203cc75
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.28.down_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.28.down_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1e494e5ed7eec800d382787d7e38e87dfd1948c0
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.28.down_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e040085c98b3f5a38e7f6e85598bfcd5768916c5079cf4fc9a9de22ecbe4d158
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.28.down_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.28.down_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..33f1b7ba27702067face8fde39e9a4bf156b8d56
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.28.down_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b04a499f59c204aa2e5bf6db24e00f8b4ba2d5fc4b9683691f53fba46d7933b0
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.28.down_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.28.down_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.28.down_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.28.gate_up_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.28.gate_up_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8fabedc6cc496ce8be924139db4d8bf44f5002f8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.28.gate_up_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:17e9381d1e2f9d7de4147537706ec21a1f3576e164a5b03bb47586dc5da61373
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.28.gate_up_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.28.gate_up_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..044ab3db6b679c9050657b0722130700464741b0
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.28.gate_up_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:55ad6758c66365d98e3de7dd77e1c219933172dedf8a74aaace277bc6703d4ad
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.28.gate_up_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.28.gate_up_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b0d49e7e8ed39a3d845dc0ca9bdb96ba694243df
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.28.gate_up_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4bcb7b965439dc1ac578e98b80761d02dba52d4d7243c3b9b523654d72f2bac7
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.28.gate_up_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.28.gate_up_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.28.gate_up_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.28.input_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.28.input_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e01e2e7bdc95d9478bfcbe04bb1505233e360d2b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.28.input_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:94210e4d0568b2a0d41cb1a205e0b952b217e28a39125b46509d6d8bde3e8471
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.28.input_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.28.input_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..dc52764f4e49f9a83d463ec4a68a3fc5ad49f992
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.28.input_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:042cfbad7884eb39f35d43676530a29d9475654737e57bb9a17cf681484d28bd
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.28.input_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.28.input_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fe7fa93b11104c86309670cd6c6cbcabb0d5e063
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.28.input_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fd6d06ed1b8ae63c10c3b8cc6da6b326492c27a8380d0989c2589dca0ddca9c6
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.28.input_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.28.input_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.28.input_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.28.input_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.28.input_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..897ad09a8824e3f644bb9790b84d08aeaae583f9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.28.input_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:97832f88aa75d89cbcf673c79b69fe11ae03a18a624d16a6223ceb315f2ce0b2
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.28.input_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.28.input_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c73d579fe409e3e698a49efaee95663ad96cf6df
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.28.input_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:08a25362f7387af7293ea827b3f8c0e9464e5ac47f581ad0432c7adad967aa3d
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.28.input_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.28.input_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a8bb1f2dd06e65d0640e9164523e81725da3d027
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.28.input_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc95cf9695dd2183be2998a296f32e360a6938a20da001490faadee3255a358e
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.28.input_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.28.input_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.28.input_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.28.mlp.down_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.28.mlp.down_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f7f1f9fe88f95b5faa1767caea2ec97e849ece12
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.28.mlp.down_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6c0cb4bbbad63244acfe0a8c3601e5c63741a4556ee004045874cc3b39a352e9
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.28.mlp.down_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.28.mlp.down_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c65b476366d4ab8fafbd4fcd86a4e3b438d5ef4e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.28.mlp.down_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aad8fc9e75946a5e87488075667a029c5b50cafce655d750c1d5f5a3aa952b5c
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.28.mlp.down_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.28.mlp.down_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c733f2ce30f0e0ab636d91e61fd3327146b49ae2
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.28.mlp.down_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:894ee49e580d160caa1772f0d3d6c9ebefb1539909f94f3821f12664639c41b1
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.28.mlp.down_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.28.mlp.down_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.28.mlp.down_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.28.mlp.gate_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.28.mlp.gate_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..460616b5e77ae232babdd78ae49ae7e8218c0239
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.28.mlp.gate_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:346d563b0fb1ce295ed59c5085192c209650215e59f1d08e75617a71650550d0
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.28.mlp.gate_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.28.mlp.gate_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d84383c99a0dc0aa87fe0ff08ec6f5b1f5a24387
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.28.mlp.gate_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd2875e6fbc372fafcba1b6b4baf8d0946c1d36774f59f5601d26429f86d363c
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.28.mlp.gate_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.28.mlp.gate_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..af74ebb886649c0fe0cae1c468c3384b83ccae27
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.28.mlp.gate_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a68b4ba6a781cc50e2588d513c10714f18da59e76b54794f820d3adcf064577
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.28.mlp.gate_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.28.mlp.gate_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.28.mlp.gate_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.28.mlp.up_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.28.mlp.up_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f26d32f07e2384517eaa47082c5cd3c3bca8287f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.28.mlp.up_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d9ea46f6676718181692101903d563eaafae9cfeb25654ea30e6ca897635814a
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.28.mlp.up_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.28.mlp.up_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..406a4457c261a4da8c1cc0fc3bab33dee067e8f6
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.28.mlp.up_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:add4a791a3622fdb75648e9aa089255719f12597b1b455cb01c9d79dad6badce
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.28.mlp.up_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.28.mlp.up_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ec5a4d853794eba29a6b840791c8385ac36296b5
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.28.mlp.up_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a96caf848e04577d80efb3fa26e4945470b7981aa52ae0462f09023936884d00
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.28.mlp.up_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.28.mlp.up_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.28.mlp.up_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.28.post_attention_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.28.post_attention_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d6221ba5fb2632e97ca51abdbfb363461e6ef2a9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.28.post_attention_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3ee8d6cce1fa95be09253fcaec76206ca8bad41c77ae6af0206cb7d75324b4ec
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.28.post_attention_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.28.post_attention_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2eb92fb8031fa7d81d67c5040e09414c07c6c14b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.28.post_attention_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:543085c19a04655c831b1e98a0c1781852d7fe7494a24fe7ec3743ddc1900ef4
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.28.post_attention_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.28.post_attention_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..943d42361f6afd02a3d3da306a57958faba16e66
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.28.post_attention_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e40cc86b30fb665e36b2f355e3589a4567f26ca984c9af15ef525d4ad0675bd2
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.28.post_attention_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.28.post_attention_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.28.post_attention_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.28.post_attention_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.28.post_attention_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3e9af9fb8aa1dd9a4dba1bb5c01e44f8a94d2557
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.28.post_attention_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:402003efe40a1e80e6b155dbc18d9bee7980f0f3009e29f9cc532c41b139f825
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.28.post_attention_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.28.post_attention_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..459c97c63410a5cf57b892ecda8e2d2bd0230fb0
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.28.post_attention_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:52083486c06dc067a521cade1ec537b7fd1537c96fa93180a662eb16e4f28aae
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.28.post_attention_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.28.post_attention_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9e2670717de1f0b2ddca8887c433bd56c9cbd672
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.28.post_attention_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ab035fc6a6e5179dddf6ea8025ec6ea290f853b7968eddfb2c64343e135151f7
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.28.post_attention_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.28.post_attention_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.28.post_attention_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.28.self_attn.k_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.28.self_attn.k_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9d078d0590d92f7a24faf441270b20b5a7355772
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.28.self_attn.k_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:51dfc4ca70a3efcf81908a5629f0d5ae3b322456e0be02e3c221158f1f4bf3eb
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.28.self_attn.k_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.28.self_attn.k_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a5a80283193ac76713b735d82ca1bce411000626
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.28.self_attn.k_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fcd6196d15e8ec44372faf7659bb759b059e656da205293bd92dc9bef1968418
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.28.self_attn.k_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.28.self_attn.k_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7befad0773446db69bfef5d94780e67d76c41c7f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.28.self_attn.k_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f45de32d273c0ef9ca2883a090fe06f62dc3ec5aabe09d81cf29790ec06ab3f1
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.28.self_attn.k_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.28.self_attn.k_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.28.self_attn.k_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.28.self_attn.k_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.28.self_attn.k_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..469c1ee7955206f416031b40671592fe8b4b68c1
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.28.self_attn.k_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b8b8397dc45c395b4103b0357a12c7b9e5a48cc735d01b9f5e92cdad43b19aee
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.28.self_attn.k_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.28.self_attn.k_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..21c61cd60721f4e442c29c659b285c31a8252855
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.28.self_attn.k_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a77136b6561a8bbf9144c8fedbb67650c024987fb2f5e2e28f5c0795c86a8f91
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.28.self_attn.k_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.28.self_attn.k_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d7ac4ea3be44ed6f05d19fb5653fc54afa95bbee
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.28.self_attn.k_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5baa8faa6d52dfecb69c7c5d0c5b09ad33c189df1982af3b412f65f294aeca17
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.28.self_attn.k_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.28.self_attn.k_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.28.self_attn.k_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.28.self_attn.k_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.28.self_attn.k_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5fdc34700de3991bc1bb0cf0aeaedd91ec03db79
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.28.self_attn.k_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ef30ce0f20d80eed8a455ba5a59135a6f5c5df596cc13231f940995369b9afb
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.28.self_attn.k_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.28.self_attn.k_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b4651fb75bbefe448d363ccac5a9caf33488861a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.28.self_attn.k_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:66eb12b380a1884cf0626b8c1b46a0910baefe69d5723f5fbfdbda80a7029edb
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.28.self_attn.k_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.28.self_attn.k_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b2d1c86805213ffc0eab810056216b0231d2842a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.28.self_attn.k_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:536ce2876931a85b2819d284d72788d1fa52a407ae1aa896226612ac85069d4b
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.28.self_attn.k_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.28.self_attn.k_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.28.self_attn.k_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.28.self_attn.o_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.28.self_attn.o_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6d7ea2cbac213152e6673bc0aa228d383af9b426
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.28.self_attn.o_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b42c839c198206a32e2c67d306a4dbb733176369e2514927eb996ef6821cd536
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.28.self_attn.o_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.28.self_attn.o_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..efae4e54b8db1deb7dc6984346c308824e717ade
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.28.self_attn.o_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a4a39621b7fc76f9e04d5628739cc85a6092180ad34ec3bd39add42c95269bd7
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.28.self_attn.o_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.28.self_attn.o_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3f0f3b18ab65f343cf32cd9f015fd027fc5f8cbf
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.28.self_attn.o_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:577f31ce47cc1532247bcb38e22461f6ab469141678915852ca2124d2204fb6d
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.28.self_attn.o_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.28.self_attn.o_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.28.self_attn.o_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.28.self_attn.o_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.28.self_attn.o_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8eef474c66846a80e2a16e7f7e114438345c7a50
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.28.self_attn.o_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ed07bbc71828bb68fabca9f082b0a34719ff997a78ac64fa75c04c91d93f89e9
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.28.self_attn.o_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.28.self_attn.o_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c4a6cfbf3449c98136d3c8377afc0b0ac034fe40
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.28.self_attn.o_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:83aad5c000384f587843b48c4148ec02d6e6970e031cd8f1b3a4172db3e2843b
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.28.self_attn.o_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.28.self_attn.o_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..74a2b2ce59e4cd4e7cad762be16a7a60bb890165
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.28.self_attn.o_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e8449c563068a76ef54f46691b56e2501629cff20134a247deee59123db5328e
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.28.self_attn.o_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.28.self_attn.o_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.28.self_attn.o_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.28.self_attn.q_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.28.self_attn.q_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1b116ff9172bafcf27e2220978b797c837066623
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.28.self_attn.q_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0bd62c472e0c3314ecf2fbbe9e5ec839e05ed248b3ad04cb981d6fc91ea21a7a
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.28.self_attn.q_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.28.self_attn.q_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..54cc5a7bd8b14598e765f99ea6bdf26a29d1d5dd
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.28.self_attn.q_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d83fb51c756427c02a8efd09b858d34a1fa7df1c255135d46bbdf88b1cb00371
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.28.self_attn.q_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.28.self_attn.q_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f7b22dba8a911abb228408586a2adfa625609e3f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.28.self_attn.q_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c44224734bfd49f4b71d7f220ed4858a5021af210881bebb4ddb342f5bbeef10
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.28.self_attn.q_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.28.self_attn.q_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.28.self_attn.q_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.28.self_attn.q_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.28.self_attn.q_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..848de7ff7f4e71e7b4531ee207a7458eb4a902f4
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.28.self_attn.q_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ed4cbad0d6a63343327ed820a6ef47851faee98b8cf2d1475d0f1c4af3f2c47c
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.28.self_attn.q_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.28.self_attn.q_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..bf7a81b1ee2432f73d29adf4e5ecfac62a242fbb
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.28.self_attn.q_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e145d89a46d2fd961fc9e7f24c4ee2364618179a802852fd4e0331609074e06d
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.28.self_attn.q_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.28.self_attn.q_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c67482242537e2159204ec4baf08562347438025
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.28.self_attn.q_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:59ffa68e1a35c79b6d1d47b04e640a1141c8204e1cf88deba2d353deb41b2e94
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.28.self_attn.q_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.28.self_attn.q_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.28.self_attn.q_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.28.self_attn.q_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.28.self_attn.q_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2dc9fad9c9758a578e857957ef5183d578aa5398
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.28.self_attn.q_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:18ce9890232e70fb8be055b87abc55a4c915bcdb4022ecbb84527b30dad288ea
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.28.self_attn.q_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.28.self_attn.q_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a275ba712b095d0b4825a6d94368c8fb56c78243
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.28.self_attn.q_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8a16d7502e9c7819fb2ed8a46490e219ad1fd6a1db5f0d6d21b6751f98a2a57f
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.28.self_attn.q_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.28.self_attn.q_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8c0d504b9b51a33dfe4d6427636476a38e2ab67f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.28.self_attn.q_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8b2b9ed2ba0eafe9c611c213f3836b64056efb6fe5901b80bd99218acd13ab30
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.28.self_attn.q_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.28.self_attn.q_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.28.self_attn.q_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.28.self_attn.v_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.28.self_attn.v_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7364bb9a10cd6ac29bf0ee76515748c1b331a227
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.28.self_attn.v_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cb964b38923c5bf2667803d96f64563305a722a55ea6a3113496a40802c99a5f
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.28.self_attn.v_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.28.self_attn.v_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fdec024d15bf4a52351ae3eb56301097a51e15b7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.28.self_attn.v_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a5684e06b170a880f58a99cb7bc185e559e66ab38379459086c4008df1d623a4
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.28.self_attn.v_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.28.self_attn.v_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..247ec9b06f8d1e0e70e4b79b913d4103f17d5172
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.28.self_attn.v_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4d41acde48014fe343aecf73604bd92d1b5b3521a76babd1502726e2e0f22db
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.28.self_attn.v_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.28.self_attn.v_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.28.self_attn.v_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.28.self_attn.v_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.28.self_attn.v_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6d36964b14910c91dd2f1d8fe4e9e05c26f85e35
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.28.self_attn.v_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f9c15a6d90caed1bcd420f44d2769ebea3e5367e693905ef3ad2b867630465e7
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.28.self_attn.v_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.28.self_attn.v_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..88fe63a9cf968ae58c486cd94e2c0bb9818c6958
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.28.self_attn.v_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f19f42d1315284db19abc94cfc987afcb9620f76274bf827ca67a847b327a885
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.28.self_attn.v_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.28.self_attn.v_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9f3a77512351076524493e5ab1387a683f464a02
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.28.self_attn.v_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b3f2ada6d6fe59bcfc8889a0622543f7f80f1f3a77367b7fc123bcdbebcbb91b
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.28.self_attn.v_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.28.self_attn.v_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.28.self_attn.v_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.28.self_attn.v_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.28.self_attn.v_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7fe1a4e28f0dba15c91927eecb64e77de3fac25e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.28.self_attn.v_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f0ea8c685455c281632b92a6a7d262b21bf0b36724d842e539a86bf21fee9d82
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.28.self_attn.v_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.28.self_attn.v_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d4af5b3ec2b5cec29b001838ec990e19bc15772a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.28.self_attn.v_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cdf2236f820477f0b0ede2aa2ba40d6ce5607d2152bf4f8ef7f2da66cf0c0de5
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.28.self_attn.v_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.28.self_attn.v_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..746c669bc7a34294da5d9818a35bc6cc831c81b9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.28.self_attn.v_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b2925a4f46801fa204d154f4795a045e4f373122bfdb9b7f80329e7cd93e8d17
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.28.self_attn.v_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.28.self_attn.v_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.28.self_attn.v_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.29.down_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.29.down_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5204580dde0ebc575b9c83cb829e2c2990fbafbf
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.29.down_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:99c76f12a542dc0c7d194d4fbd700260aa4844bb1f48f1709d50d9ca8fa5fd39
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.29.down_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.29.down_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..32d61225e7282da8fe6a367e87791fc8577d8e72
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.29.down_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:44284d53d9ec42651eecf8b3e6aaaa3b10fa9b7c723089dbff2e5795c4e0ea67
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.29.down_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.29.down_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f019e4079c196f066e0cf9bfb6f020f1842496c1
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.29.down_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4ffe92854b29c0c61679758080ea29416356748aef59492af8577bd61e7da1cf
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.29.down_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.29.down_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.29.down_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.29.gate_up_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.29.gate_up_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..eec3d04fd05d4ab21a2001df8f7d3b7f9ec0ef89
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.29.gate_up_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a152ab32c7bd51785f8bfd6695ac54b862087c72ffb2dc81c0850d0e766bd43e
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.29.gate_up_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.29.gate_up_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b02cf0c4a9bdeb281264c190b4fed4ee5ba04667
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.29.gate_up_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:03f4749c206c20aca31177634c46dc482f644598056b81183a2f72d152acbc25
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.29.gate_up_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.29.gate_up_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8b085ed1eee4c8713b40f3c22face4ae2e0f4e7c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.29.gate_up_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:99f4bfeeace37a6306511b5683d814294e0f851ceed5824ce91c0c3f86f8ab7c
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.29.gate_up_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.29.gate_up_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.29.gate_up_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.29.input_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.29.input_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..82e5d0c4f6385cc70e0c1d614db4ee278e069c4b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.29.input_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f71c22ab6e339bb4670ff56bcb4144e2730e504c8bb9de3bd4fd19191a84c352
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.29.input_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.29.input_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..227a36fd5d66d252a4fb3f9af2e0e7a3a0706625
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.29.input_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:592a776079da1a25e90b9fcad55bf89535a5deed4e1fe2c045928f7ad5754be7
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.29.input_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.29.input_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8ee1f9f87554ecfa784ace43a931b75aa21d2dea
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.29.input_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:873a070890c283084d795b1bf7b34d9fbf427796e6d16bca5d9a786bdeffa80b
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.29.input_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.29.input_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.29.input_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.29.input_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.29.input_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ca08692918f8e6779492ea6a5b363cb6d565dc70
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.29.input_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd794355983f60b70c383a99d174ecdbcc6d36544323fd998a2ab7290683b7d2
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.29.input_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.29.input_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8fdc02786b75554258f08c67071fa00ac9fbff57
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.29.input_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7188216fc89444bd11df2ba28ab82c17ae948b832c8557e9774313aebfcecc4c
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.29.input_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.29.input_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..20767b64c5b0e4d040ad34d944d49cb3a4f73f5b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.29.input_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:43bdeb91de9820c1bdc2b0164e6c516e61c721edde3285f1d32e8e050596ccf0
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.29.input_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.29.input_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.29.input_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.29.mlp.down_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.29.mlp.down_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1d01fa9542a4165fc019f85899257b6e270f1d0a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.29.mlp.down_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2aaaf5c7cdfbdf12e060680121845c09d96f8675d122e4e789c9048b63578895
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.29.mlp.down_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.29.mlp.down_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e0e7396321711a7055d9f5004fcae30bf4b3cf4b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.29.mlp.down_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a9fc2a344d0ed49bb93d888c98873a971c9b3d59fb2f4d74bc7b58c68deb7760
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.29.mlp.down_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.29.mlp.down_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d59903909513fcfd052df4736b506af3c3641936
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.29.mlp.down_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4460e74dd820ff5895b078ee8883035ee9c9e7e898d6bc3c7b29f0166e0b417a
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.29.mlp.down_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.29.mlp.down_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.29.mlp.down_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.29.mlp.gate_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.29.mlp.gate_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2eddb84a467a70436e73b714bd3619b9d994af00
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.29.mlp.gate_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fb5d7403c56ef5fd9e25394c01f486f2ed70a1629a582ea51c029ab500c84ff7
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.29.mlp.gate_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.29.mlp.gate_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..78786bf3cb5ea84d758e0e75589053c3099c4476
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.29.mlp.gate_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:572e1367321b96ab4ca9cf3e8a7e68f00e14c2a153757b466cb2c2bdbc17b81d
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.29.mlp.gate_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.29.mlp.gate_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..51e58f3336941a32bae37280c5f350b02344e047
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.29.mlp.gate_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0d02b9c11d6d29601fc0c69770db7d64f141003724b78ddd5b56f555dec02b65
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.29.mlp.gate_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.29.mlp.gate_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.29.mlp.gate_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.29.mlp.up_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.29.mlp.up_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..68283a9cebffdacb14c9f4b8be33acabe3cfb2d3
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.29.mlp.up_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0f231871f352e0f8e8bdcc57d92bb570ff6c0123f0f0ed93bc9e2a4d13ac9c0a
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.29.mlp.up_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.29.mlp.up_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5888d1c847db6af1a12f3f0526ba8df6a9f1ad56
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.29.mlp.up_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6a64bb53269f31b89425636d1867898a7e2cfc76e28d7dd223c4e4fa778263a1
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.29.mlp.up_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.29.mlp.up_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..978d6a62a00b7d881c510cf0dc9601f550809225
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.29.mlp.up_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fb665981b144d53865f1f85548a2e8ad035ab69dcf21b2d45273259ef3695637
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.29.mlp.up_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.29.mlp.up_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.29.mlp.up_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.29.post_attention_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.29.post_attention_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..272268c588480a067cc245ff8c03cc5b3b5a0af1
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.29.post_attention_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0d631739fff0029a87840f84a80a92e0a806474350230bcf792e83f5bf57a929
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.29.post_attention_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.29.post_attention_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0d4d8d9d4f7a7b83a25d1e861b857853cce77500
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.29.post_attention_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:00ebd4960bdee8d97a29236648ceb2b8e1df190b9e8f98c2536e10eb79984620
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.29.post_attention_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.29.post_attention_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..45f508c2fd7e00e407dbca7d27baf18724810cfb
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.29.post_attention_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c3147507e8a4aeb92a8dca9a461722f1fccccdde6ee0cd2264c656885831708f
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.29.post_attention_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.29.post_attention_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.29.post_attention_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.29.post_attention_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.29.post_attention_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..909e0bc29120d2149f2497f4afe839dbcbcf5917
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.29.post_attention_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:92a6a02d873b73f4c4b8897e8c94b818b6c3e1b821f129003f9f78180fb1211f
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.29.post_attention_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.29.post_attention_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..98ed4238a62047b487b934199798dcc3bf226411
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.29.post_attention_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:451fe7dfe18033f543ea22f564a4fb3f5464c68e73f1ad642777417f1d57d43b
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.29.post_attention_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.29.post_attention_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8fdce8c255ee6cca578810b0f690566f5f892832
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.29.post_attention_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fdd530e7db3d31c8942575dce37bb4585dadd0b946bf77e80a1a3640bf45ac79
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.29.post_attention_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.29.post_attention_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.29.post_attention_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.29.self_attn.k_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.29.self_attn.k_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8b8c701dc75ff1b09965249d3703e5a704eb6adf
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.29.self_attn.k_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b534b924e3a1eda75290737e549437ed9b48bfcf5cc949fb8ad8d4eff0b64d55
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.29.self_attn.k_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.29.self_attn.k_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a2a58eb6c34890fc222b699f92ef7cc93f9f5ea9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.29.self_attn.k_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a343bf04940db37057df546c470ebe635d8f4aa33e264028fc09cb8d8bc01328
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.29.self_attn.k_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.29.self_attn.k_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6399f3db2795165c5cea49f2fd8029e458dae20e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.29.self_attn.k_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bf8bff4831cd6aebde9fa96db36aa056a96c53863f45952aa69d12b6a8559492
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.29.self_attn.k_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.29.self_attn.k_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.29.self_attn.k_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.29.self_attn.k_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.29.self_attn.k_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b75aae70e4e82df637fa6f0ca55c0195cf4378c9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.29.self_attn.k_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5104e8bfe9924cc94a419944b59b8a857f423d74b3695d82630513682d683adc
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.29.self_attn.k_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.29.self_attn.k_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..338be78725ba7fc51f0a5cf064a32590ebfd8059
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.29.self_attn.k_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:23e75abdfb54ed33f59cf330fc686e71edcd8386b96fb8fce5e07457ae16dd39
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.29.self_attn.k_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.29.self_attn.k_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cc32fedc3d1f090d9f8721577036e679908c79be
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.29.self_attn.k_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f289b6d9f24ba36a36ded871a30fd47ea5a70c0937b2e81d21157beee986b06c
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.29.self_attn.k_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.29.self_attn.k_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.29.self_attn.k_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.29.self_attn.k_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.29.self_attn.k_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6d265ee69113fa5606c7a50e0aacbc01e6d7b331
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.29.self_attn.k_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:718b3d4545305e378606fd84ddd7f29e396c058eb5717b6d6352f9f0f10cfef4
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.29.self_attn.k_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.29.self_attn.k_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6a78ea75cf4ddeb53b52e2a025a73d7d005093e9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.29.self_attn.k_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f54907e11ee913063624904c723ca4bc55542505e546c1b72842b67b50c38523
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.29.self_attn.k_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.29.self_attn.k_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..462e791b4498b6e9c68a18fca61593f32a35ae31
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.29.self_attn.k_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:45ce9413d63374852af398732186ddd1fc5a66338c55e9a62f3a4adfa65ef7ed
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.29.self_attn.k_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.29.self_attn.k_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.29.self_attn.k_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.29.self_attn.o_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.29.self_attn.o_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..64d399bbc5d42d15e8d296826c96bc760099dd6b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.29.self_attn.o_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c1d68a2efab3b337f7f844c41d9021c4a6ac03628d529e411490fc8aa44340c7
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.29.self_attn.o_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.29.self_attn.o_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8e4615ececb7f0190c1c7f11d6f1ce4b199960a1
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.29.self_attn.o_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9f9f7ca77662bf7c4b1a354e580a1a6e168606f4ce604b8d0d1442a1cb1fc40c
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.29.self_attn.o_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.29.self_attn.o_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a296214c31c1dbf25527ec7f1dcee8b830370215
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.29.self_attn.o_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c30dc91a1544285c4fa0b5629f051d8950c36aa1f738c29daa0cf33e96edb46b
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.29.self_attn.o_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.29.self_attn.o_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.29.self_attn.o_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.29.self_attn.o_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.29.self_attn.o_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e7261540156c4cb237c1c6aa605328a78b87d9e3
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.29.self_attn.o_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:11171dc787b9255ab94b6d31446119de92b17bc203d9c53fd8e955a33ff2f261
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.29.self_attn.o_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.29.self_attn.o_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2351e23a6e47b37c03508bc556817f32fe91dc09
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.29.self_attn.o_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:32204c82a84eae3b53ea10e786e63c14e4d7f8f51852713a51aaf3d7c9c4dc3f
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.29.self_attn.o_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.29.self_attn.o_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..219a43aca8190fd237df69a3477307f7fa64c450
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.29.self_attn.o_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a8cb1a75621be9f451743c559479276ac183923c7dc987fe7bfa6ae4e7679eea
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.29.self_attn.o_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.29.self_attn.o_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.29.self_attn.o_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.29.self_attn.q_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.29.self_attn.q_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7cc1f6ec2eadc1f92b0e2596d0df5a3d4bb39762
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.29.self_attn.q_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:62edb68431875b1c24057f61f7da6c26a322f769de2c3199c09504b999a246d6
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.29.self_attn.q_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.29.self_attn.q_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..bf31ec93c612611d6ee1bbcc94c1530b1611c8dc
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.29.self_attn.q_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8c440531bece268d056b4d0d5feaa89ed47562f97b654c080921d75829046b5c
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.29.self_attn.q_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.29.self_attn.q_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3250ead8fa81a1771af05db81de9ee704582e6b0
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.29.self_attn.q_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a913bc0ff354b224f0f486d60dc81ab3b24c9149c4e14ac3017e57af319a61ce
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.29.self_attn.q_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.29.self_attn.q_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.29.self_attn.q_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.29.self_attn.q_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.29.self_attn.q_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a223abe2f33f84a03d1d26ad39fec605c1734b38
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.29.self_attn.q_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c4a239f134d5c717e77cb0b51d39b9032710665ad1bd227038d55fc41d053364
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.29.self_attn.q_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.29.self_attn.q_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e2a2361f1085cc38a8524df3876784599fc286d9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.29.self_attn.q_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:daee07a7d9812bf654daba695217362bb66e59ec976da5d5c4e8118fcbe3ac21
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.29.self_attn.q_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.29.self_attn.q_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..877e98043f80b24a754ae8ad72f9d1b0355f017b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.29.self_attn.q_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:372f5615c62c847bd03b2122fe5ef9bff233e98f17925807e895c8f171c91443
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.29.self_attn.q_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.29.self_attn.q_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.29.self_attn.q_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.29.self_attn.q_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.29.self_attn.q_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d9d0b42b89e8a6845532a99658868c6c8ef69867
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.29.self_attn.q_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:196e86321bd5802656984fd2d50d82c75f07c2421d3f487a859936328b5a4af4
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.29.self_attn.q_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.29.self_attn.q_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7512badcf8294eb6b4437c78111c1d2cf97029d2
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.29.self_attn.q_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3e4a07f8c955757def59f530493cd747afce3b2c410f41cb5ed0e673e36adfc7
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.29.self_attn.q_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.29.self_attn.q_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a4f7d2e86ac7c6b8eb836ea78375227ac890f42a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.29.self_attn.q_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d6f593b934069e940154fb6910731088772f6e049e2073282745eb6cc5dde81c
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.29.self_attn.q_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.29.self_attn.q_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.29.self_attn.q_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.29.self_attn.v_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.29.self_attn.v_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fe5341657a57db7aec1c3ef3519f1e33bf0e25b0
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.29.self_attn.v_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4e3abdfdb25c58299e4668ecb617bfb285177e5301ba4640c3cb8e35162a017b
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.29.self_attn.v_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.29.self_attn.v_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e708b4f4e91d1edee4452550aea13b51cf62ad12
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.29.self_attn.v_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0862bae5aae754f5f27b13e4dd439094cf3386c273b55410e8942ed81c299bef
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.29.self_attn.v_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.29.self_attn.v_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d07eb5e3638d5c5b6fcf435f8ef81ba0e9360762
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.29.self_attn.v_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:14353ecc96d5f9794d4cbc4f5f8c7ae2d8175d4e3bafe80be2585f30f4907256
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.29.self_attn.v_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.29.self_attn.v_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.29.self_attn.v_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.29.self_attn.v_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.29.self_attn.v_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..58614f8871ebd3701b923d33ee7f5151b84245b5
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.29.self_attn.v_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e6f87fc773f87d96378bafe495a680aed22cd94b7a54c76109292c29bc4bb7a7
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.29.self_attn.v_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.29.self_attn.v_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..982e1aea44870fec890875cd71815f5e1a047ca6
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.29.self_attn.v_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7d2a826822fcf61513373f15d029a9f735b2ad3f1d898de20411066714041a02
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.29.self_attn.v_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.29.self_attn.v_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7949758bb31952338c56270f7b2f887e3eef7b10
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.29.self_attn.v_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:82698c936d058b70b03592966410b3acac816792e0f3f4009993f80fb9a9aafa
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.29.self_attn.v_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.29.self_attn.v_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.29.self_attn.v_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.29.self_attn.v_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.29.self_attn.v_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..45e55069ee3ae6348d3a6bb5360566d47d13e9fa
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.29.self_attn.v_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:93efc5ab2985f645a59dfe6a899934667b9727989599394925b8118a4f4ce704
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.29.self_attn.v_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.29.self_attn.v_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d56919316ce402837619b312b6dcc35be8abc566
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.29.self_attn.v_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9b60894de33e400cbdd91ba81b13e285ff82068a6baf1e178fce8d22125ce2db
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.29.self_attn.v_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.29.self_attn.v_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2c9c8ed64227da29e6535c8e7e0d0111721da233
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.29.self_attn.v_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:33a1d84c64505fad0315cda30137c23fd52c5dbd0b0793a308917795b85c1a97
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.29.self_attn.v_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.29.self_attn.v_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.29.self_attn.v_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.3.down_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.3.down_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..88c4feccfc3dc57174912cf3b350d93a6788fc85
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.3.down_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bbbc88e7114248bcfe397eaaec74d84f52a44451edd80a7ec27669eee8e0f2b7
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.3.down_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.3.down_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1376469c9196be05fddde2d40069263f75bb9afb
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.3.down_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1211cb68af906a8d635e0738e09e656f77feeab6f3d55d87bbfc69dfc53893b3
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.3.down_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.3.down_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fce23a19359a91592dd8cd288b965bb799f2fdc4
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.3.down_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fb2cca01ea97cf723ead835f1e6f9f82c56c754b961e34ce8712569e366800ce
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.3.down_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.3.down_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.3.down_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.3.gate_up_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.3.gate_up_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0dac05b6399cd6b7e8ab4b0117b57ee726ee8c0a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.3.gate_up_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:21158037411b9df69f73493d7f902472132b2ba60c097962f15d937bc9f031df
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.3.gate_up_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.3.gate_up_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2f824a69c5909e44ebe99f237c5de926e7037de3
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.3.gate_up_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:64181929d2c8cde32b7684896a3e960e485225b1c5d14861b5717bb25a3d66ce
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.3.gate_up_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.3.gate_up_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1c28559bec38bf99f0ccde285430b2bea53ce479
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.3.gate_up_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b5efa0e3bc7662f11fa6eaf8d6f53dd3f45a5080d5049bb1ce4bf479ab55b996
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.3.gate_up_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.3.gate_up_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.3.gate_up_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.3.input_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.3.input_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..dc95473ec8c81a88d5b544a064d16b87766ac51a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.3.input_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ac7597d48d7c244a7af97e981cc09649813768a2a5a042d7879f2ccb708ee790
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.3.input_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.3.input_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..927a0195441fb47b75b7342afaa6bb1a22b12c03
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.3.input_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:31231c7d0df755a7d4293e5409e7a4626dc8e2b7252a71708a52996a061e2be3
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.3.input_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.3.input_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..92884e21d3bf43241893e6523f3c674e2de64bfc
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.3.input_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9b7fb131bf5800e380de01afc301f3b2949d6dd3ee813a8315785553ab459fb9
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.3.input_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.3.input_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.3.input_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.3.input_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.3.input_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3b8b9e1caa127c410de9096cbf65f46a2b9d9a54
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.3.input_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f86e5f3bab3e9ad71f3a2d0ae6de2e29fd1c7c2d522bfbdb6fc60cfe401dc573
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.3.input_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.3.input_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ee677b9f759179492c36d0d3433ce82098da80ce
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.3.input_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e7328b7eb70d10f391a8ba005b7fca3ba2b7d873faeef5a3032615c436f5f00c
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.3.input_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.3.input_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..03db650a58f7acc44ebe452565201c9be58238fb
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.3.input_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4d789b61b8213de04c971b85fdd2e7749de857d66e224d6d066cd92fd609a52
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.3.input_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.3.input_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.3.input_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.3.mlp.down_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.3.mlp.down_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1995da560e0ffed637b9730ab2c7b1a1bbc02f44
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.3.mlp.down_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8cc287149d3fcef0ad598623ffdf3bc09ac2c501e05aa524e0bb29770f305152
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.3.mlp.down_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.3.mlp.down_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6105fb59d4c33822572c90ff35faa63626b7a94a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.3.mlp.down_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:53cf7627ce17feeb28f3ca7778dfb7b0ebcf089d25ca6b7295a417d9e16b313b
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.3.mlp.down_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.3.mlp.down_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9dec4c820a59c197f5e08da5fcf1fa0a5b7c7763
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.3.mlp.down_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a47d0ee3057366f3ad17526eb5a5b7e7d7cc959db51c65a8353b412696ad8d3c
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.3.mlp.down_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.3.mlp.down_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.3.mlp.down_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.3.mlp.gate_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.3.mlp.gate_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c420372c298342989c312cab27dc7fea7f618602
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.3.mlp.gate_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:740f91763e41ca5e326542781b662abe288fd0d9a166733dea58218c5b5070a7
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.3.mlp.gate_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.3.mlp.gate_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a93efc1a5f8bbdcc1331afc02afa95c32a82e8d3
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.3.mlp.gate_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:025214f399c7d0b2afc99518ce148becd5c21e8ee2840c3a77fef95090b5ba70
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.3.mlp.gate_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.3.mlp.gate_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a41fcf6dfa760f2995deedfb034510ce9e440d74
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.3.mlp.gate_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:13f3e639bc712e04044fb84d1417791b8e8aa187ba690ffb1b055dcab51c5aaa
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.3.mlp.gate_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.3.mlp.gate_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.3.mlp.gate_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.3.mlp.up_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.3.mlp.up_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..367804710fd9ceb37dab7f82843e4414f8c2674f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.3.mlp.up_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e17c9b6244b570db83f6a6dfa7066b1fdd9d22cada1f5052e228f5cf8122a03f
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.3.mlp.up_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.3.mlp.up_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..065554a4a2f98bec8c4ecc9400aae08685c57119
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.3.mlp.up_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:354783344a87d3f8c927e4f9dec3d81c28ff8b27e0ea615e7acf6e88f444bd11
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.3.mlp.up_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.3.mlp.up_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fbab7c21454f5e46a170c93659413bc782288d5d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.3.mlp.up_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:36369b8bb127106465bba179400f5527a755b6cd39971cabb09780253760ea2d
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.3.mlp.up_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.3.mlp.up_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.3.mlp.up_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.3.post_attention_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.3.post_attention_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7cdfda20eb539ffb4694eed2cadba36f91cb01c3
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.3.post_attention_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b92a3b56c9c3768264cc94875456dda7971831a3d51d48ba7f9f78587d7b0588
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.3.post_attention_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.3.post_attention_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a783803188c7f42c45070f9ad3a39f7ea96f72e4
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.3.post_attention_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:44c15d92a11b5098466b3316b4a4770328c486e11868e75feafefa446a623562
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.3.post_attention_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.3.post_attention_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c751b37f066d81046f399e5f3aa02c7c4e2ab125
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.3.post_attention_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6d2eaf51b558de028c1d79e6d3648871e3f1fbf6e5d527b6078eb3df6d8d96e9
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.3.post_attention_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.3.post_attention_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.3.post_attention_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.3.post_attention_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.3.post_attention_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a03fe74e84cff11361e1fe1bdd31ed61b28350b3
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.3.post_attention_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:91dcb77a338f73eb33fe3240597595d4fb886efc2ba84584c9a28ee23ae71b49
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.3.post_attention_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.3.post_attention_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..26ecbb0d2daca068f01a6c28a12a6fbc0c725e71
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.3.post_attention_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:13aead531984d6208a2b75cece05e773fdc16443c5aed25310a36e507ff623d8
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.3.post_attention_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.3.post_attention_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1bb67c44171af95a5b495c44330057ffe2bf946c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.3.post_attention_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be7470d5f558c8d4e624c67e1d599b7dc70588e8641d2ebc9cfe6f4d90a210d7
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.3.post_attention_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.3.post_attention_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.3.post_attention_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.3.self_attn.k_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.3.self_attn.k_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..06a3200feb03fb9d1e29ef826a401153d780701d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.3.self_attn.k_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:df183974b3454ff17e0411d1d77e4c162a9e7b2e3c66b4be3d6ddd8fe0b84ed2
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.3.self_attn.k_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.3.self_attn.k_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..beced7bb6df818ffa6704beeb79cb0d219115f00
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.3.self_attn.k_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:caa4477ef78deb183787576b86e4e0b71f4f374175759d9538f7b701d77b0d1b
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.3.self_attn.k_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.3.self_attn.k_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..009b0da760ceba4c3d49fa2398ab57d6f5e79cc7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.3.self_attn.k_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:70fc6e138a67e52777fb6829e5326033eb99a1c3cde634b6d933223c3ce82c7f
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.3.self_attn.k_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.3.self_attn.k_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.3.self_attn.k_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.3.self_attn.k_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.3.self_attn.k_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7b71166fb7c68848d6c8ffd50a9c256c1a933c9a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.3.self_attn.k_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cb20a80198a281254e899b15b9278dba0101e2327652773eb2664c4831608741
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.3.self_attn.k_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.3.self_attn.k_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f22e0f453e7c6d66f7b5f61109d0c6f9d1204f69
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.3.self_attn.k_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6a550ec26fb094eb023819117d4415ad6ce17f1d310b95b79fd2112a87d98541
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.3.self_attn.k_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.3.self_attn.k_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fded7b5d87e6bcf5bfe9205d8e922d23dea6412e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.3.self_attn.k_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cf8d460cf41bde540dbb0a7b52b72041d4ec9c312a301b5a45f92f576cede5bc
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.3.self_attn.k_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.3.self_attn.k_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.3.self_attn.k_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.3.self_attn.k_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.3.self_attn.k_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4315cfec3449dae0b80be40835832e816ebb4da4
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.3.self_attn.k_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:445e12b1e5d52a62ca0fb35fd7c391b162341c97add8689847786d5bfe83f43f
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.3.self_attn.k_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.3.self_attn.k_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d85ba650cebb53d698d2a83507f877f4d799b50f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.3.self_attn.k_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:71c406ba47a674b7a6e9dfbc9d1c5a8d9df7c8c321a58c9659d94883235e4d08
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.3.self_attn.k_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.3.self_attn.k_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a5f8a6af23d737a4c142ff6fa2ecca4b93133407
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.3.self_attn.k_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f94f45a0b068a4ced241d38fcc17437f61e72d9c258013ea9db97dc539341583
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.3.self_attn.k_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.3.self_attn.k_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.3.self_attn.k_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.3.self_attn.o_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.3.self_attn.o_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f32175edb7e0664951ea4743f74f9e1303766572
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.3.self_attn.o_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d7066a00d285853862e925ff3d7d962e1b073d27bc3e434522abd854f89d8791
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.3.self_attn.o_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.3.self_attn.o_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..99ed0fa5940ab402fb15c3edf4943d57e667dcec
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.3.self_attn.o_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:40c9947e06c62ee43ad1b7ecc1b6f440ed78d55c5ae24ae6653d9e717d1ab46a
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.3.self_attn.o_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.3.self_attn.o_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..501231babb8203ff88b82c360be9887bb0f36170
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.3.self_attn.o_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:95e4495c0a13520b324f020627cb241c5768b56c1ef084e6df3427249a2dd74e
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.3.self_attn.o_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.3.self_attn.o_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.3.self_attn.o_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.3.self_attn.o_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.3.self_attn.o_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..28bdeb643047c5c801c376e0d4df500a57554ca6
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.3.self_attn.o_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d7aaea569d277360c3abe3c8117ceec2120bf10be3f9539c141845afbd20de8f
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.3.self_attn.o_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.3.self_attn.o_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0f1bfc3e33afaabe7fd770468c550c64aa02204a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.3.self_attn.o_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c098bee33971be8aabaf0f398cb20e95d7daba814c017e371ff7c086249cec26
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.3.self_attn.o_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.3.self_attn.o_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ff6380b0fbe91392feb3fd35d7b8ac2d64a9eb50
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.3.self_attn.o_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4078d8743030d6f233877bc6f04807f6f11d1adfc68284d8df95aa8ce925096d
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.3.self_attn.o_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.3.self_attn.o_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.3.self_attn.o_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.3.self_attn.q_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.3.self_attn.q_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5fc21a4aea32d14bcdd3fef52d7092033991e8db
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.3.self_attn.q_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:70b32d72e2779cc7541ba5a7832c40b3272f10e707a35f20fe361733a1f592fe
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.3.self_attn.q_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.3.self_attn.q_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1393ff0208ca0a75387f025bcfc76d191471f311
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.3.self_attn.q_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:608abaf114fd7aad8f4d02e8030b77b42de10440259931472f225f24086a95b9
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.3.self_attn.q_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.3.self_attn.q_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fe79e8d2b3958babd3e68b211ee91fb61ee7f1b5
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.3.self_attn.q_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a8803b109f2b2e7d781f7f421d8782c467b26f4472ee315d5428c02e1afd2257
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.3.self_attn.q_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.3.self_attn.q_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.3.self_attn.q_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.3.self_attn.q_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.3.self_attn.q_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8d18ddc7f72e85d4eb083c14434f55f0e5acc828
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.3.self_attn.q_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bbcbf93f6ac15de18ad23a875fd61b4565a96f323db60f5e424a5f9cbc4789ae
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.3.self_attn.q_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.3.self_attn.q_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..74aee287d369d2da0777b98fb16313694686e237
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.3.self_attn.q_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e7f663ab52241671ff2d75c53e4ff5ec812421ab645408789d2b14ce271c8365
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.3.self_attn.q_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.3.self_attn.q_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e3c42289abefb84c48c20e8eb0f1e9557a913c01
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.3.self_attn.q_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:57eb8c84997c8030f6abed59cd9d1f8d6e40d2f69c4cdd1c71f0455a97cd5b09
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.3.self_attn.q_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.3.self_attn.q_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.3.self_attn.q_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.3.self_attn.q_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.3.self_attn.q_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0786acd1d4d7931f640b23a8b6e21f3fbaeb753f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.3.self_attn.q_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d4e07e30f1f171f79c81c58a7cdc8b3a5b5b97a9cfa968be7b6e7f88faf59561
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.3.self_attn.q_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.3.self_attn.q_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f02f55ac25c2d19b0162d5102f6100d87ee7e9a5
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.3.self_attn.q_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d230a60789380f01fe29bce602070e4502b90d7aba74e2366f95b6ef3e1bae8d
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.3.self_attn.q_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.3.self_attn.q_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..806e375dd753bfad75495eae254e052ba800697e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.3.self_attn.q_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b20c7432afd18bce0a67b883d88d613f68d8c2a2d946517eca85a3d2f8f7a3bb
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.3.self_attn.q_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.3.self_attn.q_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.3.self_attn.q_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.3.self_attn.v_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.3.self_attn.v_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3870ed5e1be36de9ff695c0cb50f75376da609ba
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.3.self_attn.v_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4171cd67c0a55bd0e2d63d8c37e07ac4bebea90c288a9de1286594fefc92bbbb
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.3.self_attn.v_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.3.self_attn.v_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3f219b19ba5678ce84828ab6639709b572e622de
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.3.self_attn.v_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1517000de6acd00880140f1a71bf994b591f29904a44e53ae6a42eee6f416a58
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.3.self_attn.v_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.3.self_attn.v_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d40d5573af2903371e39ac1b835c0950cf369868
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.3.self_attn.v_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:70b33a5a03f526121a68bb78341f96b476f0ccb71591c6934a44300be0905273
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.3.self_attn.v_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.3.self_attn.v_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.3.self_attn.v_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.3.self_attn.v_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.3.self_attn.v_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4197071dcf1435c301e6e4ea08a0709f9a173cb7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.3.self_attn.v_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eee78a337880355315cb29c6e44f38cfdb96779ae695b607c52f1b6ae0447959
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.3.self_attn.v_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.3.self_attn.v_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..76a5499bc6d61bb14149ff09f3c7e509dec83111
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.3.self_attn.v_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6c1157e6eb382892686a7da42791215be96ac2f6cc0b5e887f10497581af08e7
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.3.self_attn.v_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.3.self_attn.v_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4b14f865fe10f14f58b3f40ccfd273957fec8224
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.3.self_attn.v_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d3c70472b3629f95caa3c2fda5f5b119125b01328a1fdc3a5374a30e1c8fd2c5
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.3.self_attn.v_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.3.self_attn.v_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.3.self_attn.v_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.3.self_attn.v_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.3.self_attn.v_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7fa3ded06d16477a3580f3f4c866f740dab39125
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.3.self_attn.v_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8cfc77e5e466c6160cd086d0c411b8eeb4bc516da4b2e605a060b158a328042f
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.3.self_attn.v_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.3.self_attn.v_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..706bc14b7b4e049d58d4cbf4bf03043d8b454fc8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.3.self_attn.v_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c6ec65d86df1415e1175381c843d9c8d56c9e6b6d2b5a496622cdcd88ec5aac4
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.3.self_attn.v_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.3.self_attn.v_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7af8a597ced3a2c497b4ddfa8914e740741d37eb
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.3.self_attn.v_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7d9fe58ed35a9f895b637a3b59d5e775eb81a812450fa6ff26b86e45c8a7869f
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.3.self_attn.v_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.3.self_attn.v_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.3.self_attn.v_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.30.down_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.30.down_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cac74734ea305d657c9adb3e843a3b9ebac1ba30
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.30.down_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e3d0519bc514a31414177ff8f7a5a45487ad661a089bb4c60acc93c623de12e7
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.30.down_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.30.down_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9cd433a55b822f7f617e5fda1e4e3af3685164a6
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.30.down_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:263551681cb5cef2e94daef46bd0a1f32c0b935fce89bb0fc9da8256fcfe8289
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.30.down_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.30.down_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4b934e07aa21d3d6d0463148d2c28c74839dd753
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.30.down_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:58e2861776d5ff29538d45cc64ffb51d3d382a33f16b5955ae2163ff5bed5230
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.30.down_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.30.down_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.30.down_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.30.gate_up_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.30.gate_up_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8fca69562fcfb89749a6c322950f391ec4342b1a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.30.gate_up_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5c42e93b6381150d209ddb0594c8f469dffa53df406789f366e86dfd9fdf146b
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.30.gate_up_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.30.gate_up_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a35e4b46eb36568e27853dcd07ceabaa816c4dc7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.30.gate_up_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eee5a7ea5282bdd193c44a827a8d505dc7c178a45b8fa133ddd90035e7b940a0
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.30.gate_up_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.30.gate_up_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c6633afbb1a5355d212076d1e6c54f0dd2242dc1
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.30.gate_up_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c7350b04e2e30437874c1474e5db176d45380f321d5ae6a84eb580f5c5931f40
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.30.gate_up_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.30.gate_up_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.30.gate_up_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.30.input_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.30.input_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..51b5e58c6044e829070a5e380d28f5decc5439d5
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.30.input_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:15a7c85f0a8d8654061af09873c4f05ad8415515f251a650da3c15ad37ac1bd9
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.30.input_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.30.input_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..425afb1a00e014d1bcc288546b78074eab6a9311
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.30.input_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fd1c336cf7c953a3874917d6f5e2b1496325bf419b0b70d9a3d6a46d9dea5a2e
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.30.input_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.30.input_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..26449d36c3841846d491da3ed6c988e9964017bb
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.30.input_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d6832a96468194b4b4ce1330a10996eb19f2aad5d5d10dab04d7432b09e4acfe
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.30.input_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.30.input_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.30.input_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.30.input_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.30.input_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5ae45c4a4359797c3ab71ac585ebf4e090adb80f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.30.input_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:af5ff8d0d356bad0d0642ac9570d4212c3dda0975d459b676aaa942c7718442e
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.30.input_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.30.input_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5bb0296f8e257d7b1f17c3c75d5e44235d5d5e40
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.30.input_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:781a783b27fcdb3f5b6560585d963b2d80c89c9220a2270acdfb3c2e81fbd038
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.30.input_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.30.input_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6d61341512b0cba62ee316fd99135b8cafd23645
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.30.input_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:261278e26f13b1190e3489a827fdeac0ffd4a853662df08d4804ee43083185ed
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.30.input_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.30.input_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.30.input_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.30.mlp.down_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.30.mlp.down_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c2fe16d40cc81136d2746cc11042414be4a67cad
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.30.mlp.down_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:16f7e264f9029566f8532ff74388a152d8ad2876d01b5d71e207eeb9b826b9e4
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.30.mlp.down_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.30.mlp.down_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..39745ea27a6dbb43a6d8bce810497bd69a2175df
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.30.mlp.down_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:94767348130016263d0e79f200e2340b4a0ceabbf092426dd8fcea2b3a3b899d
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.30.mlp.down_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.30.mlp.down_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ab0d48a6ca920ee9481b7cb8f5bd579f2ecee102
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.30.mlp.down_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:596670ed7a84af3cd55c2978e021f748e49cfff30ffe162e90097ba2b597dead
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.30.mlp.down_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.30.mlp.down_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.30.mlp.down_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.30.mlp.gate_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.30.mlp.gate_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..afab9ffd91d87abb06a4a21c97d713eb421b0ac7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.30.mlp.gate_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29afb819bffee8b0e311c764f3ccd95d11ee563fda6167d096cd468d66ddb043
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.30.mlp.gate_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.30.mlp.gate_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7a4cd8ffa10a2c55c3e0cce5e20d8ce328a69c78
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.30.mlp.gate_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:04abe42e636b62226d71ee2fa084f960fe3cbc5caf07075a3ced0d2b1e4d381a
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.30.mlp.gate_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.30.mlp.gate_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c1d41bbc9484e9b69fc2220b072bade002d9e868
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.30.mlp.gate_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5bd4e92d00d5a913b42ec54b6163ff133329b500f959878882515982715e5706
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.30.mlp.gate_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.30.mlp.gate_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.30.mlp.gate_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.30.mlp.up_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.30.mlp.up_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..093a1cca9395e680f127c8966ea90a9cb3f830f6
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.30.mlp.up_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e48b64f1a47f2bcaca37b828de525a220687e7f650a7b3063055bd11ba8417fa
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.30.mlp.up_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.30.mlp.up_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ddc04940bf33a7ce65ca355a67f5f216cebed8b8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.30.mlp.up_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:55319102da425d8408dc588a58c3d79ddad069e54f4b9f5b508da7b74f0f548a
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.30.mlp.up_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.30.mlp.up_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2cb38d530e0e7b104119f1af67508874e31a57b2
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.30.mlp.up_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e633f125a99e163f7958e413b91b33cd4c21df39cddfc455ccf4bb0d05f00
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.30.mlp.up_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.30.mlp.up_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.30.mlp.up_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.30.post_attention_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.30.post_attention_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9ab09d9f02169df2f0dbcea52b5bb88f72846e34
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.30.post_attention_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f9f37aacab4ab610d43132b843db324d6d0f0ea8b95d17d7b979d4f3aa992aa7
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.30.post_attention_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.30.post_attention_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..18de85fb14a85a78aa39d51f9c634eb3ff7fc7a3
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.30.post_attention_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1c999d5b64701592795d772751a6aa0a8d2db78a19bae8f7addb6074026dea4f
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.30.post_attention_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.30.post_attention_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4abdbe6aa00b3939003651e98905fac82fc3d02a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.30.post_attention_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2bf7e75791cd9774e41600272cfbabe6f8916157d192be8d032f9271d8ee7eef
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.30.post_attention_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.30.post_attention_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.30.post_attention_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.30.post_attention_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.30.post_attention_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8075ab5fc4e606eae975ee4851c06156c4009589
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.30.post_attention_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7e129d9ab7f5c678d279944984070cc4a3ea03cdc7362ac331fdca51d63fed23
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.30.post_attention_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.30.post_attention_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d6b423eddce5a4bab52731a0e1efc38641e1fc01
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.30.post_attention_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9cb983bdbc4f170ce1169dde50fa3095ca2df6b08f5779e846939257d0a2e4d6
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.30.post_attention_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.30.post_attention_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9990d100021f4202667d7c72c906d729386f81be
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.30.post_attention_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:89bb64c006ee43691f4b11389fb57e78c04f3776181c9dcba7be9c1ebd4365ec
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.30.post_attention_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.30.post_attention_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.30.post_attention_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.30.self_attn.k_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.30.self_attn.k_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b98816fb382d94558b9a6e723d4b5f98133a7fa4
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.30.self_attn.k_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9f2ef42024071da657a8d24de28c36b22de4789d5b4096cd41ae45bf9fc17566
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.30.self_attn.k_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.30.self_attn.k_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..848f38a4c10345d1d0b972acd6b62523df45c9d0
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.30.self_attn.k_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c2570d68cbe708ba8018f24887f034bd4cddec072110dd6a524817368b118b8a
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.30.self_attn.k_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.30.self_attn.k_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cdc7a637de483d92bbf55df949006e832924e677
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.30.self_attn.k_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a4c3f6efb098a07b958d801df10726179445bf5b9b62f858435c924de90d7ae7
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.30.self_attn.k_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.30.self_attn.k_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.30.self_attn.k_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.30.self_attn.k_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.30.self_attn.k_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3c757722d6b10a16dc45a0607bbdb9afaa9ccd25
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.30.self_attn.k_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:713bcf946f383ab461e6d94ae30462f18eade2620afb02bb9efae1a5f1974337
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.30.self_attn.k_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.30.self_attn.k_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..46c7e12287bf0161bde9b001d41755513839d227
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.30.self_attn.k_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8bd633c3104b91f301d719437826ed87927f81ea037c5195d3c2c00c30f7c29e
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.30.self_attn.k_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.30.self_attn.k_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c20896a2c769f784bab17c1af9d0fa9e6379a8c1
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.30.self_attn.k_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fb5fb65d98ff53fe45cb004c9fe197ace844a31fdc5dc512a6c636e84d0e4321
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.30.self_attn.k_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.30.self_attn.k_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.30.self_attn.k_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.30.self_attn.k_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.30.self_attn.k_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3703f1cb0fbf57871c9f332776af4737d607b83c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.30.self_attn.k_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:75c04296a972911ff336b04fdf16d769b4c45689df40f1716192049e61d03470
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.30.self_attn.k_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.30.self_attn.k_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3ddbf8054ac5647ee543481ad5a3698749e04042
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.30.self_attn.k_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ae9395733d741d0a20ba0a1efb55d8090c3c67a46cc56d26d76e2ff98268c0ce
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.30.self_attn.k_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.30.self_attn.k_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1222540d2f872c720b1c30732308bdf5b355d347
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.30.self_attn.k_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a4661b156d719756a8aebc82b6d20f92128eda98c23d48cc0c2efa7f1a2e283
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.30.self_attn.k_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.30.self_attn.k_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.30.self_attn.k_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.30.self_attn.o_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.30.self_attn.o_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..13c5b58c808c0fe31b3cd783c59a81a60738ef01
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.30.self_attn.o_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:872a49c022bc4519775aaa8cc6f3861d656e25eb336b530f33b677835541fa9c
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.30.self_attn.o_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.30.self_attn.o_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1e0a3f1da58f1ebc559f260320b1e23846112a8f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.30.self_attn.o_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1624d2bf87815d661470db81d8965c6baf169669e8892e5de04dad2c5be9447c
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.30.self_attn.o_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.30.self_attn.o_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f6b3bb1e1fb0270104ca45642972aa91076ff345
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.30.self_attn.o_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cf37a4a69196d9b1b34861c90466d674f928b5dc1eedeb1dddb95ed06530277c
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.30.self_attn.o_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.30.self_attn.o_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.30.self_attn.o_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.30.self_attn.o_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.30.self_attn.o_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b486c0185dabea7d4600bdcc5e1c6c62b56f414a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.30.self_attn.o_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8cb4c5d84f49020b0613f91b75e846618b052e3f514fc240591c10d8b0e86a8b
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.30.self_attn.o_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.30.self_attn.o_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8acd145f82b20022cf5952269907697230950fa7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.30.self_attn.o_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:91fde71cb17afe07719d59d2049c151421753fd5333e49a0fd62411b1bfecdca
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.30.self_attn.o_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.30.self_attn.o_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1210d0381437e4a264411b2da7428d8290cd99c7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.30.self_attn.o_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6f15df34d620af64f3d436dcebd4cddb5a5b524613692d9ecf7bb721d5356079
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.30.self_attn.o_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.30.self_attn.o_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.30.self_attn.o_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.30.self_attn.q_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.30.self_attn.q_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a00386c95a60b11ac8ab17addf66063498cf4244
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.30.self_attn.q_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:690f83b699c9f5aa40f0448dda2257d085adc97f25ca117817b3ac5b120cc7b2
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.30.self_attn.q_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.30.self_attn.q_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c102c6085429c1ca48074d6a4f9382e93c139ffd
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.30.self_attn.q_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:51b0d0fbdadcd9a1d921fc84ab07aa49fb1ddfb74093deae978db17f03fec322
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.30.self_attn.q_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.30.self_attn.q_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d0ebbe436b997e5eff52801967fe1fd457b7f797
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.30.self_attn.q_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2594821e8524aec96ce53c2cb1bc9af0da7b3df9560e114cd53a5c4c47a20cbb
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.30.self_attn.q_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.30.self_attn.q_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.30.self_attn.q_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.30.self_attn.q_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.30.self_attn.q_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c7fd9fbc3cd5ecc8d29fdbfbb9967fa5737c6f0a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.30.self_attn.q_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c5df16b0eecd34a7a4313033725f2589cc9d96eba8c17844962bf8a4c0d8aa79
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.30.self_attn.q_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.30.self_attn.q_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d04bf956a73866059828d3a52b7f4df0cbe37c84
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.30.self_attn.q_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:403448ee9ab190383fecb1e01b9699baa0045a8886c7707f9cb63c4a4bd28ccf
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.30.self_attn.q_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.30.self_attn.q_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..95d8ad2a7362c623b70468eb8d637a8e2ac80926
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.30.self_attn.q_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:62f108894f9ed501df21763b030b2bb4b0f4fdfcf27ef7db316a53433af1ab5a
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.30.self_attn.q_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.30.self_attn.q_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.30.self_attn.q_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.30.self_attn.q_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.30.self_attn.q_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..dd675868796369b4b67a0af974c137deff24489d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.30.self_attn.q_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8a1dd4fd661ecf1dde2312a3a083765ec0d9dd234a7fe08a2bc65145837e932c
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.30.self_attn.q_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.30.self_attn.q_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..790ba47af3ccba39e7c0e1ac012a844839fba176
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.30.self_attn.q_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a78924b534f158d5b2dd3fb7e552217118480bbee00e815b827f60591b97b0c0
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.30.self_attn.q_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.30.self_attn.q_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c53b74542d84c7dc32399e0cb074ba083fd5ef8d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.30.self_attn.q_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a671fd7059d8217965d86fc1c8f3a4fd07e8b4ddc9c372815095d34c2f08b34a
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.30.self_attn.q_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.30.self_attn.q_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.30.self_attn.q_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.30.self_attn.v_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.30.self_attn.v_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2047ee60d769df886936322944406eb53db78e54
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.30.self_attn.v_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fdfa70723ec347234b14722c0a29e1fa2464f1ac046b9c4e319e28a9f1627ff1
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.30.self_attn.v_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.30.self_attn.v_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..852d08ec7a01a7f1a4d9f02f4fbad3ed0d440967
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.30.self_attn.v_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1cba625d8ad8af07c63c5e5da229edc245b6ff5d4ac673503558fbaf9232b413
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.30.self_attn.v_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.30.self_attn.v_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..eac4a959a30ec638797e087dc7637c90a22282be
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.30.self_attn.v_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9eae8c1012739f67ab97bc0a80af6dc77b4f33d9158cef1c9dd75985bedd2d07
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.30.self_attn.v_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.30.self_attn.v_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.30.self_attn.v_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.30.self_attn.v_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.30.self_attn.v_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..835e877f9c4cb37390303421afe201eec187006c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.30.self_attn.v_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:27dc9a3b6d27746a5b3f1e091051c2b6d910823a24dcaa3899d803fdc3ceddfa
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.30.self_attn.v_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.30.self_attn.v_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c90683227beefb1c1d3acc5aa993657d67eae869
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.30.self_attn.v_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:038dcd564f7670421a7c5e0e0313f574f87e500383a52edaf14191d5dfcedc28
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.30.self_attn.v_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.30.self_attn.v_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..527fd773db94f2351d16110cbb66abad24603d02
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.30.self_attn.v_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2bb97b8fb0b52364b653abfcf18548cf4abff59789e92d420bf7ed462296ddc5
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.30.self_attn.v_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.30.self_attn.v_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.30.self_attn.v_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.30.self_attn.v_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.30.self_attn.v_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8c2e4afbe21a233687f5673cfac3d787aad5e2ca
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.30.self_attn.v_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1dd9e598c65d404d8a13819270518f856379829513b2890b1dc16a218c80ad41
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.30.self_attn.v_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.30.self_attn.v_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..74a6a52e3887bee33fe7ebb985580457a6e41e5d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.30.self_attn.v_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bb2ff2d464a0bd0f7eea31675fc7dfe15db67eb8e72c27d06255405b3ee734a3
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.30.self_attn.v_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.30.self_attn.v_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cef56ad9536f5ee5f32b433f26ceb652427af9c0
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.30.self_attn.v_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4053587ee5a3cf69621ffda2319918de0f54477ff21d063f013a9f1d66c2963d
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.30.self_attn.v_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.30.self_attn.v_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.30.self_attn.v_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.31.down_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.31.down_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1f1ff1250475182953318676344b0f8a6487e65a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.31.down_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:845c566f8dfd20c27cd7b0c9ac67b13225462ca6c8cf83665a077c87d835b999
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.31.down_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.31.down_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..94d6554d0eb37fccd7ae9cd65e78114a0705fd4a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.31.down_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c14e87f12b6c8dc6fa8b9988e65416ae54768ec51fc67da68f383ecc0c1c46c4
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.31.down_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.31.down_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f51454d035d53eeb585d292a682466b3d5e07144
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.31.down_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5068d81d80717ab51297e335998d2315d36c8c70b5f10c62cedbdd3aaa98cdd3
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.31.down_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.31.down_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.31.down_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.31.gate_up_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.31.gate_up_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ad8be8bca62da0eae86564c7562557cecd7c7ae5
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.31.gate_up_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c201a0ada6aedc54107d44686d4708f69bd71c5c5d5ca9b8292c635c80c36bd2
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.31.gate_up_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.31.gate_up_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..07b7ed50658f80f572ece10de944c20e4af25983
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.31.gate_up_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:da10b7722e7e4ecd041c0a0548c227731d44baad3b8c1f0072d956eb20f65554
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.31.gate_up_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.31.gate_up_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9141010c50dc300f5af06f3598c746de8356fab6
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.31.gate_up_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:75f14b0d61f28dd21ef56a6be1261916bd6507c44e5d0127eb7748bd8bee22e9
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.31.gate_up_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.31.gate_up_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.31.gate_up_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.31.input_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.31.input_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1a574bfec9b5603f2b057c9ffc1359f4f0c4386a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.31.input_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:714d3efcba61b7348fc4a91a52d8dd83bd157e92d3473e1cce925f640f5a5a6b
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.31.input_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.31.input_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ed63ab1175ac14c28bbd9029ca84f38c53538006
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.31.input_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2f182be91a7d56c468a9466abf01f8cb53aa64993056a162de592e31aea3496a
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.31.input_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.31.input_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fa0fec07b0120e63a3fce1c8fe48d4521c3ce3ab
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.31.input_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e175d92364de519a613a65478cb60ecb4ba4b15217256c2d32de23be4ed92422
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.31.input_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.31.input_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.31.input_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.31.input_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.31.input_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d5a25d8c8a0167f328afccc87c9c40392e81f901
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.31.input_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:116a84844108412ecb76c51049a58747693b109296a98e4fbb48ae6661c87066
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.31.input_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.31.input_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..197dd45bc34918e0cec92c8412f5c9a0b4ad715d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.31.input_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:06cf6c14f9101cb16c1f19543c0b8811ab92d87b59eb401c8e4b5e7184fea264
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.31.input_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.31.input_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e0421f3fcbd6a36de62995c3bf7914ccc8915cce
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.31.input_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e1bf43a6183e7b95b224b0ab0482f49be46aed857d67ff709deb3da1be732b2c
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.31.input_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.31.input_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.31.input_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.31.mlp.down_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.31.mlp.down_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..da3e2fe8fe2266f4505ffdc010edf5dd912f87ae
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.31.mlp.down_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1cd17b3797a5ad34f892faa0625e755c41c09bc75cb85b5232a1bf062972d903
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.31.mlp.down_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.31.mlp.down_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7efd34a27a84b5b8e2b5980c141c6cf06c4b20b7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.31.mlp.down_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d09e7219e1e9581de548c64803aed0ebe53a5b39566fbf990c060a71817fbcc3
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.31.mlp.down_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.31.mlp.down_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..dd1163a7a38f028eb597c130f5bd16056e0b514d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.31.mlp.down_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a098428f442590041c66339be92ed28b4e9899f74b99c51610dc6245a2754e3d
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.31.mlp.down_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.31.mlp.down_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.31.mlp.down_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.31.mlp.gate_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.31.mlp.gate_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..555b846f34e9176c2657fddef95a770922a302a9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.31.mlp.gate_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:de62e19cba1269f242780c0ec724083701d63335ae3cb8b813b71ddc25290359
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.31.mlp.gate_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.31.mlp.gate_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d1cd75ff91dbcdba49d3b36273894684c5c754f4
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.31.mlp.gate_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1938c57e0c6d70edbd210ed5e852fa17cbb90bb2b348d2506fadce9974499db7
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.31.mlp.gate_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.31.mlp.gate_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d60a0ae52be01367657956cd657c2b8beefc4daa
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.31.mlp.gate_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fb4c51b04cae75ff77c3505ffd3b9c8158b09e8498e5726f4635ca7c1bac1a82
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.31.mlp.gate_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.31.mlp.gate_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.31.mlp.gate_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.31.mlp.up_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.31.mlp.up_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f0686a72e0bc8d810b64a9b2bb3cedc7a1a2decd
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.31.mlp.up_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5ba75e1f8fd9959419e4f9759760b83cb56e083891b27e5bc3124da772c82007
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.31.mlp.up_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.31.mlp.up_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3054307b1360f5dbfb599313a60cbec683a7c045
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.31.mlp.up_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8781a178625f274fd516b0a8f9da86857c889d15737aa6085598e4e7af0cb74e
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.31.mlp.up_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.31.mlp.up_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4098bc512a7d969b38daac715c02bb9ab10acc79
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.31.mlp.up_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5274905dc0a100bb54176a38d5647781f4335e95511534ffddc1a6ea93e3417b
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.31.mlp.up_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.31.mlp.up_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.31.mlp.up_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.31.post_attention_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.31.post_attention_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..302fb96de652188e49e2ca342672b703e90d46f3
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.31.post_attention_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ebec3763991ca5a5b0da4d3371bad4b90614bcfbc8f00ef3284b734469b99508
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.31.post_attention_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.31.post_attention_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c93f4c54c6c1472f4a33184e23eab0ecde4d64a7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.31.post_attention_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6318a04bc0bf2c177af74d516418f6bd230e7532c130e8213da231ba9a99e763
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.31.post_attention_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.31.post_attention_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a8f5b17b0bc732eb8cbafb950ef426e1792eeb7a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.31.post_attention_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5e40f5a05049798c551453999bbc384b4892b97e8bc5d14ab91f1369e8f8ea28
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.31.post_attention_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.31.post_attention_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.31.post_attention_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.31.post_attention_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.31.post_attention_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6be6a593a6e4dc9bfaa6c8ebdf25195581fe568b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.31.post_attention_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:18aa5b81b4ba43031947a3f20e9b034e812b170cf132cde8f098f4946a248abc
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.31.post_attention_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.31.post_attention_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8a05b5329acc489e0ce943780fc79f0329402675
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.31.post_attention_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:85eba0c61381b17fed3bc9b10e5961cc522e41f753902c0eb24c878e2423b274
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.31.post_attention_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.31.post_attention_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4c23b6b26bbdfe3f017dfce297d2e42f0f35e728
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.31.post_attention_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a2a0e62854a0b725263c63807d96f3967a18029834d3afbf56e39718d0671dee
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.31.post_attention_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.31.post_attention_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.31.post_attention_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.31.self_attn.k_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.31.self_attn.k_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..742f77fdcfec67388d518a381d0959ee4f89e409
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.31.self_attn.k_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:df4b1f1e3e32f28396d7717ad900d3c5fd7f19d2cbf211e75a9ad641daff0d13
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.31.self_attn.k_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.31.self_attn.k_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..93776e0abc3acbfbde72f4225951c651658ea54e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.31.self_attn.k_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d62df6ca098becc4e46a37e0a71026097f6ce5dbc2a5b2cc1dcb9e548de2ddb1
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.31.self_attn.k_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.31.self_attn.k_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..45c0a00e556a6f1cd9c9cbb28078c14c6b85e436
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.31.self_attn.k_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a78ac92c4d84168bf39b278442b530bd7a675a7b972cccd89125dbaaec6c1db4
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.31.self_attn.k_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.31.self_attn.k_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.31.self_attn.k_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.31.self_attn.k_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.31.self_attn.k_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f1d2a606791d4590fa458c6f237968961c4b2ed6
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.31.self_attn.k_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b15b06c696518ff28269b83f77bd94d4296f30833a14d8f03e3857b5e375179e
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.31.self_attn.k_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.31.self_attn.k_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..74e5336582521f57bfdb249735da68dd8504fec6
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.31.self_attn.k_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b18a877a8ed113e3c8c68d9e3d16d9f11ab5d876c8cd29d80e9123723f9d2951
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.31.self_attn.k_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.31.self_attn.k_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..179e4ad6713cc024c8ad60916da3ff8c18053af7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.31.self_attn.k_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:116dc5599f86f823a8e078d4fd936d640bfe09ec66431669a4b4b553190c2762
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.31.self_attn.k_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.31.self_attn.k_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.31.self_attn.k_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.31.self_attn.k_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.31.self_attn.k_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..753ba0dd540d79b66d8fb00e5fa056678e250134
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.31.self_attn.k_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:129dece7ffe7257af03ef05a5bc6a42f9c4de9643bfcfe95cbe0af0bf615d982
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.31.self_attn.k_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.31.self_attn.k_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cb6977037d5fb15f468cd7ec8376044833994502
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.31.self_attn.k_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:846d81c705f74d7593cda7493e91c0c7a56562bde7835bf0575c47dddb7c0a5e
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.31.self_attn.k_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.31.self_attn.k_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..57f2b82e53ef6630470beb88b1ef05bbcbc2b8ab
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.31.self_attn.k_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:922eb79821f5109c073c386d8a9e47239e6c3f662dbdcac63b406bd037df4c01
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.31.self_attn.k_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.31.self_attn.k_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.31.self_attn.k_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.31.self_attn.o_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.31.self_attn.o_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8216d1e8c4f3a315743b70aac6027c087b0c9589
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.31.self_attn.o_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4b8ec15474cdaeca30bd277b336de15b54c11d31cc338693cae5a88a998feffb
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.31.self_attn.o_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.31.self_attn.o_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a9570cd368541db51be13c9aac8f9b72f1507728
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.31.self_attn.o_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9cff8473d941c21b21f15a645c30b629ee623f1f983647594d69e03cc093c096
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.31.self_attn.o_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.31.self_attn.o_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a75dbf9982b7a3bf38a95cad1734fed736456b19
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.31.self_attn.o_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c4d6d6b807a37730334b709eafce99ff4775a9165192d1261e6dc9b086b06fda
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.31.self_attn.o_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.31.self_attn.o_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.31.self_attn.o_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.31.self_attn.o_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.31.self_attn.o_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..af5b0bcb68e30a718978a07380566482a3ae141f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.31.self_attn.o_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:639a1af20021eeb2562229ba8ca9f125f02a68e5c090979578a9c9dde88e2a6c
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.31.self_attn.o_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.31.self_attn.o_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f20619867a00631e245e8cd65fd7fe1ccf6e3a2b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.31.self_attn.o_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b86d05021bbb5d02f6d82a8ece2f7754c7f897a46fcd107bc030b370ac83e600
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.31.self_attn.o_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.31.self_attn.o_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..83c906615c776762279e5d51ccac01c003c2aed7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.31.self_attn.o_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f84c5eb317f659b72a4ccbd325707cc885442bc74776ae7cb70e3e4bef7de543
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.31.self_attn.o_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.31.self_attn.o_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.31.self_attn.o_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.31.self_attn.q_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.31.self_attn.q_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1be2ad5688af6f907f7c36887becfa30c153f90c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.31.self_attn.q_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9da6ae3613f1ae0530a75612a0aab055c97cd9c91699fdd3e9c9156ded2f8c9d
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.31.self_attn.q_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.31.self_attn.q_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..50b7fab61e902a02acac8faebebacbc214a6b509
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.31.self_attn.q_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:caabf77d1b097087cad4e2035695df7dda2515893cd9795357e500d49edfe46a
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.31.self_attn.q_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.31.self_attn.q_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..66a4aa439f5ff89da000d23ff60ce0c1f0358fd5
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.31.self_attn.q_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d5e65812f3fda60751cd6a3b7c4ccea569ae246a52c36a3a0285be6645f28940
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.31.self_attn.q_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.31.self_attn.q_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.31.self_attn.q_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.31.self_attn.q_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.31.self_attn.q_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..654e3b84ff1efbf09fc1fa90a004e2595ad3567b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.31.self_attn.q_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fd2d27f5534872f9f043236eca95d143cbbc7b608d68bdfc5a80aa81f40425eb
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.31.self_attn.q_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.31.self_attn.q_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a26d76fbb27dbc7f22d4aea43059154b16423609
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.31.self_attn.q_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:edefb21dec242cec5187b6dbbf90785dc8093446e1ad4e5a7cedbebe84ca2bef
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.31.self_attn.q_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.31.self_attn.q_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a75def4d06c4a05147dca56cccafcf8d7456a686
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.31.self_attn.q_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:288e61e0928e269db2b871939f07393a35afa62cf28e8716232ac8aaae269e1e
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.31.self_attn.q_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.31.self_attn.q_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.31.self_attn.q_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.31.self_attn.q_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.31.self_attn.q_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f3d700aae9ff42d5945270c531764c490d1475a8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.31.self_attn.q_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:57e7e8d3b6f05b97a9ed3f0b47f65147408e39dfb76e46e75c6fb48fadd9e323
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.31.self_attn.q_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.31.self_attn.q_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..41ff1b89375d00e340618a1e7a36cf6d8f6785e3
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.31.self_attn.q_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:49c6a843b61896ce264d7795f9b9863e77a4b74f1ce2a36093feb7fc683303d6
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.31.self_attn.q_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.31.self_attn.q_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..80e66a5986a985ed0462a846c09e10468e131972
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.31.self_attn.q_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fdb129bc3879806afc2a9e65bed8067aa787ea3925c66b3629fd7eaa07557251
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.31.self_attn.q_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.31.self_attn.q_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.31.self_attn.q_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.31.self_attn.v_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.31.self_attn.v_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..772909f67bd4da44703796020fcb4cc279d04f2a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.31.self_attn.v_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:df13f9e27eb829109db9331b8dedf045a23cfa663993a1fd208409cd895f69fe
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.31.self_attn.v_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.31.self_attn.v_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2481525e49c0e15de0a9e7b2fb0a5dfc4b19f91f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.31.self_attn.v_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4cf02e7661f82c35370d1f7bf5c636d00f05be3f42c6c420927d4d7e86f4cb28
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.31.self_attn.v_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.31.self_attn.v_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a8673388f021ab7230c761f43d997d7e87455158
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.31.self_attn.v_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:73b80911e2d8ae45ec9ade814d99923562c5cbe7ac808f441a842dd2187351f7
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.31.self_attn.v_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.31.self_attn.v_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.31.self_attn.v_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.31.self_attn.v_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.31.self_attn.v_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1825dfc9ce943f3754d1906e54bc625addcc9d72
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.31.self_attn.v_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61b9977ffc1c5a2b40212aa627efb597c96b49ccea9fbcb906e0db03917c5c18
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.31.self_attn.v_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.31.self_attn.v_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c4ce389b1846435ac2e8b9e46c086f22ed09a166
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.31.self_attn.v_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d934e97b2345c534aeecf6c4c31784dd3f2b3319d7d149fbb12118d7c18ff939
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.31.self_attn.v_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.31.self_attn.v_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..03b2cc3bc35bcf037428cfa652fb91aa265d4f33
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.31.self_attn.v_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ffd1901449abc261f0abed8a3b68e305c6238ef51333d122ab8618fe67f81455
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.31.self_attn.v_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.31.self_attn.v_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.31.self_attn.v_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.31.self_attn.v_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.31.self_attn.v_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..71e1915e67963b9f872047ff1dde3df9af8fe9be
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.31.self_attn.v_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:754245c6f5b719552a10af42c588d0bc8a9d50a5d9f0f2bd6a764e6c05ae1d28
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.31.self_attn.v_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.31.self_attn.v_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ee7e08888ab795f4a712cb3cbedb38379d1345ac
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.31.self_attn.v_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:009c7a458e1d74a784b3e62990a07c991ec1a73f933aaa2556316d1268c55b8c
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.31.self_attn.v_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.31.self_attn.v_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0152ff6afb68efa43d2168ef65beeb6718546797
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.31.self_attn.v_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dfb0ccb9edb21fb4707b6e29ca0f892837c43017898ba4cfed3b143356d32807
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.31.self_attn.v_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.31.self_attn.v_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.31.self_attn.v_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.32.down_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.32.down_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ae3909ef486256cdedeea367ebf6d65b3093920f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.32.down_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ca217b9587d22f5c68a4b3ee3e36e504c5be2734b0a129a45fbdddc24989709
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.32.down_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.32.down_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..74742901291aaf5c97ddc7cae17008bb8f31f905
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.32.down_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:78987bf7b58c0addcc7e4fdc7c0cea90111bd701943414e6cb615f9939255038
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.32.down_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.32.down_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6d258b7886be8ac3bb335a675ed4a5b6f8181703
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.32.down_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61a6c11d36e4fc325441d097ea0fc940bc376f82baacadf18860e5083ff03dd3
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.32.down_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.32.down_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.32.down_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.32.gate_up_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.32.gate_up_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2592626e07884fe1a8991e84ac06d859c2eed252
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.32.gate_up_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5fa1019b7bf656f1414ac31a2cbfd9571edce24e69f288403fe6106e42c52d3c
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.32.gate_up_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.32.gate_up_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2d2938eebced956d8ccf4ef077df2eb6639cfda0
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.32.gate_up_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b629b185d77b18f502764913bcc79c1ceb3b8191dccd89d04d486cfa754e4c87
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.32.gate_up_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.32.gate_up_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3b35530cd168188d8067a7958735cf766fe88c0c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.32.gate_up_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:77fb487b60bbbbc21669de31189d6d44e65ae9f983e6574fab055250b5333542
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.32.gate_up_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.32.gate_up_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.32.gate_up_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.32.input_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.32.input_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9fe1a2128c81b835b681b961bf08cb70fdc6d3bb
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.32.input_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:02ad92257a3680446b6d8cc1ff44059ec964649417846f9dd0bc9409a8a9a338
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.32.input_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.32.input_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..bfe1bfe4cadd0614dcd93039458c439fb11bc86a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.32.input_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e1d0c1ab56cd8a6f729ec0246ee4c0b2ce32896b805a3bfcb113be32ea53b0ba
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.32.input_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.32.input_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8a42655216130a29193cfa5f9032a37904e991f5
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.32.input_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fdc5ce80735dd92f7ba856778846a0b7e9582be43e8f5d260190848ac3e7061d
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.32.input_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.32.input_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.32.input_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.32.input_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.32.input_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a97acd11ab7a69f84f1441b5d58c6d39c02c6216
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.32.input_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:398107d2acc2b06dea2e8edd90878697c6297a2f57705091ce6fe5d9ccb577ee
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.32.input_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.32.input_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..996ec636985c81ffc52d1caae7eaada2664fddb5
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.32.input_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:13a9c8a318da185bebee17efc634ccad5bb5706b4fec6564eb7ddd628a475639
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.32.input_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.32.input_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7345c0d9d28b70430ae9b53340378a6591d636b9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.32.input_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:875da0d5a9672eb230557bd7c7de8f04fc509975bed915d2c86fae3915563a0c
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.32.input_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.32.input_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.32.input_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.32.mlp.down_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.32.mlp.down_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..aa55cc9dcad915558c7656dc939d78bdb04c0d5f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.32.mlp.down_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4b4a692ea16a0b37359b33cc06703ffab3d206a1786b47ab17939ff7716a58c6
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.32.mlp.down_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.32.mlp.down_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9d9efd4d5e3cd67cd1ea4f7a1eec0e576c063cb7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.32.mlp.down_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e1920bd6611af0fac0796de9acc7326c9da2d494b52ef1783239ceee00f6c06a
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.32.mlp.down_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.32.mlp.down_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b66a8a52d42d5985e784cb33e40786eaee1f292e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.32.mlp.down_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:70958c98aab59453654c526b1c9a905f5197edd766737049877b3a70bed79517
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.32.mlp.down_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.32.mlp.down_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.32.mlp.down_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.32.mlp.gate_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.32.mlp.gate_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..49aa470055de80c6c8a505e52b98876c5a1a4188
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.32.mlp.gate_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9788381dcdc0dcda5811fe19dd061898fce6683082cd5c2ea83c43d1cad4f2b5
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.32.mlp.gate_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.32.mlp.gate_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..12d8fd8c9b8aa2a5db8db54ea06f07ac7a0b0ba1
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.32.mlp.gate_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1f2827c1e31bb475236916238975c93f8d9a3c8da6d6ec0c6ce7cf13c7949f77
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.32.mlp.gate_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.32.mlp.gate_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..15e9961153de5b80e34ab1562adfa773714c54ba
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.32.mlp.gate_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5e4ae293a9b68e2e636a3ec6e2815c43b9ec9c0245c7d5c28c4332e0b2698bbf
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.32.mlp.gate_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.32.mlp.gate_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.32.mlp.gate_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.32.mlp.up_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.32.mlp.up_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..387a4efa60ef7d24acfff5d356d234634276de4f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.32.mlp.up_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:482e1dcda0d739cfd10b69500d231523ab558d4eecd8073e63c213faae8f9cc7
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.32.mlp.up_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.32.mlp.up_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..734a383d18c1380d2e921cd8e202af9ad34b20de
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.32.mlp.up_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:deec70d7cdbcb12aaa74cf736e435e251052ed4e5b70beaf91f4b54ea991fbd6
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.32.mlp.up_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.32.mlp.up_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c62608f3d0bb61929f5ac6be5a5887b17a5bd741
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.32.mlp.up_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:05a4a6a970b90fc80ac1eff960a56290f1b12a879413ce4802ae1add93335edc
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.32.mlp.up_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.32.mlp.up_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.32.mlp.up_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.32.post_attention_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.32.post_attention_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3605f7eb38262163910f4d64ab052fd201690e8b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.32.post_attention_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:726957dfd8206b4330782e9a018e07e3644d574562f35613e94138be5c4be43b
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.32.post_attention_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.32.post_attention_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6b3c4c6c95513c9e082371a46a6354a615be5bbe
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.32.post_attention_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0cea855ff0633fd1f6bc68223b09dac0397e3ee3b04d9f03c1bcc22f8a8dfdd7
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.32.post_attention_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.32.post_attention_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8f4eead94e895722ae80400c39557fa3c6e9038d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.32.post_attention_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9eb6b0e7674307fb266d02ac325bbcbec649e3bf618149a4e023333ed376a79b
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.32.post_attention_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.32.post_attention_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.32.post_attention_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.32.post_attention_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.32.post_attention_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c56334320d1b93e2b5b05a8cd3f6490613fef591
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.32.post_attention_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:05789900c37d518d76a5453f4ac4b2fd501206384da949f8470e8c6549e3bac9
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.32.post_attention_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.32.post_attention_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..851149a753215ea5a621c8e3893c6ec2539c71c9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.32.post_attention_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f1d685d8866a8daf83aa6925ee18e460228607a0668df99094fbc2116869b5ae
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.32.post_attention_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.32.post_attention_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..523ad0ca03fc332160f2b7ba77e36abf140603ea
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.32.post_attention_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dcb3ee38a2581e9c787ce79c92bbd361f0d6ef5f0d1c7b355c847dcc755fb27a
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.32.post_attention_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.32.post_attention_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.32.post_attention_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.32.self_attn.k_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.32.self_attn.k_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..095f89422f82a46bf79f5adf90cf30f002ad9ab6
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.32.self_attn.k_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:82bf62fafef3882b4a849dc079aedbfbe3b917c1a664a5f39eef01c8f98a63b0
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.32.self_attn.k_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.32.self_attn.k_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..92f81c7603df47bf5afc0934c8c6447f3042a280
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.32.self_attn.k_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:51ce30227dc3c8069d72b87e3603b50d12ca6dbce362556f31ebd254773a81e8
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.32.self_attn.k_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.32.self_attn.k_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c81cfe9ab8b3668f39367a614ef90703ef9b3fab
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.32.self_attn.k_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:93485165fd4407c9c48785f8e6d92e26e1935c737b5d3596eeb7fbda65fa125a
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.32.self_attn.k_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.32.self_attn.k_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.32.self_attn.k_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.32.self_attn.k_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.32.self_attn.k_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f7ed8fc46f11be04e5fe01925d6a0329fd6c1cf9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.32.self_attn.k_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:868b9cdf435a187649e5ea7ff0623e3413ef1c41fcf821d5c828efb4c7e9f566
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.32.self_attn.k_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.32.self_attn.k_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0115f78e7ac0490de31419ee3a15f07704b9f6b3
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.32.self_attn.k_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1b6a624581154a3f8aae639c4fa274c470a6295bb21ede58ee3cd0d82837626e
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.32.self_attn.k_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.32.self_attn.k_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6cb1dcb6e68398bb674602fd52e6ca692c1d197c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.32.self_attn.k_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:52d030d2e40d4fd42e94f86e6d62cec271e14ec99e1cef49dc0276c72363858e
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.32.self_attn.k_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.32.self_attn.k_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.32.self_attn.k_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.32.self_attn.k_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.32.self_attn.k_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d89fb057bd68ef0bdf3691ef9e86ccfef5014a20
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.32.self_attn.k_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5c0de252fb81539ff57abf9db0189ac7b5eb75051dae99bfa6e3286b0030ec29
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.32.self_attn.k_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.32.self_attn.k_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6b1b98a1df9d0e33b673f80641742a47d7f3c8a8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.32.self_attn.k_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ebcb209db7992570b69cbce9464581270408980b63967ede03036ed9d6399042
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.32.self_attn.k_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.32.self_attn.k_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fb91058020fe5185d0756794ac89c709102bca69
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.32.self_attn.k_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:75ae99c65ffa9935b3c326a3a938fe0fcb5e5602ded42f27171a8ab705a985e6
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.32.self_attn.k_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.32.self_attn.k_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.32.self_attn.k_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.32.self_attn.o_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.32.self_attn.o_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fffdb91ed3a86e0dd1714020062854f2768b8919
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.32.self_attn.o_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e56ede5b826cb07ac7fba5f9c41e928a4bcb70b2f9e51cb93d060eb932ac1aa8
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.32.self_attn.o_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.32.self_attn.o_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ea59cc865d6a60eabbb296956d37172d3738d934
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.32.self_attn.o_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8cea1b7c23b246642ab643eca3bccd794a794e8272e814dbf90d0795b4416c35
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.32.self_attn.o_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.32.self_attn.o_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..068ccb5c2eb634bcee85c8367475afdf0f4acd11
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.32.self_attn.o_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bed5bed8620859e27101969969597a667bc530a9cb5ea23e740cef7f7ac2c992
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.32.self_attn.o_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.32.self_attn.o_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.32.self_attn.o_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.32.self_attn.o_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.32.self_attn.o_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..07fe176333b1ef533c96bedababe2e48598aad32
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.32.self_attn.o_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8d94509721c9b729fa3936f38a2bdc0e749bfdd239516b9069c66075153e5665
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.32.self_attn.o_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.32.self_attn.o_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..041688c6d41edd4c90d95356152b5238ac6ba635
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.32.self_attn.o_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:de2b78c737a1397eca45f4e43c7e6ce8b1b6ecb39d2567feeef3d5661335ac2b
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.32.self_attn.o_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.32.self_attn.o_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cd5c92384ede79512bedb7d54b5cb3886a583e01
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.32.self_attn.o_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f036520f6f9ad344d2c752a590076371d0b2ba6f2f22d34c2047545837840ee7
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.32.self_attn.o_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.32.self_attn.o_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.32.self_attn.o_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.32.self_attn.q_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.32.self_attn.q_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a91f951209394281166dd379caf50e47806eb234
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.32.self_attn.q_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c0affa4f61a240c012a9d593e73fb46f5ec47eb6309d49d30d20cbb3127de943
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.32.self_attn.q_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.32.self_attn.q_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9a5f2ff024bffaae8b065e2729eb5889d1a5fa7c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.32.self_attn.q_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:810e25b03b2abd2f7dacb50ceb7b235bf044ef882a3424b9095f667a1cf13577
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.32.self_attn.q_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.32.self_attn.q_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..876886823bc5de84009ceba06188879551a4df8a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.32.self_attn.q_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7dfa581d0c8b1b8a57f43e5c80bf58e19a8a3e0f6aca9706d510eeecc5d6ae0b
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.32.self_attn.q_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.32.self_attn.q_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.32.self_attn.q_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.32.self_attn.q_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.32.self_attn.q_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..779ca8fee60d1ef6a8cc897d60115bbbd14a85cd
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.32.self_attn.q_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:55e4b874031cd96233a042b02007afa516d824faaf41cdb1f1143eb25eb98b80
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.32.self_attn.q_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.32.self_attn.q_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..329d5587ef6c94acaafe431aadfd55b1dbc8f6fb
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.32.self_attn.q_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0679eff645e775b5480bb58dc0a790360d0edecfe6313a65e4548ce84d2b7086
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.32.self_attn.q_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.32.self_attn.q_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4086249814579716234f3ca866b95727dee268ba
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.32.self_attn.q_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:899d5d0aad784c56d11483b09228bf1b6e221f61f0655af5e15c344b69f4e012
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.32.self_attn.q_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.32.self_attn.q_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.32.self_attn.q_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.32.self_attn.q_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.32.self_attn.q_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fab19d55da4779ea0f4af75c7938ee4863e74822
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.32.self_attn.q_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4c3111e7fd461bd99925851bd0976a8a8c1ef7540901ab4094c87454033312b7
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.32.self_attn.q_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.32.self_attn.q_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6bc68f14dadeaf43c8825892da216182e3cf8eb8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.32.self_attn.q_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c606d05c2398c4a1c71cbe11a7a880b5b40c1952cd1752f8897eee34f7d51d14
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.32.self_attn.q_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.32.self_attn.q_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8f86cfde3d5af97cf06065c0f1ca97b44229dbce
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.32.self_attn.q_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:20c2b6bc2648ef73b57b68de7ff96cca5004e3ee11d6fa54aa856ed072328d6d
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.32.self_attn.q_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.32.self_attn.q_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.32.self_attn.q_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.32.self_attn.v_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.32.self_attn.v_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..48ebcf885f7547bae1ae6e85e0921551057c1467
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.32.self_attn.v_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dcc7c2b074efda449a53f9f837a70d6594a2049af7304daec28a15e5372b952c
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.32.self_attn.v_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.32.self_attn.v_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5b165dee1d585d4b0cb561fd1a36fd7eac5d988e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.32.self_attn.v_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ba22b8e1078361f7a481a68c61581206936b5a1e7470b5152cba283fc0b152a9
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.32.self_attn.v_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.32.self_attn.v_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..11f48c4f704d9ffede627cf9745418a9c3a18e69
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.32.self_attn.v_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b1a0681c5786aa59a0fcfeed23e36ed1f4af5e662d4bddfec792842947d30263
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.32.self_attn.v_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.32.self_attn.v_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.32.self_attn.v_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.32.self_attn.v_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.32.self_attn.v_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a0b6993c27d7d756fa8a7beae128f574d949690f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.32.self_attn.v_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bfb627cad10d946c78e68b9ce7455b2a36c72d3b8ef92ef2d0a166a298be3f88
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.32.self_attn.v_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.32.self_attn.v_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65781d88c22e9f5e00fe3a8c529bc25bdb3bb047
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.32.self_attn.v_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:58c858aba2ff3b16e24605daf208076e3b90aef5416158305f5930e38e5220be
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.32.self_attn.v_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.32.self_attn.v_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..04ea1197f85d596631c5a3e3ecc81e676de06914
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.32.self_attn.v_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ebef437b53720fd4f69ada628f3e5551f62ca88b8c15e6b9a3e3f455a3d022d2
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.32.self_attn.v_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.32.self_attn.v_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.32.self_attn.v_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.32.self_attn.v_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.32.self_attn.v_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..407c7c6d895d770fb0c628901c76cd6dd6d9b153
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.32.self_attn.v_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c61619a7732962e6a8d6dfbbadc7bd28e3734c5402da10a238b87696a844cd54
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.32.self_attn.v_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.32.self_attn.v_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ec7449c9a67a97f1451ac108163629c393ac2a70
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.32.self_attn.v_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f1c3e0a053cf87e2a34d6c6ea3673691cd4da7d7ef277f191073d087fdcef2d9
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.32.self_attn.v_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.32.self_attn.v_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cadc18dbfe2bd64eacef816766a0d728bb9c4304
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.32.self_attn.v_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f1a7d1aad2ef7d062363bfdcf825ae0175d5e0e8ed58951c568ae5afee0ca16d
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.32.self_attn.v_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.32.self_attn.v_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.32.self_attn.v_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.33.down_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.33.down_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b545d48309e1d1d0cbcc9b109d6fea43dde3f35d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.33.down_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8d96fe4ed98e4a637f6d6ab78bec1020660aad1368204e8ec5c83c8b948f0e7e
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.33.down_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.33.down_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..edb064fc6dc0024ee5571d5588a6b9b260eadf65
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.33.down_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5d509fb7f500a9ea6f3e207403cea3377f64bf152cb05878350ee279a16c7092
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.33.down_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.33.down_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8615582fee645de1bd6595b92ce39cdf6c39b00e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.33.down_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b59be455fcce30edbba968fbb529b365127ede7db00d50054438e29187cd6bd1
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.33.down_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.33.down_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.33.down_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.33.gate_up_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.33.gate_up_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fca9a2fe27409879e58c4026b04e85f63db4030b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.33.gate_up_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6a27d07a7ef55167f52346face4bcb9946e647ea2a9134983272675696f635fc
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.33.gate_up_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.33.gate_up_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f939931efdc25c50bdc3cc2fe10ffc1cbf82cc8b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.33.gate_up_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b515aca3fcbc1104298b463ca98e8766fbf88905a8a30d239f9b25f73895b0c8
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.33.gate_up_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.33.gate_up_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1476f69493dedcbfc464480f452e43dced59968b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.33.gate_up_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a84adee174729215f78eb0f20778d05377d0870b46ede2508f74530399b9425b
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.33.gate_up_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.33.gate_up_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.33.gate_up_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.33.input_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.33.input_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5fb71ad4e4157544ab508421e61fc1735a9e2704
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.33.input_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:25aa656d94ecc6d1d7421374c39a31e8ab511911c4f775cb3b247dd5470ad795
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.33.input_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.33.input_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0892e892706890273decb6cd6a70f654eaecafc9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.33.input_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:47539429939d80353a9185fb6aa3f4d8d3e7944a90afe426c7a5614883bbfd42
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.33.input_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.33.input_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ff075e94cc43d3150e9c9cf658ed428ce2104d3d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.33.input_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c55fa175d8f0ff59c8a9c2c98ca3325790fbd520b0814b99c811fddd8b60a366
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.33.input_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.33.input_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.33.input_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.33.input_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.33.input_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2e2cab5ff867fc91723bd5de59e75dccd1635dc5
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.33.input_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:10b9f1437f6bd17b887167bc271247727a011a0539690c239ec83e3c341bdfe3
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.33.input_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.33.input_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..787b9275ba232fc901aa5315eda73d7713a1622a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.33.input_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:92529fd6c60735a41f371f5f0c5c1e56c78f991e782b5da854072031a4fe6400
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.33.input_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.33.input_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d7ca82b76cc9574186e7d00b217142f0cc1293f4
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.33.input_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:49fe25a5a2d70ee1d197c0a17d7b427fbda85b6667a28bdef55472bb7c2b7891
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.33.input_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.33.input_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.33.input_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.33.mlp.down_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.33.mlp.down_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6c0ba3557c119aa623e3692aeaf158ca179b869d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.33.mlp.down_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:412b4cdd6260a8e06449244d51ceec0ffd4ab84a638551848f109b2f602f309e
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.33.mlp.down_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.33.mlp.down_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..32fccc16fda25706ac5ffbcd6eed2bef85b099aa
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.33.mlp.down_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c948a97137b4333cbec2bd09acffd1a559451fa7e5fed7ab2cc531c7127ab9f9
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.33.mlp.down_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.33.mlp.down_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..68960f29f1f6262d3633ba6356977c04d8dddda0
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.33.mlp.down_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:89f0f543e46901d675fe0c22b1ea6fb3d55e47010e6f83bd25e651c4cf54b0b1
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.33.mlp.down_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.33.mlp.down_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.33.mlp.down_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.33.mlp.gate_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.33.mlp.gate_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2c0c4293d9e4068fc885f22464174f5d9b679383
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.33.mlp.gate_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6fee88d5176fc96e59e6181260782870a6b692f4988a19183afad9d706d5e72e
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.33.mlp.gate_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.33.mlp.gate_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f445b205dfa74d2796cd49eb7d436a9881ce0ffc
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.33.mlp.gate_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a2bee23d9f9a6d52195695925fb93ae7e748e9f9aa454205d2068543d38eece1
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.33.mlp.gate_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.33.mlp.gate_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7a3a22d0f9142dd2f20de7fd28620c737799b7e6
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.33.mlp.gate_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dfc6469ff98dd9dbc9a9b2c55bc525ad32f91ffadf756c1ab82c7120d6f7bab4
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.33.mlp.gate_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.33.mlp.gate_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.33.mlp.gate_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.33.mlp.up_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.33.mlp.up_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c46bfe024dd5689ce2eaa46fe56e50936fac81ea
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.33.mlp.up_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:44b6915446c8575599fe40f63f3821a68cd593b9be69c8e6897e4089ec8f1bf0
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.33.mlp.up_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.33.mlp.up_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7f64f32ee7f740b6fc69e4774586fa5f200f1bbe
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.33.mlp.up_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:10da502121960f69b3c8f7d144afa5312841ceb88fab0cbd3f97c54250a89b7a
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.33.mlp.up_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.33.mlp.up_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4b6b115157de4ffc67ea40d1ae9edd19a93beb52
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.33.mlp.up_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1e6bf487733c5a0f8213408a3185a76d91d2df906aa4edfa363ce57b0f704998
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.33.mlp.up_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.33.mlp.up_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.33.mlp.up_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.33.post_attention_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.33.post_attention_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e7366de19a28ae655668c586b80ebb7f9c61e144
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.33.post_attention_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:06da65e69cd1ee692cd7fce2b78d4b48e3f1c03b4b20ab15d98d2e2231093c19
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.33.post_attention_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.33.post_attention_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b054051d605eab8ce0940c3282f217ef15ba263e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.33.post_attention_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:52903e359621af9972ac72607fe76b8b957563290cddb38a8c1395b396795ce3
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.33.post_attention_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.33.post_attention_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b8ec9e9e8f04db8fe4923fbcb0ab02e12ad526f0
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.33.post_attention_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:840b00d6a252310fae711afe7349863ecaa5ef1c23e9b5be522e9da2839d40b6
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.33.post_attention_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.33.post_attention_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.33.post_attention_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.33.post_attention_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.33.post_attention_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f7a3c487baf2df0fc8141b2947cdd2a7e1bf383d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.33.post_attention_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f23275d9d48d6aa3b4c66310bd0114e906c6e972d28a47a4d6ecc7ce66a2f277
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.33.post_attention_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.33.post_attention_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ec55a554498e8ffb9e5fef21a808b65b875f39e3
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.33.post_attention_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1703de0b90688bcd309e239d47d03f106714fb47d02d4d94c49fecb45a9ee604
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.33.post_attention_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.33.post_attention_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f8f21cd8f444f4eadc08f31378e5d9ea5bb3c7bf
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.33.post_attention_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:337e7d79fef883166b8df0b3bcfb77c6f279b3bfa275600c0eb4021d2c400c4f
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.33.post_attention_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.33.post_attention_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.33.post_attention_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.33.self_attn.k_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.33.self_attn.k_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..66d35a0a75b1553290f4483681e9fb425d52409f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.33.self_attn.k_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f35fe67fd28f0fcc41236d61b7728d6474ca7f448134c985ea82f1b0e258bdec
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.33.self_attn.k_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.33.self_attn.k_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b60554cccb10fcd87c26d74bd4d9dadf7927173c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.33.self_attn.k_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:292afeefbd3fd25eec2a6543b2645130b37cb48448e8427c1cfc29b6fe2c8a32
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.33.self_attn.k_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.33.self_attn.k_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9bceba698bcc75ec631e207b7d97825746e07cc5
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.33.self_attn.k_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5442121d2eff9d6fe9eb7f72e065d4e90c2234abf8a713353f3d1688c82b068e
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.33.self_attn.k_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.33.self_attn.k_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.33.self_attn.k_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.33.self_attn.k_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.33.self_attn.k_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..bfa21b40d924c9bdf0aafb213b9ce4166df3320d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.33.self_attn.k_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bb18fc0adb046263a4ffafbf58b14f55bccb1d2d8b5d8aae994569d8c28dc836
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.33.self_attn.k_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.33.self_attn.k_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..be7f88c3411849ac58317ae7e6d7e0679e26958c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.33.self_attn.k_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f2ea96224ff3944f73d722db8105072c28c3a0b8745a1625637a0aa915258b00
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.33.self_attn.k_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.33.self_attn.k_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..eff81c3d207948f1484884650ffcf5c40df1c948
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.33.self_attn.k_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:814c9e9b4105fa7c05780d5eaddf603b42bc07a4429d31c8b330ad590b49fce5
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.33.self_attn.k_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.33.self_attn.k_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.33.self_attn.k_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.33.self_attn.k_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.33.self_attn.k_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c0ab762d342f793b82fdf40a14819ccf4916b674
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.33.self_attn.k_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9ad1327264b07dfa30310cd516769d93b2ae0b7980769d5924334a058c59aa95
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.33.self_attn.k_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.33.self_attn.k_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f918a98d77aac5a840d84f127527a70217aba121
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.33.self_attn.k_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f320152497293d366e6505ff1373debca4df54522baf27e36f33032a9d22e9ca
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.33.self_attn.k_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.33.self_attn.k_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..363bb0ed0d1e6be7ec1b050229b699f7ec108c88
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.33.self_attn.k_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9fba16c88153280e23a6aac4f329be37c9f049ca5f9be85c67297773ceb9c389
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.33.self_attn.k_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.33.self_attn.k_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.33.self_attn.k_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.33.self_attn.o_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.33.self_attn.o_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5a87fbf7055fab7e7aaf011ca715c8c5531febe8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.33.self_attn.o_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:99473f7a0871309abde1ea2d7ee032cf67879a2e84bb876e5a993144fa99fa8c
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.33.self_attn.o_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.33.self_attn.o_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ba20c8370b8ff079aa037c955e39ade6016572ec
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.33.self_attn.o_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f3dc2e67c507cd19f5d90fe09133aa093e38646d3da38d4943e56c9e6767339d
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.33.self_attn.o_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.33.self_attn.o_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5b7358c993fcaa6c18cdf32f1e52770da05f76f4
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.33.self_attn.o_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c67aeb72a64144b4f94432f9b9c72d8ad8da2cb4b4f70a1faa9ed44adf096743
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.33.self_attn.o_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.33.self_attn.o_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.33.self_attn.o_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.33.self_attn.o_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.33.self_attn.o_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c428376bf450c065dfa1913f662ab07dfa28184f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.33.self_attn.o_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3450b60a4e920f9a2e387acb6f7dbd06032425f7b6e9da66b2ec568ac0078ffc
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.33.self_attn.o_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.33.self_attn.o_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fe38f1425323b2a981d326e445579252ae130896
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.33.self_attn.o_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5f7cc697510e8d44d5f94af11817c2cf54b791e7d98d9b0edb3dd095845b6cfc
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.33.self_attn.o_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.33.self_attn.o_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..64633c3429f3284bb40d51783f841ef9c963f1aa
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.33.self_attn.o_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:db4c81cf1aaaf373fb3467be8f12b26a44e9ec4670f4c897a7d6d5463e1b8e1c
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.33.self_attn.o_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.33.self_attn.o_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.33.self_attn.o_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.33.self_attn.q_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.33.self_attn.q_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..557a7d98ba1a640fda5abf58779390213a372108
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.33.self_attn.q_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d7fa5e0bc482f7ce4132d37042f76266a4f3bf844dea741ee6f79a4e31902a04
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.33.self_attn.q_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.33.self_attn.q_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..93d9b61cdb33955e7788b2dfa02ed017bd9ae2e3
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.33.self_attn.q_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f7be834a68bb4630fd80631da58c67ce0eb57cca0e517006e76ce6c9b6532307
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.33.self_attn.q_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.33.self_attn.q_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..67ac2f23404097ff8d5ec51c27793242b3676b07
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.33.self_attn.q_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c69447112fd3a8f671a5dab78972163798bfe2549d07f0226c7ffbc50984eeb
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.33.self_attn.q_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.33.self_attn.q_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.33.self_attn.q_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.33.self_attn.q_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.33.self_attn.q_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a9a38221edd0700c0e1568dfc6bacef698a8acae
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.33.self_attn.q_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6a26696fd345f99302cb34fb1e54af7d638ab9aa46bf7bdc84f2c25c90e2e4ab
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.33.self_attn.q_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.33.self_attn.q_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..90e8764bc97ab17c62d2ec01c45226738c76e06b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.33.self_attn.q_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:50ae67fc8151453483aeb1eb5786d701b2b152d03aa031c109d5515894dd8eac
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.33.self_attn.q_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.33.self_attn.q_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f4d8a742f848daa596394bb102b7107bb7d1f0de
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.33.self_attn.q_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8a1b67dad0b6fde7e907a8e67a0802f5de48ee92ed08b132319fb3a17949e7ff
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.33.self_attn.q_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.33.self_attn.q_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.33.self_attn.q_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.33.self_attn.q_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.33.self_attn.q_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8289793b59c333567b99b62a4cc42ea9d47fc13c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.33.self_attn.q_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b49da509515f23175a64dab2e13331e65f18014b82d163395295d9aa059b3e3a
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.33.self_attn.q_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.33.self_attn.q_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a07bf9e40a492b8b51fbd05271169ba9e6ceb313
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.33.self_attn.q_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ad296b1d42e67dc9865d839e7a62a23f6ac682f87ecb88293b3b1022f733532b
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.33.self_attn.q_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.33.self_attn.q_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ae647233b6964ed73eb985e3e22e025ef9687c93
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.33.self_attn.q_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f63df6c8d8cba70a6eef1a4f15940e628ee8f3577f6a841d50744ee510a6c203
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.33.self_attn.q_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.33.self_attn.q_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.33.self_attn.q_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.33.self_attn.v_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.33.self_attn.v_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..bc015d085ef76a9fc9c455be47c7cea03a4b013a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.33.self_attn.v_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:36b19f7c82bdfbdfe11cfac77a195ad8930d0df50479e12dc56c452ee8f3c2d3
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.33.self_attn.v_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.33.self_attn.v_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b58b5c4815dc194715b1449aca45a907cd079bce
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.33.self_attn.v_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:efda75a58197457c5a0f670d6f7721b2f4cebdb7f6e04a99efb0f8e729af696b
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.33.self_attn.v_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.33.self_attn.v_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7f3aa09713dbfc8b636f6e77cf6f40b9f2e12227
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.33.self_attn.v_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:07d2f7ad4c455625971b3590699997c65cd6597923c7077d00f6198adc53ce22
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.33.self_attn.v_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.33.self_attn.v_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.33.self_attn.v_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.33.self_attn.v_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.33.self_attn.v_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1b39b7b32e5429b16bb163dd3bb8de881ef22799
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.33.self_attn.v_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dbc4538e8db46cb502a1c6e291f49dbd6eadd946e50422ef90e2f8010a5f810c
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.33.self_attn.v_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.33.self_attn.v_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9c6d38831ce958492124848640abce72f8ea97ad
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.33.self_attn.v_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:76373ad9b73763a7d093b666a35c08a384fb41dadb4bbd94b14d92ad74bd8c3d
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.33.self_attn.v_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.33.self_attn.v_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4f84adbe8934d70d7bdb65a6cb2fac9480a0b616
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.33.self_attn.v_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ee761f6250e2bd07d83eb01fbe2afe7ccb2fde74a595df4aa812aaad83df9305
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.33.self_attn.v_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.33.self_attn.v_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.33.self_attn.v_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.33.self_attn.v_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.33.self_attn.v_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..10b33b445c36dc3bc61e8ebd1e10a45f669b0f2b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.33.self_attn.v_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:07c5acb176d06143ec904ed85c855c3448f53bde1fe345f87581f4a98d3d6477
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.33.self_attn.v_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.33.self_attn.v_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1951a253a4e7d0633f09006f2c3b63415e898fc9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.33.self_attn.v_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:641db9753781c29e778de868d662f4968594f4de27c2af5d19ec9c369d155139
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.33.self_attn.v_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.33.self_attn.v_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..45d437491c2331eec6409082f42fcef4657bc628
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.33.self_attn.v_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:697ae8230dd7d02ab14d77d3e867a6f9d1950fa59e001bf6d5772245a8e2a840
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.33.self_attn.v_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.33.self_attn.v_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.33.self_attn.v_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.34.down_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.34.down_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fc174fcc296638632309091e0825f752650fd301
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.34.down_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc23a461f88b8553b13bc109c2b1a3112b1f941a38b817153dfd1c33615983b6
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.34.down_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.34.down_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b8ad3e6082c3a8d6cf7832f0d96abcdbbcdff591
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.34.down_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ed38f89b418062ea58000760de05111910f89ec4705c6d0c4baa1ba72f555c11
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.34.down_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.34.down_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7d4b6f019eb0436e1d43e8a3b498d60123f68dd8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.34.down_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29ca01a5318a809daa48cd967a3a98955265524fa328ea650cff8294e1d4d292
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.34.down_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.34.down_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.34.down_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.34.gate_up_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.34.gate_up_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..30f6c2ac66a3a6d6420a189887e95317b6b4ec84
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.34.gate_up_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0362b1a32ac79dbea7a1e0e10ced3187a7024827a29ec393e65518457d918fb5
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.34.gate_up_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.34.gate_up_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..42b93ff1b979e616400b5261e1c16da24abb3412
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.34.gate_up_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aec5bc2edd02191dd640c908e8ad65798da9995d090eef91c03e23fdecf38887
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.34.gate_up_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.34.gate_up_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ce37d0e24b165bc8a9f6591ffcc760219727daff
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.34.gate_up_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e3601927c0bb5fa00368339cb02528e7d9e2a2618c6a25711c2c9d499dac4b35
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.34.gate_up_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.34.gate_up_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.34.gate_up_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.34.input_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.34.input_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..820d57ce79acfc59cbfed697cc09842af0a07bdd
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.34.input_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a26a89339d77b6bf79c3c8dc7a360caa978b4cf8a32813e05259f956046649eb
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.34.input_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.34.input_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a4d3e9650b819742a7b6e304149a169c319377db
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.34.input_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c60a1519dc1f90deb78ba8c4df4d7705f0929a28280e0a64d408f5bb25cc0b99
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.34.input_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.34.input_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..afd415a9a73e891d770c6c23a46fc6a6d0f7f7ce
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.34.input_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:df5f921134e9c8fde508de3ce7b76a47319edaa82be37c851023a5315a5ce2b9
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.34.input_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.34.input_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.34.input_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.34.input_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.34.input_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4a92ebbd86a38f0c37506750bcdfb6335991e2bc
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.34.input_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fb8a247f3d661c54212d34bf0509b822fcd53d38c592c311414dd5891a9fa706
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.34.input_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.34.input_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2101de76d1f9df3a0802196fe58d62b98698a77f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.34.input_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:88d93dddb2bd2d109f0f60145e4f31dfa74d06b1d646b2ba1e001a883b4c2a45
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.34.input_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.34.input_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..49c7a306d51a0dff23604ade474688a7691462ce
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.34.input_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:57667ddd76bdd6f18300f009f45ddc0448d99b7989c3287e9d2dd4632198bef3
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.34.input_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.34.input_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.34.input_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.34.mlp.down_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.34.mlp.down_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fdd9304f47ebdad0eab0d138da0216ce8ac09240
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.34.mlp.down_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:022495223402d46d51e8607bf02ec43d778fac1545538ce4699f6c7083c9f03b
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.34.mlp.down_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.34.mlp.down_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d6af98b3c751837f716989ce87aae50088c3febb
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.34.mlp.down_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:643ed1ab318123174376343c177b8141b6fbea903421c351c3547203860cbf68
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.34.mlp.down_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.34.mlp.down_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ce49cd71dbe76ce428dde0918ca5fcaa79839bf5
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.34.mlp.down_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:74dd78cfc94584d0ffb9e9adea56107ab36b266918aaea4ab12f5990edfdf84b
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.34.mlp.down_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.34.mlp.down_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.34.mlp.down_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.34.mlp.gate_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.34.mlp.gate_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7639f6a885ef852dc872a9947d4de5feb6429b87
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.34.mlp.gate_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:edd21f4d73d4d09ca24fab7700449a8a20df314a6876cd6e34fe85513a25fa80
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.34.mlp.gate_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.34.mlp.gate_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0572908828a4faf6def2742da3f5694153120f0f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.34.mlp.gate_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1b1dad15a7d70d02f8f378dded70d51f1cbfb63c0b26f5138b7749be4e363e59
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.34.mlp.gate_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.34.mlp.gate_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0fee5528d624542bc82b413f1f1ee3ec712390c2
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.34.mlp.gate_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd72d3dd7cf4a3fef83b783f4d1b6f1edf2e22e8314c1a93790847fe2a9bd37f
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.34.mlp.gate_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.34.mlp.gate_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.34.mlp.gate_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.34.mlp.up_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.34.mlp.up_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a6f77172db433a90d9a7bd7e3705a200dd390eba
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.34.mlp.up_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c94da00b0774af42ea966d353fbb787a0ffbb10c24c00680f51c99e2cc5d33ec
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.34.mlp.up_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.34.mlp.up_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..66f2ef34804a00fc78de13495059548ed961c4b9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.34.mlp.up_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3c986407901ffae8d514c60ac7e7f5aba6fc5cb4c74c9ed4556910ceb7ed4e95
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.34.mlp.up_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.34.mlp.up_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..55c518b25a50e53e12d597263b4b31311e982392
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.34.mlp.up_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f6c94241361e904dd00529b500f969b16deb26aebca3948d66d431254df709b7
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.34.mlp.up_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.34.mlp.up_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.34.mlp.up_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.34.post_attention_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.34.post_attention_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7ec356142c04863c2af797a4841dc65597b50c0f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.34.post_attention_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7f5cebf5d8550e632148936c83ff58771a84a61e74e15141bd7949cb9ea1445e
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.34.post_attention_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.34.post_attention_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cbf242511ce9ce9f153ab2fd73e2c63193505a8b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.34.post_attention_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6e65ed8519a711783b88ef7e8716b282dd429077ff489bb75d8bcdb025d2138f
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.34.post_attention_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.34.post_attention_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..566fd7b0fd4d3fc4962013ca656d9258a2006986
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.34.post_attention_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4dbe41905053aa087975d58d13f1db09bf1053392316929d9d885a069d4d67b
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.34.post_attention_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.34.post_attention_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.34.post_attention_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.34.post_attention_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.34.post_attention_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..371cc21967fc368f15cd44b5f17c582973f3b9c5
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.34.post_attention_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b3a9543b0bdec2a2b634b515291a6bc9615a9c0e8c613378130097113cb9a0f6
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.34.post_attention_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.34.post_attention_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..63e3289b1207bf04c5aba9cdd3e62203cdb4cc36
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.34.post_attention_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:df0845d65d9de022a0dc0ad706fa9e3b16d28c7c30345d79616079e710c602c9
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.34.post_attention_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.34.post_attention_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8dd9e9f728e020958526b35275c73dc16b22c9d1
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.34.post_attention_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b14726436af0af0a293d328cbb031753b7440387aee971bb63d996df62b94f66
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.34.post_attention_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.34.post_attention_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.34.post_attention_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.34.self_attn.k_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.34.self_attn.k_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c8d32623429dc5858f0e4236c1f3dc445cf3c9ad
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.34.self_attn.k_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b74293527b2cb6bcb95f8fa3aec8cc0383037861b240f6345478224badbe6d06
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.34.self_attn.k_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.34.self_attn.k_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d9a8aaacb0ed0822b03fdf246bfaa92489ee164a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.34.self_attn.k_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b48ad6fe30d89031f6c7124c8e328f688c0be4175087af4e231f6ae9e0ffb23c
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.34.self_attn.k_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.34.self_attn.k_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..70af25f165c18b4ab558a847cffb7cb69a6d2080
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.34.self_attn.k_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5e6b33ba59d48a86d2a8a29fa2d2c08e35bc6af260f62061d6886378d075bd54
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.34.self_attn.k_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.34.self_attn.k_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.34.self_attn.k_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.34.self_attn.k_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.34.self_attn.k_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fd48882c44656f5b355b1fd85bafd0204cc9c7e4
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.34.self_attn.k_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:618a2f016c9ee345f2bb3a67d29dd65129de3f92dd0192b393bb697c91d2f05c
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.34.self_attn.k_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.34.self_attn.k_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e06b56bd049c2f8768bcce83acfcc56aa9267b6b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.34.self_attn.k_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:47060d6ee5b0600f5fb2dc282e22897545993281b46e4465e947f884662cbbca
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.34.self_attn.k_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.34.self_attn.k_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6c2e26bcd0b10ba6bbc52c6d0a826cbd65cb3a0c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.34.self_attn.k_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7c06943f01cccee2a1f8af16da0ec37adfe16d9b41ea4fba9a20e0271617bbe0
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.34.self_attn.k_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.34.self_attn.k_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.34.self_attn.k_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.34.self_attn.k_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.34.self_attn.k_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..28db734379721e6f564a858efe7e633c86d5a63b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.34.self_attn.k_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2d7a5ecc54ddbc1482c8ba84e7fd6ec0ea080028e6041d8cc75708b63015e553
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.34.self_attn.k_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.34.self_attn.k_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1d0e87c5ea74d70113b4159f225fb7b49ece6059
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.34.self_attn.k_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:816cbaecea51035fe10afff06f8a98b3761a8a86ac940bfe00b4fe729437a376
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.34.self_attn.k_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.34.self_attn.k_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..852e5a41ac0d6674e5ff88cf60ee220abdbe6c98
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.34.self_attn.k_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2c916664434db2532b173566819ddeab00366beae5fd3c02cae8b016e60d126c
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.34.self_attn.k_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.34.self_attn.k_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.34.self_attn.k_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.34.self_attn.o_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.34.self_attn.o_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..adb8d0a7177494c1feb63ef734b4dacd3056bc69
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.34.self_attn.o_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e907fa08f6c6872f9f04dc18934b519879923a6cd5ffbe368b0a24a781f2006b
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.34.self_attn.o_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.34.self_attn.o_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..53a4f3de8f441545b8e02a63261b5395b44cb659
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.34.self_attn.o_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:65043aa0ef01122ed525c16c4c1f98673f2eef2af024dddf2dcbace81a04e95f
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.34.self_attn.o_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.34.self_attn.o_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9fc62cc7e899ef9a68404d5d470565885dd83240
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.34.self_attn.o_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:14d94eb67fc95e91a21243ed8e9fa7fad2b9457e1c8716b3becfb7dba5b4d8e0
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.34.self_attn.o_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.34.self_attn.o_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.34.self_attn.o_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.34.self_attn.o_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.34.self_attn.o_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4a3138fc3ce5aa2b859a63c8373dd2d0f4dc03f6
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.34.self_attn.o_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d0f0af52c1a5ca136f43414f35abc2b2764e4aed1a554c17907142b4acf4e436
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.34.self_attn.o_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.34.self_attn.o_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..abb7d38f3549040f4a23b6a1b998c8c46d12b3a9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.34.self_attn.o_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:24fc3854c8bc448d4cade4a8047a1276d98091d2b040a5c33fcd90886c523d2f
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.34.self_attn.o_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.34.self_attn.o_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3bd4e477989a684f1cdf9600bf2c448585f9f115
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.34.self_attn.o_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:afdef18c3f076c0842ed5f19ba3a0e6d0db59ba9c1a2b2c5015167369d672ca9
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.34.self_attn.o_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.34.self_attn.o_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.34.self_attn.o_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.34.self_attn.q_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.34.self_attn.q_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c0aab7ded80d8231c31acf8b0a2c51feda45c6ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.34.self_attn.q_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f92dd549922511fc71ac9a4babd658f0cefd6d8c51210d09f798e614d27ff806
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.34.self_attn.q_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.34.self_attn.q_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d6455855cb26087acd4fc8c5838279524946bc5e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.34.self_attn.q_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3f66f382fc716ee66c254ef2eda7f1c6050b891b91d1f2415c5ef7828d70ab07
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.34.self_attn.q_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.34.self_attn.q_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ac0248224ec6a6df387d48809264672657156711
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.34.self_attn.q_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bf677f8ac39552358c576c966cb57b596ebca4e7e853fdb2d6c4244613c0d09f
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.34.self_attn.q_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.34.self_attn.q_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.34.self_attn.q_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.34.self_attn.q_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.34.self_attn.q_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d6182badea9129e9800faa90f32cda891434abb2
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.34.self_attn.q_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c152ce226315f58efc56d0a448fcc363e8e97ebdeb7c1256b175cabaab8e2965
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.34.self_attn.q_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.34.self_attn.q_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d8f8a2e21fb02f55fabd30cafc65967b9d89e7e8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.34.self_attn.q_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1d829a7ced1cc3a7ebc492a06cbe8d8f233c07bf6df249915a70b61e5eb51964
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.34.self_attn.q_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.34.self_attn.q_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6175f7f55ce8057e4f4e436acbafb318ec762a77
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.34.self_attn.q_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:11be9a57afc00ca7f99d70b1ba1f5aee3d8471b121663a8f15b596f97cefa221
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.34.self_attn.q_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.34.self_attn.q_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.34.self_attn.q_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.34.self_attn.q_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.34.self_attn.q_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8178d5e971b6818ae63d7ffb6f03dfb89f615da6
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.34.self_attn.q_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:40faf6cf631ce6204a450710c80e604806549e11d80baa3199b2912728ad2387
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.34.self_attn.q_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.34.self_attn.q_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6b42bb4c00065ace5af02cf32f689659b8b9174c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.34.self_attn.q_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:73fb2d50da031678f1f0b0d508fd9dfdeda88b321344b26a752cc35a4d30be41
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.34.self_attn.q_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.34.self_attn.q_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e79f85030729d07827f0c4c21482cc41cd9412d6
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.34.self_attn.q_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3a7e71a9ea1a58a1eba06a077d591663dd0995bd758174c93f034d81e7ec379c
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.34.self_attn.q_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.34.self_attn.q_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.34.self_attn.q_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.34.self_attn.v_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.34.self_attn.v_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5805a1c988167eb046627e8544c0b2fe295283f8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.34.self_attn.v_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:400f758619d14266248f7487d139e6574de5e324132b8ef5da94cc6036088c1b
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.34.self_attn.v_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.34.self_attn.v_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d15e2caa0ff374ff540850d26315b3c7a7dc2532
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.34.self_attn.v_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6d44d25c29fd28db68ab02d5d8c648f225ce776ffc5406a5105e91ac84ce3bb3
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.34.self_attn.v_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.34.self_attn.v_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2b9f12ae52cbf06073499c8568c5b632d1ce337a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.34.self_attn.v_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d97085827bf9f9bf9ebb88b144ce50adef90d3b50fccee6115c73a7bf2e146ac
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.34.self_attn.v_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.34.self_attn.v_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.34.self_attn.v_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.34.self_attn.v_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.34.self_attn.v_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e88d37afdff25278a1a115b6dc160eb38def7fa8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.34.self_attn.v_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b036e1d038e1bf277a2f3984ce2da47114f9caa2d3a500aeecb9fd50199899c5
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.34.self_attn.v_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.34.self_attn.v_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4b83947bb4433461676cc5769a3108a198c0c095
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.34.self_attn.v_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:378ecf83da02e04b044170ae852ccb72541cb18446ae824192407f8bb7669896
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.34.self_attn.v_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.34.self_attn.v_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d001b81f22a669baa6adf921c8da9b77ebfdcb79
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.34.self_attn.v_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef3d2e0374502dfcf502504b16cd1571b6a92d1bb7d9e07354b8155ac247c409
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.34.self_attn.v_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.34.self_attn.v_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.34.self_attn.v_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.34.self_attn.v_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.34.self_attn.v_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..711b13d09522dd430f781528fcf774f48388e756
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.34.self_attn.v_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:10c64b7a5566b75eb3925582d2afc7a82e2f41e27141790b4fc7359a30e6027c
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.34.self_attn.v_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.34.self_attn.v_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c82753415b4693ba849909f09b6f9025e4b3dc15
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.34.self_attn.v_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cdbe3e60d3ce742dc8e0174443fc5297a8f69513e75d877a2652e35936e86625
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.34.self_attn.v_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.34.self_attn.v_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..863a0494e5cba77b844ea07933b17c9a3e5d9338
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.34.self_attn.v_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:94c0b9dc47aaa2de689ece73a98a870256c43b8a1b1193012befbc8f48e25345
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.34.self_attn.v_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.34.self_attn.v_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.34.self_attn.v_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.35.down_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.35.down_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..03bfa75bf15b012543e832c976c95bfe87b0bbbb
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.35.down_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:66a84a7865bd987a792d1c872259c72ff96a502619f15e25d6b3fce40329d001
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.35.down_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.35.down_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9dea2d41007638ff418a2cf970fd267155a48236
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.35.down_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:11578b1a786dcd56e8e8fb80b68710b336bc173c54f56b0f63b982a0c9291403
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.35.down_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.35.down_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..088e2bbfcde34fb71a9a1f11bc6e88b17c812eaf
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.35.down_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:441da85be66fb4e89cbe17c9b40cf0d88484774f4c19ca255b4da674f83dd9f9
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.35.down_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.35.down_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.35.down_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.35.gate_up_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.35.gate_up_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..90431d0f5717c70e1e0a89dc18a65e8d25c802d9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.35.gate_up_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:318f0203fd1b9fb2e030b5434454424fb86a7c3aef0503b19f88766cd369e504
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.35.gate_up_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.35.gate_up_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8b7acc95f86b3e84dc60e7107b1e1bc70ce8eac7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.35.gate_up_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:82429d62fcb78de6084133d521b664084eeaded0550be25b414521747a4e34fd
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.35.gate_up_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.35.gate_up_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cdcf351f78129aa284c6b8047148b661efdee7e9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.35.gate_up_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5f9c929d7966d2423519bdd90fd1c356b8678a5b1632b177eb6902da3abb9e90
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.35.gate_up_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.35.gate_up_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.35.gate_up_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.35.input_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.35.input_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4df54a6a4e78c344157065135acb64c275cdf064
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.35.input_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0e46be8b1cfb3654eac46a4c3f3b1aea066ae643d485d8a0a80b86dfc73ffd13
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.35.input_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.35.input_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..38771cd0c7fce372879306b98379b65bf01965fc
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.35.input_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca67f703b8a80cc8b8ce3c0be92e9383b649f7aa55e7427006d74bab827f03af
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.35.input_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.35.input_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ceeec30016b9dc2bddda355cc8c39cfdebf1280b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.35.input_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:158936ab3935fa562d9f9d14cb239fbf829a44d0ab577d45a13ac78989e3b3a5
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.35.input_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.35.input_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.35.input_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.35.input_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.35.input_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7f095ec2ecb9de2f8eb4dd09c3c74445a017a35d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.35.input_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:30f9fa39035433cfae6a882c5575356898817a16d8c163f760f0cb55bf8ebb86
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.35.input_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.35.input_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..95ff9f9639afb1c7f56f80cdb7cf13c67bc36e3f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.35.input_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a5409ee056b0f06339c3c77fb294dd67b83beda6379977e0f8475c85b7fbc1f3
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.35.input_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.35.input_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..84520d97c914c5d476bf501dbc55e610b4068d5b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.35.input_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:730706d19f5e8a4c7e6eca67e8f383bf8699cbeb0679797d079eb5825f6ca8fe
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.35.input_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.35.input_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.35.input_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.35.mlp.down_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.35.mlp.down_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e4fa720befe86d290b0ebd3745d091a2148a4006
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.35.mlp.down_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3fd88a01675ee8cd2921bd230b2a16b1af52df34a8ca1af3500cbef08dcf51f5
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.35.mlp.down_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.35.mlp.down_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f4e2d19a5f6ce512932ed6073bed6253b2f351ff
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.35.mlp.down_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ced0fe77575d32127de27bc87a90228296d6e7022c1d34ce4e47cad86801cc60
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.35.mlp.down_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.35.mlp.down_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..114b5009e8fab03e893a19ef0bb9df010c0c1fbb
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.35.mlp.down_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3b5e311e80bf8cb11f2ab8415dc12c5ca41cd672d97120440a96ead71f9fb4c6
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.35.mlp.down_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.35.mlp.down_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.35.mlp.down_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.35.mlp.gate_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.35.mlp.gate_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2a6e07a70c752f70ecbed888b2f6df6199ea3d83
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.35.mlp.gate_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:959e22bb7cb4b9335e51e381148bb1d85ca12c6cfc098d07cebaf46b6ed9c525
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.35.mlp.gate_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.35.mlp.gate_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6574275e384425262b099d9f4c64edbb717e3ad4
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.35.mlp.gate_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d4b109fb004ab317bf5f64ab0b87c60b47f77fe5e65307e75495bf7f9ea6a49
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.35.mlp.gate_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.35.mlp.gate_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7b4b549ba759e9fad88c28ca3501d501024d3532
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.35.mlp.gate_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:57a2713e5180a92cd6e40126b22251e4208100d71c187bf223b5ccf795aad84a
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.35.mlp.gate_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.35.mlp.gate_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.35.mlp.gate_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.35.mlp.up_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.35.mlp.up_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1e4bc064451880b05eaa23f9f0b104d06cb5642c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.35.mlp.up_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:90dbdeba1ff6fda42b55a9e032f5e15370eb4df1ce46d03ceea42aa3b81613fb
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.35.mlp.up_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.35.mlp.up_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2894cbc9b49fe842bd75a104e5a73bf73ad07b34
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.35.mlp.up_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8c593dabefc8b00bcb30f118e146e89d9f4a46bc6ba1f376f117e212d7d271de
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.35.mlp.up_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.35.mlp.up_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..873426f6da19e26b29e00d9516388deead828cca
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.35.mlp.up_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2ee5a1e4e71ae6458ab9b1bffde7163d26efbc5f0c685bcf23e0992bc825611c
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.35.mlp.up_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.35.mlp.up_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.35.mlp.up_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.35.post_attention_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.35.post_attention_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..256ad052a309068aeb2138cd5d50a6140a5ad1ef
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.35.post_attention_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:23c70c3d0342a6c4120d369a1074f8a0572c59205734f07ef8fe12f88e9a031f
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.35.post_attention_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.35.post_attention_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..969014e2cf397d2d6a5d540ca535d2e7927a67fe
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.35.post_attention_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9ff02f2008657b4a95f827226856ed143378a3cce5cda79892850889baf0f1ee
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.35.post_attention_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.35.post_attention_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f248742d2b53a51fa63650ca2e6efa7b2a271537
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.35.post_attention_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9ff83c22e77ee7c5ede2a8f50d1185a670b39b86dd2e3aeead0e3d1796ef1d6e
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.35.post_attention_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.35.post_attention_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.35.post_attention_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.35.post_attention_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.35.post_attention_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c2bd2409de12bc9fdc537a0c3fd4958c172bcf9a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.35.post_attention_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4c6c8223b0a13afc76c5a9d28f90afe176800fa4bf53b13a20c59bb54e8d1c3a
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.35.post_attention_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.35.post_attention_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2be30ef0655882eed4993c229d8c409e17ecdda5
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.35.post_attention_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:520d664767c0172b373dc2d9fac208db21d37a518156ac80c8a3543534e19482
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.35.post_attention_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.35.post_attention_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c269beefaa76bc3de7d32184b0ff9742c0de0462
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.35.post_attention_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ebf3a8ffa420cf17d0b81b7b1c9b2ce9881610d92c8472cfc9c1a48105a9fe3a
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.35.post_attention_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.35.post_attention_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.35.post_attention_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.35.self_attn.k_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.35.self_attn.k_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..550ab56a7df3dd96d8a475f64e5afe89db5798f1
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.35.self_attn.k_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7e2491805d24d98cc9fdb3504fddd4d0aa11e54c02d2c3d0a5eac7e8bc7a9eff
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.35.self_attn.k_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.35.self_attn.k_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0270c164c8896ee25d649540d1620cab1d525c38
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.35.self_attn.k_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8a58ea8c0c7f7841fadd011ce7015ec7501aa370c7d770aa1562ebe4db9b9abe
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.35.self_attn.k_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.35.self_attn.k_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9d185bfd1f1d599071b572452cdafb089468dcc8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.35.self_attn.k_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fd140f7477490e4978175492a0b5db4ebd9702a8ebcd460708d90b9aeb07d7ed
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.35.self_attn.k_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.35.self_attn.k_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.35.self_attn.k_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.35.self_attn.k_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.35.self_attn.k_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b23356959f7c75e77638127e560e4f6204defe59
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.35.self_attn.k_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dacad58aa7f59e4a2ca3dde9345c159fffda8c307d3ffa76726e71594bf7464f
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.35.self_attn.k_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.35.self_attn.k_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..826c90d047f1fc0a03dae3c34c75bfd339cabd1a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.35.self_attn.k_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c9ac89395ee94b595a9647010546d1a6d6751c936f8679411cf627b0c1812407
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.35.self_attn.k_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.35.self_attn.k_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3ca126e3519af9ef9664c78b7ee162f8d45279c2
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.35.self_attn.k_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:afe24fac7f890192150ba584e457fff966f65c117775f7b0d7430351ae0b7ee5
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.35.self_attn.k_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.35.self_attn.k_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.35.self_attn.k_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.35.self_attn.k_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.35.self_attn.k_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ba23bcf8c6c788019ef2f4dedad2120d28f662a5
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.35.self_attn.k_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:838ecd27817f0987dbc62e18188ba1d24b56ea7cfe0b4e33da0a1b159b1f69d9
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.35.self_attn.k_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.35.self_attn.k_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..607230851e4437685cb2be1fca14bc299dab8363
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.35.self_attn.k_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cdb4360a447efec0503df7e1be66904a343679c1e8554a8a5e57b2af6067357d
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.35.self_attn.k_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.35.self_attn.k_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fd46694bc7e0270b9f348aa6ad9fa5e21c961aaf
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.35.self_attn.k_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a60314cf02e4f83f5260fd87099a35d12fb82a1e3810045f18b18dfabeb04d4f
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.35.self_attn.k_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.35.self_attn.k_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.35.self_attn.k_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.35.self_attn.o_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.35.self_attn.o_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ee57581b412f1263d933280fa590b32e296fee32
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.35.self_attn.o_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b81411a23a4f284de79e76a4b0dab7e3e9c5774ab00a4c4abe0eeae47ddf1d44
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.35.self_attn.o_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.35.self_attn.o_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..61ed490773153d828aa63f1d5274f0c3402e52fd
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.35.self_attn.o_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:98ddf1ef9f5ef8c5f82d72ad77ada3f0c7d517c9820e39af395753100d896704
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.35.self_attn.o_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.35.self_attn.o_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8a0ecd6739fc7ec75c16eba639f17f4427a6493f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.35.self_attn.o_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1dbfafab9af64d2a2036f63d8837d005a22083ed19689b480bc9eed8f2b17531
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.35.self_attn.o_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.35.self_attn.o_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.35.self_attn.o_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.35.self_attn.o_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.35.self_attn.o_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7f2ca2668e069f0132f3ec7fedd3cea59931d266
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.35.self_attn.o_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a160c3182e1c7c1e18e408768af1c29b73e5949b0f97ef48e9edc1e333658657
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.35.self_attn.o_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.35.self_attn.o_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4928cf33b9ffb51578709329efaa04ca47bafb70
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.35.self_attn.o_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2b6bc7d4b2428d06be567e549f8706e3bdaa0ab30402b8b9d179b5bd8f572a71
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.35.self_attn.o_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.35.self_attn.o_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7b8126623d885f8d71d2455ec9d72c8edd885373
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.35.self_attn.o_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2d852339adc209d113e70d47d346a8cc748eaa6d43e3b0d993c88abaecd3026a
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.35.self_attn.o_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.35.self_attn.o_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.35.self_attn.o_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.35.self_attn.q_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.35.self_attn.q_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4165bd91f9376e4835446121c197a0067b9a98e1
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.35.self_attn.q_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d2a45201f84b5a198e333e90415869d0702c2bd05b18a20f144e1718cac7a865
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.35.self_attn.q_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.35.self_attn.q_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c13dfd10347ec780ddaa7229fb868e5c83763f0f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.35.self_attn.q_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f6e528ab9043c8c10dc4161c02392346a2ea9f8385e0f10d3e150f8d2eda3f83
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.35.self_attn.q_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.35.self_attn.q_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..24293f6299d4c7019f7a229a7289f03e27b9bad1
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.35.self_attn.q_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ee9fce90a723f3c8abcb30b204c16a5f83bd94083f25e323bc6fe90823a0e851
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.35.self_attn.q_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.35.self_attn.q_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.35.self_attn.q_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.35.self_attn.q_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.35.self_attn.q_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..76d691e25213fa8f525e970a4eb495beb3e9541a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.35.self_attn.q_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6532b6451cce39cb8916e048408859862b522423834b431ab8e380240cc09d68
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.35.self_attn.q_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.35.self_attn.q_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e013c350a245c7501946700cda6b402928916268
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.35.self_attn.q_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3b5eda4e5e32316c7173477c6749f00191f4cb4a3b7c5cfe84b5112a274e7c18
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.35.self_attn.q_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.35.self_attn.q_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..399631ee55d3cf49bc4143527e2d3d31f3a12802
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.35.self_attn.q_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:818e319bbac3e5fec19980cdc6b7b2fe41b016afde326871ef47565ad160d56e
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.35.self_attn.q_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.35.self_attn.q_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.35.self_attn.q_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.35.self_attn.q_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.35.self_attn.q_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..121ddb854718da013c7e391996439ce03efbbb76
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.35.self_attn.q_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:159771944a57fccd23dfed623751a524d4259f5e02b14c075b856016b755f07f
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.35.self_attn.q_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.35.self_attn.q_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5b07add89269830e8c673c97ed8a7a96f42d85ae
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.35.self_attn.q_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b580b471c74ec361cf74006587937ec417e82f7a3de02798d34c2f744245a895
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.35.self_attn.q_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.35.self_attn.q_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..376db30361663286cdaa5de31c34e0f38afab192
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.35.self_attn.q_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4ba0075fa8b1458439d1a230ce3905a6c05190e460e78072424fbdbf73f1ad9
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.35.self_attn.q_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.35.self_attn.q_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.35.self_attn.q_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.35.self_attn.v_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.35.self_attn.v_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7fdcec16d64f78d014740059cc33a9e28cf88022
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.35.self_attn.v_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:89dcd33c8b2ef414bc999c798dc8c46be397c04b3dbb5aee0244d5740fcaf7be
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.35.self_attn.v_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.35.self_attn.v_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2727ff5d42ac3e09a184b42cf4e4b0b036045f88
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.35.self_attn.v_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:74971663b1f67f2f5d2bc1924b76baffb5f21941eb15787c9ce40b1cf90af3fc
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.35.self_attn.v_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.35.self_attn.v_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3e42953d803d72ffd6c1e38895f4ad6602004c82
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.35.self_attn.v_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2a102c19b07fb006cbf0aa24bbf3e9e785922372779cca1011c02dd3ce30e4f9
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.35.self_attn.v_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.35.self_attn.v_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.35.self_attn.v_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.35.self_attn.v_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.35.self_attn.v_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a4ec55673441527dd1cea2d15ff7096c34222c07
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.35.self_attn.v_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:50495dca96efdcbddce66087a38585056599674acc8a01ee684719a26eb4230f
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.35.self_attn.v_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.35.self_attn.v_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0d9c966dff3e355e780868655971375411d150da
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.35.self_attn.v_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2dfa6aaedbf9bfb1b0aba534a5adab36b9f8f9b6510f480a6577a306c161366a
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.35.self_attn.v_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.35.self_attn.v_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f26db318d90e072c654cae11b6bbf5e9f730c9aa
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.35.self_attn.v_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f1c60064365a8ac071f796f7456b34020977a237468f44a6b1e948c82aedfdd7
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.35.self_attn.v_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.35.self_attn.v_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.35.self_attn.v_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.35.self_attn.v_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.35.self_attn.v_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8e47578da6efc0a2349a1ae8c1d1c0f2970ce8fd
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.35.self_attn.v_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:60dfe507b5ee0642a38e09655092c96fcb24f4469b7498d9b918e784287b8516
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.35.self_attn.v_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.35.self_attn.v_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..93f38010dac8f55a79651b13ad2c3df59e73896e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.35.self_attn.v_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c59e2807fc33a32b6462131e8384d7908edb49952d5da46849f999781c1f1c81
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.35.self_attn.v_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.35.self_attn.v_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9a1583df855a6e41c778ab14f5a22af655a9a21b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.35.self_attn.v_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fa39d75b0723f05842157376ef575d2b8943a8f1b8dccdc2f02edfeda5c40519
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.35.self_attn.v_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.35.self_attn.v_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.35.self_attn.v_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.36.down_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.36.down_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cc486d69dc2732e6e4a3f552819c8e8d74deacdf
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.36.down_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:912d4f750469575ebb30ecc154b7d27945854df56d47a94d385ae469b154d4d9
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.36.down_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.36.down_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..aec299f5873a416a58a3ced91029ef68ad2cbaa5
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.36.down_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7ccd11ec3355708da4d34e47cc2d63d3c16dc72c03b3484ff7f1f7abffb262c4
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.36.down_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.36.down_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..665a5961c1b0eff0676819060e671fc20d32f5e6
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.36.down_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c92f057324ca53f7a13635fe3afd39ed8d6960cc88fe44808133cc954e53ca6f
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.36.down_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.36.down_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.36.down_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.36.gate_up_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.36.gate_up_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0697f4944edbcdd649fa83540f78109797ad3cc8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.36.gate_up_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d1fad7e3c727815ee3b5c6add80800a1e2cb3d40049bbcab4ed11bf69813c2e0
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.36.gate_up_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.36.gate_up_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7e6619e0a574cbab5d85d1ebaed51eec27b19b20
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.36.gate_up_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:374aa3e1820261fb323f2513dce0f57f10c87b960d1549742737572e61f7ac95
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.36.gate_up_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.36.gate_up_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ee6808615e7921fb378264d903d126cf61f1fea7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.36.gate_up_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d103d643a78cdea04619343ae125a878e70b749c9b15492d535a6376ba8e01a6
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.36.gate_up_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.36.gate_up_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.36.gate_up_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.36.input_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.36.input_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b8c9f10a188c88b90a1cca97046674e5c3dc79d3
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.36.input_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:570f5b31c838d2e34508ace1e4025489ae36a42a10665a67b64d546eb9b7cb4c
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.36.input_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.36.input_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3dba2c3752ad1e6e43cc597bb5622fd31116bde0
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.36.input_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:55a7e91eba24b095bd0df0c943abdd3095f6db91b4a5389f5e838f63dfaa863d
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.36.input_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.36.input_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7451bd8ad5228ac3f99c340cf26ec752e306c3f6
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.36.input_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f6d4e773e37b8bb9e4a20f6a9434aedca8f5ce4dee28aa0c4b7fb87acb6a3803
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.36.input_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.36.input_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.36.input_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.36.input_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.36.input_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..465a1371d8f7fe3b0b2da4e7ef693baa24024afe
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.36.input_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6a5cca9c2ac6e307da44eac122fe52e24cd37337e07697a4072e46f729d42275
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.36.input_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.36.input_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..38719fc720a3d23515607ab9153aca3fd7767a51
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.36.input_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b5c0f38e85aa99b44363789fa092832f4d6dea8bfb53b3004759e259536c043c
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.36.input_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.36.input_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..04973559b5693382b0d6d09abe75a20252c3d6d8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.36.input_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c027ca08528196822704a1bf545060d34532624791a63d57ec19da347c83b81
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.36.input_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.36.input_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.36.input_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.36.mlp.down_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.36.mlp.down_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d554663d0f601ef3f2643d2223d5501b56e99cd6
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.36.mlp.down_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c6a091292efb5ccfdb55dfe3971802cd8d7b67338d32baab2ba56d28aecf4fba
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.36.mlp.down_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.36.mlp.down_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65f187ee7fe7f514027b6f89e1a1c67dfcb3fcc5
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.36.mlp.down_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:11687e9cea4d12de2c6dcad28d13533cdabcc4e2146a89a2cb305e971f37437e
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.36.mlp.down_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.36.mlp.down_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..daa8859e38257f206203cf90d97aec0e26463163
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.36.mlp.down_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:389eabdb1674e598042759eaa858a1df27b5d5bd87fd5aeada60f10a4f2cf3ca
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.36.mlp.down_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.36.mlp.down_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.36.mlp.down_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.36.mlp.gate_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.36.mlp.gate_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..47628733c2849cd87f9bc82d5e1920426aae4ed2
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.36.mlp.gate_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5180e25caf310b292ca15e29a6da9b64512ef35bd39b7822e9688dab803b4ed6
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.36.mlp.gate_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.36.mlp.gate_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3b54e6ffcc920fae2e48641e7daa2530975fefbd
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.36.mlp.gate_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e195478673b02fcbbcbab828443be624270e262f25b0da247b03fb21e903ab41
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.36.mlp.gate_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.36.mlp.gate_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..839dcbd75d6c0866c860a5fe855a6de307615282
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.36.mlp.gate_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dba4b04a314568256435b60ddd63e876152281aed65267a363e28d699f750e18
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.36.mlp.gate_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.36.mlp.gate_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.36.mlp.gate_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.36.mlp.up_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.36.mlp.up_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..df0c5bec72a65d3c94efdb9f47edacd7523b8f61
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.36.mlp.up_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:18998323f9ab0ba81f565bc4a437def549fece6ddc243c0a406d4389907044c3
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.36.mlp.up_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.36.mlp.up_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a9a240d52d1fce19b00bb596b311f229cee3b35c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.36.mlp.up_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:889ecc7d6cc70221443cb00dd25bfabc20c523e4978d70bd50b2d82617ce96e9
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.36.mlp.up_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.36.mlp.up_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d778a123140ef7b1d3055366fb8cf98206a7b1b4
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.36.mlp.up_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b024f93e78d961797e7b107a0c89fcb23798d975e8bbe40dafd80e20298f43d9
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.36.mlp.up_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.36.mlp.up_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.36.mlp.up_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.36.post_attention_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.36.post_attention_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2c9de6c65e8cc1c448016b7ffd327123fe57e69c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.36.post_attention_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe155400e164575a1775bf346562bf32e15629b84df80ba3a2c99e4723085ad2
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.36.post_attention_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.36.post_attention_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7141e9410fe29d7d30845c41e2c561f24f6d8674
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.36.post_attention_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5e59c5baa391451b45ce9c0f0b6ccb94ad347fedb9057bfc23c15c2352fa73a8
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.36.post_attention_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.36.post_attention_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cd572b99dcf8965b77baca2ac6b53ec62e56273e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.36.post_attention_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e8eaa3316d5ed1435b89efb4ae8f71f3bb8a7f6afff80e333a3186aa0ee034e3
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.36.post_attention_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.36.post_attention_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.36.post_attention_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.36.post_attention_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.36.post_attention_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..18427afdaa7a27ae771f58e650ef38f0bf7d08fc
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.36.post_attention_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8e427a2af17e80df78963e421ea8b634e1449f14760b2d97370e56b24894cd5c
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.36.post_attention_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.36.post_attention_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..10de4a45a419b104f319270dc3113a8b2f1b9fc2
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.36.post_attention_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fa959d3971b907b30da93a735e6c736401ac6a43a566794cb83631320a0727ca
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.36.post_attention_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.36.post_attention_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..850fa2c16e85accb4f8acb2010dff152eb040773
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.36.post_attention_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7b49e997955c9b88c5ff4758a9a47671bc93fab97e2518049220fb02533f9174
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.36.post_attention_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.36.post_attention_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.36.post_attention_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.36.self_attn.k_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.36.self_attn.k_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..542b3b597fe35f4875405d89e4df81f2af9b864e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.36.self_attn.k_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2005ec7e5c6f38a26effa77a12caa6aa60f54c8885e75f342fe528dc29df6af7
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.36.self_attn.k_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.36.self_attn.k_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..929fb90737cac5c969cbecd3159ff3b259ee8279
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.36.self_attn.k_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8089af0a7f9327c8bc938bce63d971646f0f1560fee2ce8f6e326c6f8d0efca5
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.36.self_attn.k_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.36.self_attn.k_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ee6981183228fa6382a55653993ab4d07adaab10
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.36.self_attn.k_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2c0dbbcff00923976da0b28c901bc550131f11e5911c4e42910d402204522bd4
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.36.self_attn.k_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.36.self_attn.k_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.36.self_attn.k_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.36.self_attn.k_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.36.self_attn.k_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cfdaedc2d8f9bf8840b572f26c27b316b003e171
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.36.self_attn.k_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8e51dd38baa339a431ac40262b8aa315ebc3c6c7c8e6bbe9082540e7c4d58b7d
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.36.self_attn.k_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.36.self_attn.k_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3078ebf1684e9e075946a9a6d78593d792ade78d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.36.self_attn.k_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0c7eab7a3edc79e072b87a85c081d845a3a6c27919c66d6780ffbaa0a9c95584
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.36.self_attn.k_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.36.self_attn.k_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a8d25463168b03e0f04f216078fe99c8eb4525cb
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.36.self_attn.k_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9620ed99ed12bf3b999f0897ebf2f5e1053bca03dcf493ad3c131e3866ad78d1
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.36.self_attn.k_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.36.self_attn.k_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.36.self_attn.k_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.36.self_attn.k_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.36.self_attn.k_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..51b01ad408b9b88d4e12e5f3d8aae71c13ccd45d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.36.self_attn.k_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c3af84ae48e7858b9f0069cd87bd7716fa00322607fe34a156b75fe001e10aab
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.36.self_attn.k_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.36.self_attn.k_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a0925de1766c61a11be63a2c587ee658c451052d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.36.self_attn.k_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9041719df89bb9e5ac19c4db3493b513599923b710be8242fe6436ca4a9ea170
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.36.self_attn.k_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.36.self_attn.k_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..db5189d018d889d94d6064f9a49e92f349d7c77a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.36.self_attn.k_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ec307d564be704206f4e68c02d008c2d684b78232ab21dd3506fb818bf5f57bc
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.36.self_attn.k_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.36.self_attn.k_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.36.self_attn.k_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.36.self_attn.o_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.36.self_attn.o_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d057d62381d7431782e0f46f6a7c7cdcc87c2f0e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.36.self_attn.o_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fb25df7a4a05b8dc87ef6a394a69506c6eaf2b2179be7281f2c1aaee1ed8061b
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.36.self_attn.o_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.36.self_attn.o_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9f0eb92087b8acc6e83627bfaf016d82e00ef6fd
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.36.self_attn.o_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:28723b2f9afe405aa01779949187a40de0176a29dddb7a8803fb2e3bc5b9926c
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.36.self_attn.o_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.36.self_attn.o_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..76dd157bdbe9fa6c4635f0fbce1a358679d8a21a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.36.self_attn.o_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5c3f2e1d045750b34cad98f23321b12819e55b0796259c79c33812f7754081eb
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.36.self_attn.o_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.36.self_attn.o_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.36.self_attn.o_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.36.self_attn.o_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.36.self_attn.o_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c5efb41547081a859d2b6e4c16d16d3eec49629e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.36.self_attn.o_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e199b6273eaf087200831a0353c54c1bd1b833822fa41788ffab811b6ed01f6e
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.36.self_attn.o_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.36.self_attn.o_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a6a27c0e4285bd197056fe102e3f6ebe1c9eb60b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.36.self_attn.o_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b10cbe0be5a15a73142e938ef0b0fa669d7150ad8e7771bcbd10beed82f4c240
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.36.self_attn.o_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.36.self_attn.o_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a770d214ec7da437fd64ce8fdadab5f19799c54f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.36.self_attn.o_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:111c19b3e78a5f3ac06e69b258002090a4df6aaa219af18a0a41c633821e3279
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.36.self_attn.o_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.36.self_attn.o_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.36.self_attn.o_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.36.self_attn.q_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.36.self_attn.q_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..96fe8ffd305c631ab2faf72d340c7d35b1606445
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.36.self_attn.q_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8ee5e67bf2eca7d8a7fa960c4b14f06454a163cfcd005ff9c5c29503f5703baa
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.36.self_attn.q_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.36.self_attn.q_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3a283aa11a31465fadab5bbe2637171710f086b8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.36.self_attn.q_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:380db0ea5339cbdfbf5b2ac710315fa944c29220a41d4c8088515200182f6070
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.36.self_attn.q_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.36.self_attn.q_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9b248ac1a6120545ef3f744fe4bde124d76efb72
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.36.self_attn.q_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e70eec24d7bb5b52f22e2c03cab5ae41814c10f53050585abb4e1c466b812394
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.36.self_attn.q_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.36.self_attn.q_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.36.self_attn.q_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.36.self_attn.q_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.36.self_attn.q_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d42c3c902aadef6f46acdba2370854cf6350d85d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.36.self_attn.q_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fc433cf590f0081e7b123a8fca041efdd03be6f7f7330a83520084571fb7091c
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.36.self_attn.q_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.36.self_attn.q_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..bfbf86b87b8e04de13c488f36d8921837bdedd45
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.36.self_attn.q_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d0ac2ef4ee1a470a55c6058d57d35a559fdca27c342d718b93bb625a49212d03
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.36.self_attn.q_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.36.self_attn.q_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..53cfad473aeea04ecbd53e39ce911910f0089269
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.36.self_attn.q_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:81af553a257383b2a371748da57eaed60b473414d93dbf618fd529c4f039b39e
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.36.self_attn.q_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.36.self_attn.q_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.36.self_attn.q_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.36.self_attn.q_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.36.self_attn.q_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c1cefbddddd142f60d1c162fe3648c2ea7b3cb92
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.36.self_attn.q_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8ec771c8c4aa95d03763064656f957564aa7e4a578279d8433fb53c30686ac78
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.36.self_attn.q_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.36.self_attn.q_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0bdd6610d5da09724e807d5346b82569fc5f13a0
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.36.self_attn.q_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:05aaa8b9d5949d6d432621196dd3b65657c58883e8fe22a9487ce3602caffc7e
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.36.self_attn.q_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.36.self_attn.q_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8b6c4480ba03fef6435ebf430c70ee2b52ec9beb
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.36.self_attn.q_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bcf5eb93f33c2d068208617f419532e041907a2076affd0d76b7e852ea72096b
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.36.self_attn.q_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.36.self_attn.q_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.36.self_attn.q_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.36.self_attn.v_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.36.self_attn.v_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b64d0b66eb103f3f45b57e05dc236f0ac50c88d4
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.36.self_attn.v_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:abe748453d269e6ee4080107cf9fcb888c1dba4c78caa303848829d2081c3080
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.36.self_attn.v_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.36.self_attn.v_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..badd984028bd539eeff547ee63ef678b35862239
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.36.self_attn.v_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c869cd30d25098cf94bbb8f568b622336d90e942b63c089f23ae5080f602d185
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.36.self_attn.v_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.36.self_attn.v_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0f94549e143834eec6fde639d2e863b732691680
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.36.self_attn.v_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:433876717513b830759b905180e55a39ddb1edf9d28f341d1cb96b64ef745284
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.36.self_attn.v_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.36.self_attn.v_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.36.self_attn.v_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.36.self_attn.v_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.36.self_attn.v_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8acb0c8d54bd2894c2f4ebf9a3be6b6d5dabb496
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.36.self_attn.v_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c4541f228a72fb2c0b21f1266d1ca112736b043fdeccf877efcc730c93fb01a1
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.36.self_attn.v_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.36.self_attn.v_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ab53b10f0ef52162902eaa63c19833003ab2bda0
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.36.self_attn.v_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:21357d05c599593c99a095e77a1af78f0f72c424c29cef58e4a7c8464ac69f34
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.36.self_attn.v_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.36.self_attn.v_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cb1d01f98098e5887c91586cae581252bd069c7b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.36.self_attn.v_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:38927dafd75c1a69fc025918faf563653cfe553078ae764b4c6cff4727711ead
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.36.self_attn.v_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.36.self_attn.v_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.36.self_attn.v_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.36.self_attn.v_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.36.self_attn.v_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..eb0dcdb3bd5193e8dd32f3e4574a8feae4b90097
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.36.self_attn.v_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:22580f710c23dd3bc0f90711d29199e09a6f48e231d059cf608eb433c64442b9
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.36.self_attn.v_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.36.self_attn.v_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fe48b3a3b6a130ad670a0d394716cee99eb1498e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.36.self_attn.v_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:85cfc99c617d0983e93c0444a8ad5101e9a106a18df7c57b05ab26cd9f61c0b7
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.36.self_attn.v_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.36.self_attn.v_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4df426570bba992cac2621bf9a1b7938135f24a9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.36.self_attn.v_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d30d4f22f230cf07918229aa0edc05f5ed0af431f6f8b17a129535867b0e07b4
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.36.self_attn.v_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.36.self_attn.v_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.36.self_attn.v_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.37.down_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.37.down_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..27835f7a9ed1ca97c4ee6ca942abfad833cf29ef
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.37.down_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3c5fbf26c91ae1be303e797de28adf56aca519bfd15fba7f44da46d49ea1513d
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.37.down_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.37.down_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f41e284e1a25ee758102b4ee22d109d51a62bf49
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.37.down_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:60379ec132a5fc639800440477181dc44186820028bb82ceb73119958ecdfffc
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.37.down_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.37.down_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..88c2c42a13e89c9307dba91ea558413c27344040
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.37.down_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:215b1797a485f339648ef5fcb40fa9612fd4732ec186f99dd55b5431a18f9c9c
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.37.down_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.37.down_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.37.down_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.37.gate_up_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.37.gate_up_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d92a6cbcf1b06f9e83facba9ba1d2f955312a97d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.37.gate_up_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:03b8a9ba348c43318408cb19079752ff5758b939610546a9e31911754f0a8ae3
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.37.gate_up_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.37.gate_up_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..842e6772f2e8158229b0566cd2d48e70c8e70ed1
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.37.gate_up_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:137ae1792a5b839fd261b7f9f5f0036c60e80fe49fa3c6ae2018c28c08869ead
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.37.gate_up_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.37.gate_up_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f023e59dc80ab36cf7d7026973cfdfbbc41e8052
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.37.gate_up_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ee220f0041629f5a1b0c1f354669beb49441a7eff371928d71cbebf5634d9e76
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.37.gate_up_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.37.gate_up_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.37.gate_up_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.37.input_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.37.input_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..30c060344c34cfeb79da9f8399870b636d217e37
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.37.input_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6d0cfa963a54057c5c87a660982cd9192d1eaeeef597596ff0ee7d324d740b87
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.37.input_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.37.input_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..30f2d2fe4125e7b5ae37d06ad9e9e4b0508969cd
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.37.input_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fc5ebb8272836317e4f64a2ab69acda45d96ad2a4abcbd8e8615247df2e153c8
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.37.input_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.37.input_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7c2c65dc7cec5fe103dd1f1cac37da14da748206
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.37.input_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4f956059930b00573ddf33a51b2bd6bf09b3b58a0d52fb73a08f1986feae0d73
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.37.input_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.37.input_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.37.input_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.37.input_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.37.input_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cbe7c546804967b49059ea10887c40622321c846
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.37.input_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3f022f6c0e0a5e8b7c600634817049e87fed392ffc025f853274bf9a12f8bbea
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.37.input_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.37.input_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5173395d08a17ed7eeed37aa005636d338bd697e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.37.input_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4250982fa252a8aec02e4cf71f7092a5a8bd7a362721f2f6af863cc28eb75eea
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.37.input_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.37.input_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5037de53abb8a83ebc72ed87a7930736b7067eec
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.37.input_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2a7d7e9aa22e013f02f8d604649e7e22565cc73e8a7043a96df72939f0afdc4a
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.37.input_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.37.input_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.37.input_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.37.mlp.down_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.37.mlp.down_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..aa7126183e561052fbc06401e7f1df0be2db1827
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.37.mlp.down_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca6bc025fd55381d567a45a48194cd5926e3f986388c0eb8f475d85c6cc191c3
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.37.mlp.down_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.37.mlp.down_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..64ff38e14b0db9aa44fed2f848fa54df6fd6d129
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.37.mlp.down_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:03a918ecb08619e661f4ea6958105d558ad699b16a52d56c96669b960892aa7e
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.37.mlp.down_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.37.mlp.down_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..85f8d5f2c2ec7bba08c71a8b63636bd7b485036a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.37.mlp.down_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:21243614dd05b079852479c89666d602ad64ffc52120a943e18e30a4f1ffeb38
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.37.mlp.down_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.37.mlp.down_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.37.mlp.down_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.37.mlp.gate_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.37.mlp.gate_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a7134f4907ba5516139a85c13abd95ac3b50949a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.37.mlp.gate_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3797ecb963ee386aedd76ece5d9bf89698ced89e6149b5b83246906c1e0ec933
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.37.mlp.gate_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.37.mlp.gate_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7cfef16ca52f76dea43f291bc037d8e6850e94e7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.37.mlp.gate_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:551ef12ea2b524bf59b98ebf6f7641c9da64a47368a271595f1420052b9fe593
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.37.mlp.gate_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.37.mlp.gate_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..15735bffd84961dae737dcc597dd76fbe030dac8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.37.mlp.gate_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:756325be4288df9df967ac9e4d207c836f75d7752773a28a06853785d1b8e6a8
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.37.mlp.gate_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.37.mlp.gate_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.37.mlp.gate_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.37.mlp.up_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.37.mlp.up_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..42ae37e93c4b9e0c6bbe34d9c42d1bd74a4790f4
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.37.mlp.up_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:28706e851b5ddc003f4256790b9fb40a09664bc8ec95498586afcd968c74f0ba
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.37.mlp.up_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.37.mlp.up_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..142ee3237a6a7ebcbea9a0ccef40086767d3cd53
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.37.mlp.up_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:467501eece4a079e9627b050856f925e54a4c17e941bf7c68d7750c8cd0f6e2a
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.37.mlp.up_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.37.mlp.up_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3e3c8572fcd3dd084b9b3d4792bf78ea41426e16
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.37.mlp.up_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2267fc7a2dd678b9e3c9a06030db89761d328d4860372187a6269c81b51102f8
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.37.mlp.up_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.37.mlp.up_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.37.mlp.up_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.37.post_attention_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.37.post_attention_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9c02e21aee9fb85a27ae61d0aa5abccdeff23d73
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.37.post_attention_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:080af440970aa5a6f502b5536ef84a2dffa79af7baeba904521b028a7acba56b
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.37.post_attention_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.37.post_attention_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..810ea88d5514f29eed4628ed27bd1dfdecef80ee
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.37.post_attention_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:71ef8a5199242b4d97c64ae203ff4cdd74dc80d0ee59c902a2e1f7a68767b92f
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.37.post_attention_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.37.post_attention_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..73ee3c46468bfe87f0b4be6543c72b5e95ee673a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.37.post_attention_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:baddc15ffa52687a2bdefbdb990b904df481834f781367298fedc11c4825cca6
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.37.post_attention_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.37.post_attention_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.37.post_attention_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.37.post_attention_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.37.post_attention_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..641dc40706fa4340a55431134e5733ae46bd0f1f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.37.post_attention_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8da8a56d827e10c69b500aa7b237f32f3f19d279175d1b6eb394440f58f9e5ea
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.37.post_attention_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.37.post_attention_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5a6b291e731b3c0b58b0281b282226b7a4472dbe
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.37.post_attention_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:412e7200698509c33a8a2d0597728d472013d9f1de42889114530d1b347323ca
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.37.post_attention_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.37.post_attention_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d49ca7daef37d3ee6c8ceff82733820f81982274
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.37.post_attention_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3bcfd55b94cdb8e277d3ad56dbd8046e97ed74b6f707b9c34f62654d071543d7
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.37.post_attention_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.37.post_attention_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.37.post_attention_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.37.self_attn.k_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.37.self_attn.k_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..563a4eb6b0e02d9e4a66271327778ccc089677b8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.37.self_attn.k_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4b6b253c497203ab8093da9ece98f6d6ceb4231647b046fbc7f0b5dcb9cc8fa0
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.37.self_attn.k_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.37.self_attn.k_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9540de6f09addb049d197258089277795abb1c26
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.37.self_attn.k_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d778389b19e55eed664433029b091f1167ae89b370ba0d58e35970829ed2b23c
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.37.self_attn.k_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.37.self_attn.k_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..bcfca564cd963a9fe602d38762f966a40b2d5108
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.37.self_attn.k_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6d6488c832b3d2aafc200e97b5e7b3fc60c17c5b6ac53feccba7db1e3b81a87a
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.37.self_attn.k_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.37.self_attn.k_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.37.self_attn.k_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.37.self_attn.k_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.37.self_attn.k_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..bceeb2b1c7a7475ab967ef5fe74e336fabee801d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.37.self_attn.k_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:49aece494070573eaaef2fa930e4a58377cafc5200ea2286b9bc2d2921c2769f
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.37.self_attn.k_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.37.self_attn.k_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9922f745daacb68a9be2219a323949c762b83a4b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.37.self_attn.k_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5f046485974f1d491781b14c5739d22c60e3a93e3d46d267b32893b57100d638
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.37.self_attn.k_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.37.self_attn.k_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..40f190e8147fb37e575797878612a0fedea92759
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.37.self_attn.k_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4c74350a827b0cd6b8268e628e3d41b7968665a4bd1ad9950847a94346cde650
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.37.self_attn.k_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.37.self_attn.k_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.37.self_attn.k_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.37.self_attn.k_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.37.self_attn.k_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b6452172a28a79848f48ec46d644fb0dac988b03
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.37.self_attn.k_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:80f02eedc7a63bd94f705699409a66271b7be050009ebc98d5d6b4e7aeb0bf9e
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.37.self_attn.k_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.37.self_attn.k_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e8cf8752c9f506de7f23960a941497c4f03c22b4
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.37.self_attn.k_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f279b5bd255f848b1a67a5114c81ee4daa2326d515a05b5d16aae0a8ed9146f3
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.37.self_attn.k_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.37.self_attn.k_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2bfa74efc8b2baa4af5162bc074bc280a44a4674
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.37.self_attn.k_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f65c471a095dfa13763ef26fe219d818fb449cf0f5274a8d46dcdb340f0d1c4c
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.37.self_attn.k_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.37.self_attn.k_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.37.self_attn.k_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.37.self_attn.o_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.37.self_attn.o_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3ff95e6bf8aadd5d947fd4aff3d4869aeb51b304
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.37.self_attn.o_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1c33ba3f5e25f4cbe562e75bc6f7a5d0265143766816064e9dbd3e9d83155fe2
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.37.self_attn.o_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.37.self_attn.o_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..bba178e41b3f70d7a7c1c28ac3a9321b095f978e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.37.self_attn.o_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5b7a330d690316878eac525b7ed017fc26d2f92575e458671a5f8a0cedce846d
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.37.self_attn.o_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.37.self_attn.o_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..349d2080dc2d754057ec0b6a3892c5a2c8eccce6
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.37.self_attn.o_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:84a60d95817e94a75b76f1eca757ecfea4042ac27e5a48b5094dcff1bbab1640
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.37.self_attn.o_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.37.self_attn.o_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.37.self_attn.o_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.37.self_attn.o_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.37.self_attn.o_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4e8542d229f87f1712dd96b7d57952b152a33118
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.37.self_attn.o_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d74cf38e5fcffa3d3556f4cbad8985f460e19e9b343c680960664efd2eefcf28
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.37.self_attn.o_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.37.self_attn.o_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0ba2259d399ed39db2a042f289210d36c02adf49
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.37.self_attn.o_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a91f05259d9668ffc79d1f1071f6a835be2cb327290b37cf1e77afe958d7781c
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.37.self_attn.o_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.37.self_attn.o_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5a434ed9f41f3a5066dd394f004527e87217ec91
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.37.self_attn.o_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c2ec5c59c75de998cc992df682102da51f82a6caed60d85227278c1b6a038977
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.37.self_attn.o_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.37.self_attn.o_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.37.self_attn.o_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.37.self_attn.q_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.37.self_attn.q_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ded49c666f03b90dcaa7eed12d4a224ee10e8055
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.37.self_attn.q_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:56fb23efb90054b27cb14c39233ec54c0bbeadb23d0157f4078fd6e6ae16baf0
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.37.self_attn.q_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.37.self_attn.q_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..095b9eea0048feed6d7949d30a9b8f223ebb9df4
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.37.self_attn.q_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:81c57ca58940d7ea46995be802f274f4c9778e5925a64a1b9f0f31692b521605
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.37.self_attn.q_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.37.self_attn.q_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b1dfdf95b3640fc94dcade035a0fc0db69b51ae8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.37.self_attn.q_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3e8f7147f451269cb276c1539bd516fa55e9392d7a8f7ab1713ead376cfc2772
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.37.self_attn.q_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.37.self_attn.q_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.37.self_attn.q_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.37.self_attn.q_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.37.self_attn.q_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..42b4af867fbe7cb26eaee26de16f27db98fae110
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.37.self_attn.q_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bacf72ff9ae9b17a3ecc1c0d4331314e7062343d3b7ff60edce82a4a6b17ba26
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.37.self_attn.q_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.37.self_attn.q_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b2c963ba96e36817f10e26f44466ccd5e7365e7f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.37.self_attn.q_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d06bc4886a61cdd3f688b4522c61e0f8cfccb6f07f892ea94ac848c6cb42dd0b
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.37.self_attn.q_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.37.self_attn.q_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..aaa90e090aa2422688cd4e8a4feaca3a9d349c4a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.37.self_attn.q_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29dc7276669cb7d80a1bdefd31fe36ae129dd9aec949c6aca3a59262aa68a875
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.37.self_attn.q_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.37.self_attn.q_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.37.self_attn.q_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.37.self_attn.q_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.37.self_attn.q_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d1b0a3ee23be95af451cb5685f0e82e95fbe7792
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.37.self_attn.q_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a5b2eba30b952a5954db8886b2d74cad1a56338a1568826f166cba60f14feac7
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.37.self_attn.q_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.37.self_attn.q_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..95e859b0df8fde5ce817412ca822064ccfdd4112
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.37.self_attn.q_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:af8917d33728838c7d861cda809604e16e25d5729507d953e8f563f4023aa58f
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.37.self_attn.q_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.37.self_attn.q_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c7dd98894211ad2ba61b97a420cde387ee30e43c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.37.self_attn.q_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:af4dae3db563269adc04a724728f87f8d0a70c815fcbc75f5c349e54db5775b8
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.37.self_attn.q_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.37.self_attn.q_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.37.self_attn.q_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.37.self_attn.v_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.37.self_attn.v_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..10d711d9b9634349b999d9c3a5daa84e501a2617
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.37.self_attn.v_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:802418747b33e27cb85e8856cb53ebdaa34758f0407678e4661ec9247b1d093f
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.37.self_attn.v_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.37.self_attn.v_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7a75801249bdb6bbef9f3b40ec629053a407117a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.37.self_attn.v_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:82ac36a7f759a1cbf8971cdb3ad074dccd0402857ca63cb73d54299d37e7fdf3
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.37.self_attn.v_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.37.self_attn.v_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..438d2228a2ccd122871df58005084ef884c812fd
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.37.self_attn.v_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b8173e4c1df63458824ef387421f34bcb4c9cec70be51095e79e210215dc3123
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.37.self_attn.v_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.37.self_attn.v_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.37.self_attn.v_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.37.self_attn.v_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.37.self_attn.v_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..48be876617482a4f2e9b37d73109ce79cf9566af
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.37.self_attn.v_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:06fa4123c08057cb705621867e7e1e6e4cc5e151a7483137b1bb4b7592cd2592
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.37.self_attn.v_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.37.self_attn.v_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..188d32987fbcec0169daed36729e58b7edc4158e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.37.self_attn.v_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f7c3e6c588a70f93a67d112e47d43591994d243b5739d3d937458cbee6b3ba2d
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.37.self_attn.v_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.37.self_attn.v_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3e4990b2020e0739277da4d4304f45356db961e4
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.37.self_attn.v_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:36431e1021977042fa2f9ea0454a061bffa8aed85df3c267680afaa0bbcdfd68
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.37.self_attn.v_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.37.self_attn.v_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.37.self_attn.v_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.37.self_attn.v_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.37.self_attn.v_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3df3b31438a8bebd677662826484fac18f2bdeb2
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.37.self_attn.v_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:14afd03b9aa5167f3778595d0e77913a1ac34a5478795130f4ec7b9e9ded6e31
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.37.self_attn.v_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.37.self_attn.v_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e9c32ed15eb210e8a43ff1e80a5e28cbcd24f482
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.37.self_attn.v_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1853260f0e65f72093024438694ce84a4b28480c896f6b83466a0bd2d494c9dc
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.37.self_attn.v_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.37.self_attn.v_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fd4af4364897090521c133d1accaceaa034fd067
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.37.self_attn.v_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:320336e74910a940e4d0445371f1e453e4cb34836fa157d778caeaa7f60e7677
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.37.self_attn.v_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.37.self_attn.v_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.37.self_attn.v_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.38.down_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.38.down_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a80f48424d9edd064114db5856f43c23c07da6b4
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.38.down_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a610b18dfeef0da81c53b1fe91e1fc007a715c33b6e218909b0124344f0d0aa8
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.38.down_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.38.down_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a70566463472dba999c6364f4111406b75ca80f1
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.38.down_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e81b6148557db27bb295b8dd079281024bd3513163af39681bf3729dd454ae28
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.38.down_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.38.down_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..255e984606247ea1157112f9ab1d184a08129085
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.38.down_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:748b399ef64f3e6df63baf72cdce06c3334849ba14babe43ddd82b20cd250386
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.38.down_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.38.down_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.38.down_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.38.gate_up_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.38.gate_up_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ded81e3dac554b471a26b8bf7280e020bed1d51f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.38.gate_up_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c0ce31474f0912d257dd3544f0ea9d51f29c869f16d925963e1ff1ec819e5f9a
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.38.gate_up_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.38.gate_up_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a57ff7f5b598ae3622ba4d16bf0422efa59989fe
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.38.gate_up_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:539735a2e1d570fbfb69e09aaeba4ddb7b4559ff986fdfbceef8de29218139e1
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.38.gate_up_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.38.gate_up_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..901f937ddc068c9a1bdeb69e3197d60ac6112816
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.38.gate_up_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2e95716800fe2cf1867bb5fcdacb1b6ef937d82434983766396804e7b065b8ef
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.38.gate_up_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.38.gate_up_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.38.gate_up_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.38.input_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.38.input_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3eec08807577e1779258da895a4c014cc0431307
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.38.input_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc0b650fc2bb41e57ba36bacdddbc3273515554aaa0a0cd102715acc7ef1bf5f
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.38.input_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.38.input_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c5b14aa89d35f123278f182f033fb7e2203af97b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.38.input_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e52da57a62d3a9c8a2fff5d9777a425e6141e12bc9aa106ef49c67e2d3bdf690
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.38.input_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.38.input_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..27ed0ba133aa392d0f44ae70639b90d6c92a550f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.38.input_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:14210d61096286e6f865041cdabe8851e27bf1367f01f194c4b1b51519abe891
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.38.input_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.38.input_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.38.input_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.38.input_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.38.input_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3bc4b3434256e3d407ef2481b964561bace2ac7e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.38.input_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ba9708d969598f859fb6b1d9c186e728d4ba9e10564ca1983c6b324e663f54a7
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.38.input_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.38.input_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..31c149a36e25009a25b28d8b3bd973c7a4b6e8c7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.38.input_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e408f200a5006bc9a5d30d657ee7c70a3cb7a15b4f31968affe34ac7df526158
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.38.input_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.38.input_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..70e79079b9a3cb087861973b05b7eca58c61667b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.38.input_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fa605b71055de621c57d3a626657955b68ac9a9e1878ed5d0e8c99c10007b91e
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.38.input_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.38.input_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.38.input_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.38.mlp.down_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.38.mlp.down_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..97fa33b810aefda06c9340b54d6797e5a5760de3
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.38.mlp.down_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:874193a27833ec595c396f3369f43f5e4d7fd54de01a4a8884555978d3e8a962
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.38.mlp.down_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.38.mlp.down_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..98caebd7b6cda309f6725c72ec9585f4d8a76ddd
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.38.mlp.down_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d176596eae98c49e240b675ec2511eb655c6c6e51fd4c879a97f794ad7be4452
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.38.mlp.down_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.38.mlp.down_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e960c5b9fe8c7f88bbb82b07cf51e04b3ac72ec5
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.38.mlp.down_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:564632003df40147485a4777d2f0d725d36fbb0c341b8c694df891b6c51d3520
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.38.mlp.down_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.38.mlp.down_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.38.mlp.down_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.38.mlp.gate_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.38.mlp.gate_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c150752e652568eafa0c42dde9eed3de6679a87e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.38.mlp.gate_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:88cbde5ddbbe1d1db616604bab1796d6d2c445f43e19cb3d64de4df817220c13
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.38.mlp.gate_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.38.mlp.gate_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..228a2fe5fc40e0b99f58126eb9d9b40cc2ccbfc3
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.38.mlp.gate_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:10629e155d15a3c4a12dd2a47396e23d3d1cb7a61c5304e22fdd62545ab52ab4
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.38.mlp.gate_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.38.mlp.gate_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..acda5e59531a173fbf23f4e2493fe82f4bf308ce
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.38.mlp.gate_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d6a6b6a7bdc28838a24fc96574a527d6222245277cda4bc4697c1e225ec93472
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.38.mlp.gate_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.38.mlp.gate_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.38.mlp.gate_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.38.mlp.up_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.38.mlp.up_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1a8c6434b26a4767337b932b8e9b150e32ed180a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.38.mlp.up_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2d269c25ea803652c03b20370bfe9e0869f39aed06a9f51051cab5f7eb5804f5
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.38.mlp.up_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.38.mlp.up_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9cf420bc5aaed6208d44a47eefd01043cd23eeda
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.38.mlp.up_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1b37097fe8ad5c551b345301ea788716451e0f58a7c3bd21619d33cc6ec97890
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.38.mlp.up_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.38.mlp.up_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8fe1f5f4cd5b8ed7247555afca5cc96c4a2e6ba3
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.38.mlp.up_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3185b30acefa22630f08dbad80be70d9518cf150751ae0d49732a394316e41d6
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.38.mlp.up_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.38.mlp.up_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.38.mlp.up_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.38.post_attention_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.38.post_attention_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a7b50c6e3a4b55a4f410aca3887f02aa603e67c1
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.38.post_attention_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5c1608c3d5365fc848c780483aec4688c6884818b2c2c4a73744dab9842a0653
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.38.post_attention_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.38.post_attention_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d0f9cf325a794c91f7be87373bcbf41af8dff9e1
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.38.post_attention_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:23beff5c3001fc2ae65dc1d52fd1231547871791d896a39c4ee4c51de5d3b7e6
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.38.post_attention_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.38.post_attention_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..33312d71b8e2bc692b4be5b73b1c536fda429497
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.38.post_attention_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f7514b84e49362f4181537fc66a5eb63aa665a7729e792802bacb6acf22b4583
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.38.post_attention_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.38.post_attention_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.38.post_attention_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.38.post_attention_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.38.post_attention_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5e8fdadd86e984e822627e4518420414b15cc6bd
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.38.post_attention_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fc6be7ec30cdd68112088f594883e47cb83d4e7c14aab61d0447edee3f83f3e6
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.38.post_attention_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.38.post_attention_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f2bbb884de2e4bd6bc52a7632be70218088a5cc2
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.38.post_attention_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1c143d48d659e0b5387fcb1131ca68664919a17f761a2427721feaed790bbaa9
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.38.post_attention_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.38.post_attention_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..41334921271e7e6f491f7a38667784cddeaee58a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.38.post_attention_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:324bc58bd0431833ba1623a1ff58b53039dbad084d0583b996a3581a373e5ee2
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.38.post_attention_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.38.post_attention_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.38.post_attention_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.38.self_attn.k_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.38.self_attn.k_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..df286a5ef032f9b4a79c8513c485bac46f993e01
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.38.self_attn.k_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ffce8265c0ca2519adcc486f8da74779e37c76c33460f133f6b56c538a6b49c5
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.38.self_attn.k_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.38.self_attn.k_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..13fde3b3e6c6ac246167500894aa52610c6774e0
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.38.self_attn.k_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1e8c2cd4c2751f69ec1ad16052e70eb4789ba04b2af71ff41c99b171b3fd09dc
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.38.self_attn.k_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.38.self_attn.k_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..54d834f730a22ca5860e4e8b9237940986713226
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.38.self_attn.k_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0e8cf848c624dea40fcef51c5bd86e857ba6ff6cca2ee6745c9217a13cf44443
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.38.self_attn.k_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.38.self_attn.k_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.38.self_attn.k_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.38.self_attn.k_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.38.self_attn.k_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..67b2698d1acde6f9f6989a0e8c2ba5d348adc184
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.38.self_attn.k_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:12e00781b40b5642c7e15aaa52aead42413b6e99f283d37e6183f733d052cb1d
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.38.self_attn.k_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.38.self_attn.k_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7d122bc6ba2d4d977bbec38047e273dafbd6062c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.38.self_attn.k_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aba336a04e315e74aa7a8841b8167f44a775cac00c7172e781af017e28bf860c
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.38.self_attn.k_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.38.self_attn.k_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c800aa417411c6baf3012b50659d1d30d9ab7a4f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.38.self_attn.k_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5e603b3dadbc906cd8fe9b9570ba0a79a41845e3e18a16e5c607fd9e387d5cb4
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.38.self_attn.k_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.38.self_attn.k_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.38.self_attn.k_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.38.self_attn.k_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.38.self_attn.k_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1abbacf1cfe96d4807099e4f0d28df9ee9894ad7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.38.self_attn.k_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:601df17f5f3984577e25c70640f56a27eb9b5ff62dbc742693eabeaba7913305
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.38.self_attn.k_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.38.self_attn.k_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d85df6afa4393ce202dbb2e9d017133a886721e7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.38.self_attn.k_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7cb1855b435626bbffdef1d5a269469ae931b931c6f4b8bb83e028e9f0c03900
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.38.self_attn.k_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.38.self_attn.k_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e6263a95c6e86aa5bba1bfd5ca2916e648bb79f8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.38.self_attn.k_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8c3650f8140ce581439979dfe10fe46773243d820b5a9b401078917e67e21918
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.38.self_attn.k_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.38.self_attn.k_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.38.self_attn.k_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.38.self_attn.o_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.38.self_attn.o_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6cf90887d69c81a0d25c1fbbf506cbbadf5abd6a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.38.self_attn.o_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fac1ce4e5e009f0e5e532d3e1a193fbe413d9d7523e23f1f5bbad7b5ebabfba8
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.38.self_attn.o_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.38.self_attn.o_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1dfa3cc51669ad2aa6947420d1eb7573aba17e00
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.38.self_attn.o_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:008f00fc0498699b63b69e0b9c806665c2a49f0ce3860dd5ab2256ceb9057310
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.38.self_attn.o_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.38.self_attn.o_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ab406018f7f9caa6c157f9ba8039503bb4fa01bd
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.38.self_attn.o_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fcad392b447ecf0da615005d37e7022ebbc15b6771b8262c2ca25437dfc05ee8
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.38.self_attn.o_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.38.self_attn.o_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.38.self_attn.o_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.38.self_attn.o_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.38.self_attn.o_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0504c873d87c41ebcd60ebd8266f6a54fe78ef7f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.38.self_attn.o_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e931f426b402371244b340ec9c60523d01ed09578cc7178ecd3cf90cd1dec868
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.38.self_attn.o_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.38.self_attn.o_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a51bd9c6328e79b4800b7534d4991043fede8128
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.38.self_attn.o_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc598a655c96823c9e1e2f926ceb0572c292db90dbb955fa44fa761bf601564f
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.38.self_attn.o_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.38.self_attn.o_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cc836b9431b4c51ac6a1d282f224628cbe366455
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.38.self_attn.o_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:62b2297dc1556ec52777ca32ca32bd0dd50e3db03dd1e27a479cf86fa651399d
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.38.self_attn.o_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.38.self_attn.o_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.38.self_attn.o_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.38.self_attn.q_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.38.self_attn.q_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5b6ea35a21110462c4265a2f47c66fa3177f67fb
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.38.self_attn.q_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6d14972c13fe173fd15f7a0fea3a476cd196c84d16d05f5559d00b4b415757a4
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.38.self_attn.q_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.38.self_attn.q_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a4bc335361509a2b75bc472aa10af7b58a080c72
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.38.self_attn.q_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:14e8ee4ef6a9d84c37e9937ed2c6bfa98c6b79870465d9020b24218853a696f8
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.38.self_attn.q_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.38.self_attn.q_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3d2fd11cdf1d28ad7f325eb740a27f2956a1d065
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.38.self_attn.q_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f4d93b32b7b5e35c23e95270381db83a7fa041e804807754a0a5690fc5deafdd
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.38.self_attn.q_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.38.self_attn.q_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.38.self_attn.q_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.38.self_attn.q_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.38.self_attn.q_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b14090b4c4fa27087b99d522a514c8a1d82a8723
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.38.self_attn.q_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:79180d1d75ad2ed117dd4c0384ecff53b14b0c5e4c689303c958a71452e5dfe0
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.38.self_attn.q_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.38.self_attn.q_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fba57df194e53b26f85bcc3f92582ce8069ecb6a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.38.self_attn.q_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:501b4d1dbed99e5a89a0e543fd91e6522fdabd2dcf735bf3f0e6144f973b52b2
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.38.self_attn.q_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.38.self_attn.q_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b7467c52bc5266fb17c36f38199365e9ce933820
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.38.self_attn.q_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:068d3e04d2b0ab47c69a2c1db499e3bee94b0c083e9ac0830b25de02f37d07a7
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.38.self_attn.q_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.38.self_attn.q_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.38.self_attn.q_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.38.self_attn.q_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.38.self_attn.q_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..320361dd73e459514a40977688202f9459cad8a0
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.38.self_attn.q_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9f79c0e309799137b0bab3f0b85c0e6f11755dda864322f1850b1601205293e5
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.38.self_attn.q_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.38.self_attn.q_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4af16bf6ac4b807c7f4237a84ff12d1892b12c85
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.38.self_attn.q_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:59c289abba40ed89744e166f63075ada62a112d752525cce83fa24c464dcc43a
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.38.self_attn.q_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.38.self_attn.q_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2fdfd2384f6404074b1028641cb3095e9b669ec7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.38.self_attn.q_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b99d387cb082b8c6e477ade979d7d3a0376c75d57b72dedea32686a6daad5fc1
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.38.self_attn.q_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.38.self_attn.q_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.38.self_attn.q_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.38.self_attn.v_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.38.self_attn.v_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3174f1b3d8d0d6ae6fdb73e37dd367c0f874dc53
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.38.self_attn.v_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9fd4b4d4b8fdd4d4e7c02d8229ceb8cd262d6c5325c97bf909d52116bdff7f18
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.38.self_attn.v_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.38.self_attn.v_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c8301d7f12d7581a64cf7dd08ded698197015e0f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.38.self_attn.v_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d1f1ccfb7727b9bb451eeb6232148ded6149c67cfc152fec8e4fe25b184314af
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.38.self_attn.v_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.38.self_attn.v_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..844e950216cf2e29a0e04a75d57071560bbcbecd
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.38.self_attn.v_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:53848fa46921f090fa6a3e34bf8f59a0d9bf4aeb0f85c35276e6eeb28c44b48b
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.38.self_attn.v_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.38.self_attn.v_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.38.self_attn.v_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.38.self_attn.v_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.38.self_attn.v_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7e7071e6bd51b0fd989d5055ec9f34dde15777cd
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.38.self_attn.v_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5e2cb7b6b0cbe60107319375113393e19dfd01ac0cd4b9c8d8c7defa6df6c680
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.38.self_attn.v_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.38.self_attn.v_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..253ac526ecff9c9960e0753acfb80c2695279bc6
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.38.self_attn.v_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b3e513afbdb1cc329918393b533e79ca2fae1fa25e7098dfdf71109102dfcef0
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.38.self_attn.v_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.38.self_attn.v_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a5b1d125fb5d0f38976de3b88762723d44f3296e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.38.self_attn.v_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a9db3a3b477d6b00447d0b3b52d4c3694aed5291d0dd013f7615caf93ccc673e
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.38.self_attn.v_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.38.self_attn.v_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.38.self_attn.v_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.38.self_attn.v_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.38.self_attn.v_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..334e38a1804828a2b859e74f97aefd44f3f2e70f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.38.self_attn.v_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d0eac810125d3626e6574377ae7d1e38283c2508d6a4434341b898a1b311456f
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.38.self_attn.v_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.38.self_attn.v_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..73251014999a353f6de6a5ba80b36f7fb9fc1c00
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.38.self_attn.v_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c8cbb103511b53c4ece330fb59b8079dbb9cbf6f1337c8b2e264dd10722bca22
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.38.self_attn.v_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.38.self_attn.v_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a14cb0eb287b41f049c8b3cfe9f852bb6ab69d50
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.38.self_attn.v_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e9d592d41ee1510cb24a5d8256f46e66d97b7efb5b25ed311aed211d84080b1a
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.38.self_attn.v_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.38.self_attn.v_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.38.self_attn.v_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.39.down_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.39.down_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..142b8bd4d77e2da6e4d05759bb8dc83127a6efc8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.39.down_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:adb8365019b5c0dd6868ee51927e545e51a400c5a129ffeb044d1e19d7d2c878
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.39.down_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.39.down_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e0dbcdebf61fff8274cefa4f5afb873e3eaf9b8b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.39.down_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc1d76b08b8493ad62368696cbac18b10fff47e9a3d4aaa434ded13f490cfb96
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.39.down_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.39.down_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e0180ac2eadae16d7fc4b11b489b22a4669346cc
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.39.down_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f86325e06afa83cb243d74c8f04a4c4f6a4d081169d263a7783cc856b5c7e4a3
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.39.down_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.39.down_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.39.down_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.39.gate_up_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.39.gate_up_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..63074fd78a71abdb387fba22f20c9d37ff43e735
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.39.gate_up_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ce01cec1083cbbb43582bb59c5a10e70e46ba8b17df2751769a96164d0c572c5
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.39.gate_up_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.39.gate_up_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..37e2068f082d5a185ee3f31ffc7ac358b7e9e9da
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.39.gate_up_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ff27bf205911ac180c67e9f9e3d0e7e535b3f83a1c2ee5296493ff5ead1d4b7c
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.39.gate_up_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.39.gate_up_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cfa1de91e092d209d724b0644054aa7909ccaf42
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.39.gate_up_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:85d1ab786285ba360432ad0ae739ba21eb2193b10d89c0ddf6341e5063ad4199
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.39.gate_up_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.39.gate_up_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.39.gate_up_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.39.input_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.39.input_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e2063130b9ec43a05240af827ab82f099554836c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.39.input_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:519c8f3abaeda27ff2d77ab2f873f8053712d27bf5db77a453c80a885c22c001
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.39.input_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.39.input_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f1646dc64e44bbb94701ca9b0661833ebc7d9ab5
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.39.input_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:05aeb477ee16028464bd6df68a64b57605ceca21dfb353b6d301b39986845e14
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.39.input_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.39.input_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ff1edaece3c4ece31df945065059374b53f5512d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.39.input_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b9a5b0dd4e54a18838bb5f23ee181855e1a0112097acd1d30e26593eeadd1a7d
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.39.input_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.39.input_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.39.input_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.39.input_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.39.input_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..83c56797a1aabbbdc7e861298aa98360653a004a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.39.input_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:76f7e37972653e544188db98ed0dd7d5268dd034b15be0c0d658a4db2b9e762b
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.39.input_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.39.input_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c3f678e5f634acb066d1988eb22555dfb988f37a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.39.input_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e5001c541efe92fb2c6949a374b8708b57dbfc8b1864435c78443795852a0b1a
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.39.input_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.39.input_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0b0c80c7df274cb4aef7744b1b676f1a856b5a69
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.39.input_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9974efab360924c44470ad3f6b0ba964df3866dae18b55308efbb21cd8880941
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.39.input_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.39.input_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.39.input_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.39.mlp.down_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.39.mlp.down_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..953c9c52fc8e0e362ac55ba62bfc567886c06ca0
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.39.mlp.down_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:81697cdffd74225b2c182e8030e7186af0c207272b36bdbd3b0d299bcad646ab
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.39.mlp.down_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.39.mlp.down_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..39eb4f20830f45c72c376deda182138f82c8d5b4
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.39.mlp.down_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:079d13e00a9f3b3ea9880208d031a3ee7ca75f9b5009bc5341c1bbffd4298303
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.39.mlp.down_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.39.mlp.down_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1bc0df492c28e0d1ae2348a1d974aecbee045f18
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.39.mlp.down_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4b8afd3869de71385f3bffda538f662c576ca1237e0f479ed974535be1e8aff6
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.39.mlp.down_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.39.mlp.down_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.39.mlp.down_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.39.mlp.gate_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.39.mlp.gate_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..25ce3640449a118a765a707f6bf4c2299398a958
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.39.mlp.gate_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bdfc5d8bddb535e3d661026fe24b60ea2360cd4060585aefe7142b17a9694ae6
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.39.mlp.gate_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.39.mlp.gate_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d5ad75524c22842b72419a0411eeb38a1cc2a044
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.39.mlp.gate_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:37f2bea968607da3465df3cb400534f08d448ed136b128fdd3a54b1a776394ae
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.39.mlp.gate_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.39.mlp.gate_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1d58759c449edfae824cc3d0e342383e6c3559c3
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.39.mlp.gate_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:89ac61f624bbe33f0f0fbbf949724d2c179990244f92b51abaaf184e5dd5fcf3
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.39.mlp.gate_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.39.mlp.gate_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.39.mlp.gate_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.39.mlp.up_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.39.mlp.up_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1d15a0dde022696f853ec1b036860c03b551cf42
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.39.mlp.up_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8b2d894eaa867645185a28b6e65eb663920ae3f796f9abfdb86765298e0c7d0a
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.39.mlp.up_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.39.mlp.up_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a0fe2c3205c9448b5bad31fcbe4e45cd2f2db1b3
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.39.mlp.up_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b93ac28ec6b2163ce4b23de457b1cd4baa89945646d1bb1a6d7d449f121003bf
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.39.mlp.up_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.39.mlp.up_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..57f1af67d8cac3278507f5ba8da6b1c8b9c97d59
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.39.mlp.up_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ac985524e27acafda908551fbdc3a31afcd1ef83963bc99c2ed184fa4be6b179
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.39.mlp.up_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.39.mlp.up_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.39.mlp.up_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.39.post_attention_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.39.post_attention_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..600a9b995b04c675b4b48303d5a44ec51e68794e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.39.post_attention_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bcbfd91b486a7b0448bb282e21df25641335580a72f02264659d39edfb3a48f1
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.39.post_attention_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.39.post_attention_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9065927669c917b4571e3a9a7e52e7512d5081a4
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.39.post_attention_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:df0be5ac905cad146b01e25ba8f711df6ac3af59e4bb3880db60f8a835e14e5a
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.39.post_attention_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.39.post_attention_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..01994041d4c6676528ddb7517b6fc1c18dfcc81b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.39.post_attention_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fae35a907626a146fc5aecc6e64858f0141c126e6186d088f49165845597fd28
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.39.post_attention_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.39.post_attention_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.39.post_attention_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.39.post_attention_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.39.post_attention_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d6dd6ae49290f16651ba6749e9cf0e2d1c0a6a92
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.39.post_attention_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9ce73a42bad1e08cc864ff96f71cf9377ae04ef4cee1e975744de95f6ed77e2c
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.39.post_attention_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.39.post_attention_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2fd65c29627d5e9774d1fb09edbf91e4a7cfc1d1
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.39.post_attention_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a9969d3b19d5238a366e1586ed90a322b2d10a6782781e276aee44b804df92df
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.39.post_attention_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.39.post_attention_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1d020d147978b94e49768577953a31ef36679807
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.39.post_attention_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c32eb51b8e6a29d6e15384bbe511f9168d0277b62b86913a059f71741b18f4d4
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.39.post_attention_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.39.post_attention_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.39.post_attention_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.39.self_attn.k_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.39.self_attn.k_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..13bc8ac076f282bb6701b67ac68734bc62558125
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.39.self_attn.k_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:41fd90690a40ca28f264853212730da6ea4a66f98c274726993ef8ad8a7cff2b
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.39.self_attn.k_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.39.self_attn.k_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1fd8827a1c12d316bde8d3952af2db9067311e9b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.39.self_attn.k_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e1763e85e6cd93e5a58241a2510d3c059fb9d41d8c12e23a90da354de94bd20a
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.39.self_attn.k_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.39.self_attn.k_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..32244f3ae65a3ed9c40b28cf0a63fda3475402c7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.39.self_attn.k_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8ce82355c33908c8fb6adecfcc0f6700cf0dcb1b10fb5120b8681c7135713c81
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.39.self_attn.k_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.39.self_attn.k_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.39.self_attn.k_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.39.self_attn.k_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.39.self_attn.k_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..44384323e2cf5164d6defeb5f61524818cca5ca1
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.39.self_attn.k_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b40357e16dd11ef61ac799c2bd1af438666acf69e50377a9049c6b60b9eee4ca
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.39.self_attn.k_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.39.self_attn.k_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4cedb0b820446c6a24676cfa614efc9576eb80b0
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.39.self_attn.k_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4974d17a9bf27c62387a8a38fba8c37fa76b03e3f283cb31b06765ddac115c4a
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.39.self_attn.k_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.39.self_attn.k_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1edac5d26dd4abeb12b8c5cd04c0eb6370df332e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.39.self_attn.k_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aed880dbc4e08e18016897ca242ec0e96374c32ada86dd02f38ab49dc76168c9
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.39.self_attn.k_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.39.self_attn.k_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.39.self_attn.k_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.39.self_attn.k_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.39.self_attn.k_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..bac4577386fe3bb0188d83a74797c9fe0d7f25ce
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.39.self_attn.k_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:19679481b7840214eb8b443e7951dab39240e31b96a4fe833660701979021bec
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.39.self_attn.k_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.39.self_attn.k_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3219aa40e7877c68b38759175a131494c622d045
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.39.self_attn.k_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9987fc95d09165861c151058329717e41239d612474dd1ab9601a83cbc6b84ef
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.39.self_attn.k_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.39.self_attn.k_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2d4c4f950eab35ce0bd541b79bc8f95fc576eec7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.39.self_attn.k_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ff963a730053e6666ba7b099618cc7626db80d5ef329fd89b113c8f39976006
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.39.self_attn.k_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.39.self_attn.k_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.39.self_attn.k_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.39.self_attn.o_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.39.self_attn.o_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3cec55e80ee8065dd4f251af46dd88cfe1b0ee33
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.39.self_attn.o_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0c0c7ee49b5b9e462a36aa8a686f28b2af600b8e868017842502dec19d78d395
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.39.self_attn.o_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.39.self_attn.o_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f5f52d33eeebba46a5b46bf0305a0ae6ea84016e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.39.self_attn.o_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca82a0442555c34b9e4e152d472bd2dabd1ef41247e5531dfbeb1845fa208b84
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.39.self_attn.o_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.39.self_attn.o_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fabdb83100045988a4d79408c9ad1a7322d5bf0b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.39.self_attn.o_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a8e7d9c6161ff774633b210b3dc0cf2fbaeb8aab8aff4193d3fdd623f65e1030
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.39.self_attn.o_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.39.self_attn.o_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.39.self_attn.o_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.39.self_attn.o_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.39.self_attn.o_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5c2d90a7ea8da0d90c7c140449e6a78a0d4cb7ae
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.39.self_attn.o_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:43a29ba5603e0002eb3bbfe13b295fce8a82bd829e84c8d8ba4c8bd6591f8763
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.39.self_attn.o_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.39.self_attn.o_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..009b21f315007b0c04cca93f3dbd5e41d07f91c7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.39.self_attn.o_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:442828b8547523addb9b065bd37c4c4ab54587a3ed4ee39121b945bccc2c4067
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.39.self_attn.o_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.39.self_attn.o_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..bc47892b9b7b11f74a1d91ced5b6f29b950ff337
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.39.self_attn.o_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0f460f4ddeb5a929772643ae196f274998190a1c98e90c915f907a3336b57c47
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.39.self_attn.o_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.39.self_attn.o_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.39.self_attn.o_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.39.self_attn.q_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.39.self_attn.q_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4aec3084631159e1cbc1d0a0082e37c0c22cdf3a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.39.self_attn.q_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9eb3b80161319e2eb6f3df9b73d1bb187b914d69102959d296f7428c1788f091
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.39.self_attn.q_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.39.self_attn.q_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9786ce07fd3f74e617854d813750c3e17bdc22e0
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.39.self_attn.q_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:72bd30fb1055d761e7e0121dd999a74bd0f2bb4ed1f2e3029912c9e240b946f3
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.39.self_attn.q_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.39.self_attn.q_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0575efce45ee8e1768239430de766ee003e5eb2f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.39.self_attn.q_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8167681c687270b0b7861cc94a0a2ac13b523fbf8cab3618e0ca5846d31769f3
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.39.self_attn.q_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.39.self_attn.q_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.39.self_attn.q_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.39.self_attn.q_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.39.self_attn.q_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ff7a6bd114ee5284eb70c4a138d3e3efe62324e4
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.39.self_attn.q_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:de801c65eb479d5c9b225ea0d36352e96eaf1744b6b70191b26d671ebafbde7a
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.39.self_attn.q_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.39.self_attn.q_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d1890f72f5c1f2df616eed028c3b1fe634908414
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.39.self_attn.q_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:62429acf27b722eb5c743e12430f158abcf2e0ef5e4da9794ef05a559e0e3e41
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.39.self_attn.q_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.39.self_attn.q_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c4e4b6d4bfc18afc296a0224e97eabd253101c8d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.39.self_attn.q_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2a82a30d84f7f86ede1b0410890930876ac2a779809ca0efb4763e70c6dd0a95
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.39.self_attn.q_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.39.self_attn.q_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.39.self_attn.q_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.39.self_attn.q_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.39.self_attn.q_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4c220eac75230c363a957da1454ab2385512f062
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.39.self_attn.q_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:935081d37da09f041ac630acec1214b49f7e3d0ee33209f07af8e22f48e482ff
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.39.self_attn.q_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.39.self_attn.q_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0731bc880e57e3d0c0abf1a2208b8fa6874c2d40
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.39.self_attn.q_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:03bcd23301f72cc5949cca5923769559b1fed24fb8553fa2878ee6cb94709eba
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.39.self_attn.q_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.39.self_attn.q_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e49ad0420f65a53c92ee4b3d2b4cbfab0b560765
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.39.self_attn.q_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:37549bc68fba738c64d8ffa573182c9d39ed1a2fbf35bffd8c5de981183ebc19
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.39.self_attn.q_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.39.self_attn.q_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.39.self_attn.q_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.39.self_attn.v_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.39.self_attn.v_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ee025629821827cdf267980ea827ae8730a4e2dc
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.39.self_attn.v_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:962b905e2016aac73b441371c722bc33c50b0d47ee603a547d2f99d0190a2232
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.39.self_attn.v_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.39.self_attn.v_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..414c7f8e86578e770de38408b7f3a26f5609f49a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.39.self_attn.v_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eaab2a5185d815bec2921e832de5ddfb605a57c48549d9160abe21e3b08d15ba
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.39.self_attn.v_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.39.self_attn.v_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..30ec6a62421c4b65b03d4a346e28e3ccabdf798c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.39.self_attn.v_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f0e6d7453df579035bcd238edc632bc93a85804a9e4d384fb30e77ff456a52f0
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.39.self_attn.v_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.39.self_attn.v_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.39.self_attn.v_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.39.self_attn.v_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.39.self_attn.v_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..41b923a2c17e4abc3d81c9f24def98fd274ce217
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.39.self_attn.v_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e73438db3b70061b89337513673cfafc7c878ad8d5cb0371df179baddb20d48d
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.39.self_attn.v_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.39.self_attn.v_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..14f1f8b455668872b8a4985e67cf094a3a4eb211
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.39.self_attn.v_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5cac2f9f26dc64418f296b9f475536649da680bba54f1c0fcbbae63a7a516619
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.39.self_attn.v_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.39.self_attn.v_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5ba83994f92cc9b366f9c215a007391d899f7af9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.39.self_attn.v_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:18d39455821378179685e888395393a0477f8a3a1fa44ae45e0ccbb0d34f0687
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.39.self_attn.v_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.39.self_attn.v_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.39.self_attn.v_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.39.self_attn.v_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.39.self_attn.v_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9ae93b7a4e59084383e70b06af2f043b2ccc8cd0
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.39.self_attn.v_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a42b8ebd5bcddbc7a96525eb2dfdb1ac48d629b40285dd70e5645f54efc1cf1a
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.39.self_attn.v_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.39.self_attn.v_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..854f07003c558ea3408bd6c0182aad333f116df6
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.39.self_attn.v_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b014ef6a28d8acc45a43f19ac6808237a9f41e7d5b1c4bd99209ed3f661e9cf6
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.39.self_attn.v_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.39.self_attn.v_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1780dcc8aab96ae496a391903d5e8577ca49ce25
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.39.self_attn.v_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f6528e7662eb1fb6a0141fafcaef43e3c75287d0d73a7498a171b6b206cee2e3
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.39.self_attn.v_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.39.self_attn.v_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.39.self_attn.v_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.4.down_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.4.down_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8eda00cf314c003947ac6c57bd0ce3070db6ff43
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.4.down_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8bf7a1b4d7c647fe31ac5ecdaf9df92ec844b668a0c25c101a853bfe890d2ad0
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.4.down_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.4.down_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fd2548cccc427d9f75b565528c8043a91e428728
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.4.down_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f4f952e030eed8e5f2b7f036c6d2718b7e3fbae39f4ee9bdaecdcee73e63f06c
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.4.down_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.4.down_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7051c0f4e261ffa928eac1f8be6a3c3b1c37de21
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.4.down_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3eba6285923ce631840abeace757e9a34a2b4a2903b538edaf82e87dc45b27f1
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.4.down_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.4.down_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.4.down_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.4.gate_up_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.4.gate_up_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..752083220366012ba7540caffc3f4cf444e16e25
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.4.gate_up_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:28c34eb5dfdc329ab64319efc752170624f61884977fb7e635c8ce79f1704372
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.4.gate_up_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.4.gate_up_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ec10334a621cd576fbe325e9b713ef02f8271dd6
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.4.gate_up_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:42c072e9cce8b9da093af8c8a629c9d9351b8b2c70ffe8eb1cbd72bb249bd1fc
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.4.gate_up_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.4.gate_up_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..303e289b47236ba2b6411d143487b8d9e9bba752
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.4.gate_up_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b776ed683080b6d23c9bae0c4dd060ee5240019e161bfb8347d58dfdac4abcfc
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.4.gate_up_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.4.gate_up_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.4.gate_up_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.4.input_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.4.input_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..79080e4ae2535225169624a92f7b41aa97566247
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.4.input_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d6e846a60d03a9a1d8f8f39a8e3e6a77f5d5e920f711416f7c4ce47357a90a42
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.4.input_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.4.input_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..317a5593bff14185219676fa717cd8976be3491c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.4.input_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:21cb0f7d79abed38424c6815c44d48266c43d4ef2525540ccf9a8b8830a7188d
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.4.input_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.4.input_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..88327c93976a40dc60ce98fc926846b39ab83391
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.4.input_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7f89d83e86e3d6e98e0c60a61a3243691d033e61e1a9b6a368b0a329c967ef1f
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.4.input_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.4.input_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.4.input_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.4.input_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.4.input_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..29429776f5851d606f2111efe558bb7dda01472b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.4.input_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cab2ce900eec2ad64c0be3400f1c8bbdb30b12c72205f39b0437cfdec78da550
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.4.input_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.4.input_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..163b4c6393ec0c04a03016bc01540ffdc4582453
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.4.input_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d687199d4eda7ae1a6502468d22737eb9e9965180e58d9d72ced1cf697b3c0ca
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.4.input_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.4.input_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..159b521240aa8dee864af92da7d0e7104b17143c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.4.input_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca4faebb1c06c446ac83cb9088a2c7bdb8736fd675399bc78024f7fcc1619d07
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.4.input_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.4.input_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.4.input_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.4.mlp.down_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.4.mlp.down_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..81c9d9a4eb7ff2aeb8d590eca2f3e723ccd2dd45
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.4.mlp.down_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2a3d9903fde15ce10012c422b3803af8e14952ca250bb7693f6bd51142e15229
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.4.mlp.down_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.4.mlp.down_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..56404f78f2226526d31643e85482b2d72c9a6950
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.4.mlp.down_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:80424ecfa05b482862fbf25d4d06135c5ad29720b4728d041dd6cd8031e287fb
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.4.mlp.down_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.4.mlp.down_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..45fe6b232da2e92c29b739a259835f9352ff7191
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.4.mlp.down_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ec4fb8b982934d9745cdd20c62d489189d4afc78f15c63ddc17705b3045b99ca
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.4.mlp.down_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.4.mlp.down_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.4.mlp.down_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.4.mlp.gate_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.4.mlp.gate_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..abef6d2bb6212bcf84bf72d8e4c77181c3f187e7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.4.mlp.gate_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4b6763c786df27c83f6b55f8854741092c71d37d91a1914470c252fc5c00e9b9
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.4.mlp.gate_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.4.mlp.gate_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7c8ec9e61906c03c7d083639f133193043b950dd
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.4.mlp.gate_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4155c7b8292fedcbab6c6d57e7d03bc24565e37b651c287a4eb5f23dff101289
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.4.mlp.gate_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.4.mlp.gate_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4e560aa2d232dbfd181c056385166f114f347a62
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.4.mlp.gate_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:28ea79b8ddbcfcbaf40cfa7fef585d87cae12b004c35e89f7807b5d5879774d4
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.4.mlp.gate_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.4.mlp.gate_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.4.mlp.gate_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.4.mlp.up_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.4.mlp.up_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3b28e73931fe9cd0069f9633859f8d2a7f67cd93
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.4.mlp.up_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:24673107389708df9c6e794bfcfb45ae3004e96cd2ed878b66495dedb20e4a6b
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.4.mlp.up_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.4.mlp.up_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..39e5c0c12eb2bb90a3ee9ed5266c6ef0e4a8ece3
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.4.mlp.up_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f34478b1cdd729ec987fd4e6bd9026a900a0b0a6d1145b641ace0d9192e5d2a4
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.4.mlp.up_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.4.mlp.up_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8c3a39333d96f678c6deda141b92e5a2b44310e5
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.4.mlp.up_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c30dedb04295a01a4e89efa78d63a623118903a7ef8978453bbe0407cc8f7f85
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.4.mlp.up_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.4.mlp.up_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.4.mlp.up_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.4.post_attention_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.4.post_attention_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..baed790ab234944433a868cb0248306f480f826b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.4.post_attention_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:58361e2b1bed5d3db13128841fb53b527b812c057aa8355de33f197b21791ea5
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.4.post_attention_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.4.post_attention_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4550ee0a287f3d16a8190a57d51232819ca0bab9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.4.post_attention_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e4ec05dc2776cb1ec45b31d7114d0493114911b0380d8b293cd554092454f1f8
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.4.post_attention_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.4.post_attention_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3748598016a58191bfb79824289ffdf16bd44dfb
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.4.post_attention_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:20b8b83510e72b8e3050ab3d87ee4764701bf3f63449bfc203f71eb1f71639ed
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.4.post_attention_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.4.post_attention_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.4.post_attention_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.4.post_attention_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.4.post_attention_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..eff98293ec3e91964c647df507eb63fb14af6078
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.4.post_attention_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4177775a8cf2e6527d947877420ebad0c5a3794133749dbb8598059833ffcaea
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.4.post_attention_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.4.post_attention_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2cd9e449622bd1e492947aa42871d244a4df2fdb
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.4.post_attention_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:16a2c82ffa0b6cf8eae82a9deb075cf55c89a89bdec50f6007dfe4a180dcf77a
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.4.post_attention_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.4.post_attention_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..abe424e08f337f477bc661611821f3e36907e6ab
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.4.post_attention_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:10815eaa4e0527894e56e791fc556ab4e6ecdc6736368244ae43ad42f75ae505
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.4.post_attention_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.4.post_attention_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.4.post_attention_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.4.self_attn.k_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.4.self_attn.k_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a8239a5fd1508b12f6f360e8cb6ad7aec330d125
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.4.self_attn.k_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2ad9c4c0e231eeefe35019d38dab20ba7e227c17f3903d689f34e0c35934ca62
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.4.self_attn.k_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.4.self_attn.k_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9975cc229dfc9878248bfd8c5cee6e3514f6156b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.4.self_attn.k_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ce1ec29b7d8b5186fd1ea56955bb6aad03b56e911a3564b3ee3e2f2ef3f09c81
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.4.self_attn.k_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.4.self_attn.k_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..628c77784afcb5e4312311e9888174a2c575b46a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.4.self_attn.k_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dc687c8ad139ad22ab69bf84c616e6daab9eb2bd74f57c0165e450f986b831f3
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.4.self_attn.k_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.4.self_attn.k_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.4.self_attn.k_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.4.self_attn.k_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.4.self_attn.k_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c5a9ff06bceda12fbc612ae86ed27b9f91bd89e3
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.4.self_attn.k_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c2bd2b975abe8e22f22ae1706ef4ac9517171a2a42e3192f834a27962507c7b2
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.4.self_attn.k_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.4.self_attn.k_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d6a8b50ccf2cb209ff6b0b4f6e61c258ce55f735
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.4.self_attn.k_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:142656a6afba84fcfdfaf6ecfd3fc049fc43c0d1bbcc9668b3b654eef61096e0
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.4.self_attn.k_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.4.self_attn.k_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..004daac7bc7340175ebc3e957f726aae0964f241
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.4.self_attn.k_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7016ebe3970fa21cc8eed980a76cce2796eb32d36219fc0615735b4b60dddf1e
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.4.self_attn.k_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.4.self_attn.k_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.4.self_attn.k_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.4.self_attn.k_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.4.self_attn.k_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3f6d3f3b46d7de7bc97d33d40b377ffe9e091991
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.4.self_attn.k_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:820d580c0cd7464bf551a26a0706d215e1a77d005adaafd65e7b07952c4efc40
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.4.self_attn.k_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.4.self_attn.k_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7ab9421b0c3370837898710d9511dd7a7308b86e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.4.self_attn.k_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:174fd8aee66ebf715ae25797be0f3b365ac8707b8263f0fb46a1263f5d20ea78
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.4.self_attn.k_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.4.self_attn.k_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cc24eef624efbe307c8f62b393170d25eef0c62c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.4.self_attn.k_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6791649ca250ccde246702cafbad8e83410120d77c6162f0317dd8b3e9df1e90
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.4.self_attn.k_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.4.self_attn.k_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.4.self_attn.k_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.4.self_attn.o_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.4.self_attn.o_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..58e11bc59f5115f59995dad66b27d46507510e10
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.4.self_attn.o_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:59c80825141fdad1b17da515e79b6b2cd52e49b8e24add6301a4263129aecade
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.4.self_attn.o_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.4.self_attn.o_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1fe26307b90e4f3889d1f968c2bb0aeac6e35942
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.4.self_attn.o_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0f04e00d89c765b11bd3986fa995d6184f123d34d8468b5da2ed92b39769ee05
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.4.self_attn.o_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.4.self_attn.o_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e66d53b9fb925bc692dc0d8531092e5378f8cd3d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.4.self_attn.o_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8822aebdb1c2d1dda1cd355c644389437c96c50a1e58f21ea665fd285070f42e
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.4.self_attn.o_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.4.self_attn.o_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.4.self_attn.o_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.4.self_attn.o_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.4.self_attn.o_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b741560f08eeccd90a4b4262954b4898adf5220e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.4.self_attn.o_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fa992beee6bd87ac5f772603038dab91d1f0eb2ca9ebd53c5ba515424c28a073
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.4.self_attn.o_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.4.self_attn.o_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f01d7034e9f864945e5453aa99121f29ce3be820
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.4.self_attn.o_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:45c6fde2d40b9534dccfcb1a284758faf2aa28c401daea867df405586ce9ef83
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.4.self_attn.o_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.4.self_attn.o_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..243892e05594bb209f8fc83336e1d07d2222efcd
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.4.self_attn.o_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:578c12d292ca46603c47c1c925ff5967a2edbdafb4ff02976e03c786cc14063a
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.4.self_attn.o_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.4.self_attn.o_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.4.self_attn.o_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.4.self_attn.q_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.4.self_attn.q_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fab286c1724c0d9eb63d9a1a81c87342c46c12fa
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.4.self_attn.q_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:48c87d18908965961e044defda6df920b23a39b0b3d126af4a4ac77a62282c31
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.4.self_attn.q_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.4.self_attn.q_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ce5752ffcb1c190031fef87bfdb1a8f4863f5ce4
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.4.self_attn.q_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f4920e9b9b5218b0797591bdf39a616ea4acf933820cc7eac6ff63d302a1aeda
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.4.self_attn.q_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.4.self_attn.q_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2b22d97b84d55a46264a6364014ac5260bf3a41e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.4.self_attn.q_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2a89a4e51e773aecffb3a5d8072a4ecb9d88f06d0ec6726100e77860d6f98103
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.4.self_attn.q_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.4.self_attn.q_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.4.self_attn.q_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.4.self_attn.q_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.4.self_attn.q_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..dbb2f381aef2e42cd2ee8582e1a4878f7106e2a8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.4.self_attn.q_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3306d6ce09e6c2f9605f869672813287ab75a13e7896c3e41668dbe00fbe7d10
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.4.self_attn.q_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.4.self_attn.q_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1262308208b5c5cfbeba5bc0dcbcbbfec76e888a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.4.self_attn.q_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:35270d6c7d454adc6367dad9bbd6e643a97cd88c1fbcf448cef1ab0623862ee1
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.4.self_attn.q_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.4.self_attn.q_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..37ffe879391f3025f52906303d55d58d7a578278
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.4.self_attn.q_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e5b180ff9ede61e6a4267c1e7806e706056543f8d14d3aada7a6ff9f082f17cb
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.4.self_attn.q_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.4.self_attn.q_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.4.self_attn.q_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.4.self_attn.q_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.4.self_attn.q_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e4b60a6f92f26ddb2b823e8f521dc6498485134e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.4.self_attn.q_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:000cec352afacbf9126ca110399ab0b091c72e10e689d6f0b906a237ce379820
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.4.self_attn.q_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.4.self_attn.q_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0d9a76003049cbad0a76573838b0c5f9d65a2b13
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.4.self_attn.q_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:105a91bb53a7157be275552072ab17d4c51b6f8de2c04b784b5b2637fdfc83ac
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.4.self_attn.q_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.4.self_attn.q_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9ef8d1547e76a323eb38974d5b22fe57b87abc96
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.4.self_attn.q_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2992e15d505be34cf140b176ca207d906692fcf16a2f60a7a270e9be6613cdee
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.4.self_attn.q_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.4.self_attn.q_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.4.self_attn.q_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.4.self_attn.v_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.4.self_attn.v_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5b6510cb7a2c9405fbb01c6a222742988ef4deba
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.4.self_attn.v_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:db00bd452fd08c034bd9be8728020d1022043ac2034245f169f46906cd508bc3
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.4.self_attn.v_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.4.self_attn.v_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d3bce73aeb2bfc45ec817805bd08d5af9b49ab1e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.4.self_attn.v_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6101a7fe5a75d786ba2da86a6d765497ae4e1a08dc3a8b2c8642371559e378f4
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.4.self_attn.v_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.4.self_attn.v_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..266fe9ce08aba3e123a2b69ace939bb449d7cc0e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.4.self_attn.v_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1953bc31088f365276d2e23f65224f907bb806bb40978d11d0b4ae28e86826ef
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.4.self_attn.v_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.4.self_attn.v_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.4.self_attn.v_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.4.self_attn.v_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.4.self_attn.v_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7213ca026269a8aa97e5c31b8e7b326c951147fc
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.4.self_attn.v_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:42bb064f984b237501a204e1c35bc93d2715c00fb051f3a69525b84536db00e5
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.4.self_attn.v_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.4.self_attn.v_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d484fa787165e75d3477e4cd45fa3cacb6217cb8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.4.self_attn.v_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9cebcb77002e486dd3fb1d9b4c13abb4180df311d9be46bed83fa306e87d1940
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.4.self_attn.v_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.4.self_attn.v_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c92ba8932c141712d381c6914537b983ea942e29
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.4.self_attn.v_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd31aabe8ec84be292d7b6abfde91fea95adaf57a57204ab0b0d2e1876c394d8
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.4.self_attn.v_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.4.self_attn.v_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.4.self_attn.v_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.4.self_attn.v_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.4.self_attn.v_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7fd81f4e1d806d21022b46fd1b656468033e2eb8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.4.self_attn.v_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:acb5a35253f9cee096b272f201b4c303780de5e3346305ca086ce0a115d00eec
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.4.self_attn.v_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.4.self_attn.v_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9f7f0d2a48be1fed15c2009b79efe72542ffb33a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.4.self_attn.v_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c6ee265e04686adc1222be53777d1969114a4abbeb0a5b3764c548c36b4be444
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.4.self_attn.v_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.4.self_attn.v_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..dc446d05c259a8d8e23e0a259934a88a7edbcc78
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.4.self_attn.v_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:afb5d985d1a95611ba4e5ad81ef37d6261a539ecb03bd738a3793546464310c3
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.4.self_attn.v_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.4.self_attn.v_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.4.self_attn.v_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.40.down_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.40.down_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f1a7c94aaffe5b71c1237e1a6a89411a19174d45
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.40.down_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:53deb010a7a8a19aee1cf621a8d523d1e172fa9ed636328bc04254a21fe924cf
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.40.down_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.40.down_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..549152f772e5b159a2fc0da9d38c1758d82e9e2d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.40.down_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3aa273942ce798eb4d873336697f6fab52f8da4b3d3ba643eea1b3595adafb68
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.40.down_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.40.down_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5cca0305636d1a41c8f0ce2273bfc6a49c928b99
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.40.down_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ab63de022142aee3951d647a227ca25e2f65b30fe90f32477c33857cf0cfc79
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.40.down_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.40.down_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.40.down_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.40.gate_up_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.40.gate_up_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0f2986e6ffccad363a49b3c14beda41b555a1ef8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.40.gate_up_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0690969080b7004a1cd4444cddc1120b838c8f1b3bac9b368b9678f173eca35c
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.40.gate_up_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.40.gate_up_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..72e0be0053fcfb5de78261ee531f9ebe18f53a2a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.40.gate_up_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:366f6b7b39c2dee9a5b5f143e0d3732a7d5ebef72fb6389799fcbcc4f0a241b2
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.40.gate_up_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.40.gate_up_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d4cb3d8fb2a110c89f9e9b13ed740772f80a6aa3
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.40.gate_up_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c7256e458ccd6e33180191557de2e2866f02b8c2e6e76ad990c21a4bc31cbaa3
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.40.gate_up_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.40.gate_up_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.40.gate_up_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.40.input_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.40.input_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3c2dccf8f643550272a864be267fe886d6433d5d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.40.input_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3b1167680ee299ac792f9e4e196e97adc4d17b5ea9c229b46481c67d1843dda5
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.40.input_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.40.input_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..18815b52944b54a4c0ef6f4087959ed7d12a55cb
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.40.input_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:767e4781717a0842bb3a98647312cbb2e38a375a2ffc808bfa04883bdc845b77
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.40.input_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.40.input_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c87a48f34dc5f23de66721c6f77a49e5474695d4
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.40.input_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:28d0171a149e5395e32d16a4613e0a4284405a2297d1af9136b07bb84861994e
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.40.input_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.40.input_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.40.input_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.40.input_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.40.input_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..162b0fa2642b11d6029e732c02325979b72e35e6
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.40.input_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:350ad3d041b064f6639b1a63ebc915ee8455bfbc07a3e68a469510cbf919d4a9
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.40.input_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.40.input_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..067dc4d60ecb1b69accc3878b82d0d9e261fcd00
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.40.input_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:54a4e33f8e08c2750e5a28f5adcd05a9b07dbaf43d515b0c06c317a1463785de
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.40.input_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.40.input_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b40ce0a16667ebacb36fd65f9aa63096c78b2a8b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.40.input_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:261c2aa0f82a02b2e322b22c1bcaffba9a47542b2051543073f420b5826dafad
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.40.input_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.40.input_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.40.input_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.40.mlp.down_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.40.mlp.down_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..634fb6f8d65f5dd040bba9f081d8a8ff0c92f740
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.40.mlp.down_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:258d3eac4ca68e75c59e86cc0833aa14c35358b108dff8c82e314e6873b678ae
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.40.mlp.down_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.40.mlp.down_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..07775d92d9704c9c1d0b580f967332e6bde7642a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.40.mlp.down_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:37d794ed6e32fb54c8c780a7afdc424847f8c80f2e5ab4faa5c6f2a7aeb8e0a1
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.40.mlp.down_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.40.mlp.down_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..12daa4e0a79a815d70b0c61b150b6d1962118ef2
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.40.mlp.down_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:30682eaf9bd1347a7d52085af4b6c593a8765978168cab28c9e9707c62635b22
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.40.mlp.down_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.40.mlp.down_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.40.mlp.down_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.40.mlp.gate_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.40.mlp.gate_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..115f31d52c86c4efbf8634827d120d155719c2af
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.40.mlp.gate_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7d044a23123ed97a62c1dc363264056d695e0c833928ce583fe66f1060216f2a
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.40.mlp.gate_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.40.mlp.gate_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..79486355a2f68b90e95af45b2989a7b1c023e554
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.40.mlp.gate_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:16939168edac70c9014cb2a636afacc03e782735aee7a3656068c8ab9c2eb09b
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.40.mlp.gate_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.40.mlp.gate_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..dc8e356daac5f5538c75f0e75cbd747d4295bacf
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.40.mlp.gate_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bcaa4c4dd8f996e3df9f9b6d3bb1954289090113eb038d80968bf7232dfaa51e
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.40.mlp.gate_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.40.mlp.gate_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.40.mlp.gate_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.40.mlp.up_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.40.mlp.up_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..63ec939bf3b62830c170afeb38f292d6847cb480
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.40.mlp.up_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:012926c2532f9e1617b50137a7691253e721a268435ee8f711d219692f72463c
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.40.mlp.up_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.40.mlp.up_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9e3d7b4a5d82bfdc948dc959f833aeea8190e0f2
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.40.mlp.up_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cf2ffd4c69126c92a0e66e12c0dd9388244cc6f3e016b2c1fada0b125ea10004
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.40.mlp.up_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.40.mlp.up_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0594298ab1f0ec7f91f24647ea42912f1ec8f198
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.40.mlp.up_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:09cbec2fe555561194b8b96786774fb02dadc29f9f7e152ac8051d6bbba30115
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.40.mlp.up_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.40.mlp.up_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.40.mlp.up_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.40.post_attention_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.40.post_attention_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f86a6f90348f3e26bb01a937bfb64199d53d4c3c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.40.post_attention_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cd54d7df7afc43df4be8f74c85d55cf8110c56881ae969061bc4e592b312918a
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.40.post_attention_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.40.post_attention_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7ca9d61797bca5cb13731ca6a65591a0b4bf2e3f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.40.post_attention_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9710aff29b7e544adae89a86dd83b77b144b81933006a5511f27292981b23fce
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.40.post_attention_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.40.post_attention_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..04dc173e7ed4fb6ccf93eda3fddd63149f05137a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.40.post_attention_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:37c3956ef22adf157986b964b865e91250093dd06c282d72e2dbeaab4a51ca73
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.40.post_attention_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.40.post_attention_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.40.post_attention_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.40.post_attention_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.40.post_attention_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..94884e559a5468c03d01ab7629355cb3ba99cba9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.40.post_attention_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ebb0b0c1c6c110648fb442bb06464abe3c68a75ccf2a70eeca4303a73439d968
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.40.post_attention_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.40.post_attention_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ae75c079f94d3317aa77c4e8b957ae1410fd75e2
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.40.post_attention_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2c345e162c7e7fb87a78ea6e1979597c6c435bc2b3651a2c91398e56729077f1
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.40.post_attention_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.40.post_attention_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..10c81d663bb507314633d3fb285f0fb595ec49e4
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.40.post_attention_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a448079b24e8c1d0ca90b5325fde9f32395540f413310bc3aa3a84978d3d01e5
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.40.post_attention_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.40.post_attention_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.40.post_attention_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.40.self_attn.k_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.40.self_attn.k_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8608ad7ce73f5849df66e2dec99f7b03326e93a9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.40.self_attn.k_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:73593f7b41f4a148945b5227f50d2e3243533536e6fa1758608ff72786e65260
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.40.self_attn.k_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.40.self_attn.k_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..71473583ae00c5f3cfad4dd5cd8911e4f4e51302
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.40.self_attn.k_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ef2f3a12268c5db39c72ec863683bc78b724371440b5ec46881f3d470c98618
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.40.self_attn.k_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.40.self_attn.k_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..899380b6b5d137b22d4848bd982755a8af350a0b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.40.self_attn.k_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fb8c7e883fc6b7485044bf4b5ba6753771483879e782d9803f4b2ad55b5e8823
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.40.self_attn.k_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.40.self_attn.k_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.40.self_attn.k_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.40.self_attn.k_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.40.self_attn.k_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e9698a0293e830e8aad59b38fae5b5939c739a49
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.40.self_attn.k_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a2a6b338d6a739f7b0ba6f68f1cbafca5f4a985373ecbb6aad84d35854e2e953
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.40.self_attn.k_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.40.self_attn.k_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c76fe477fc26c0d41808cb004a5c646c605cee29
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.40.self_attn.k_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:debd06f27fe63fb1da2b8ea245cb4a10820cfd152593213a287b9575638069b4
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.40.self_attn.k_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.40.self_attn.k_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fc66fa5f0cb5db60799066fa68e789a9537b05b0
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.40.self_attn.k_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:54c04e77bc84663963057a4f5247bd4935acdb6b94b530a5037f80fd5e14ab98
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.40.self_attn.k_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.40.self_attn.k_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.40.self_attn.k_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.40.self_attn.k_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.40.self_attn.k_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..62656f978dd5730f6af8a9cd57262a3fde6b7ae8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.40.self_attn.k_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2e9168e343b2109e2917ad2028181ae13262664fec0576b0f5fbf85d142d2cc4
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.40.self_attn.k_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.40.self_attn.k_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c0abe176da3505e01a0949ae2022a48f56e0a549
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.40.self_attn.k_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f16080561f10d8bd4d6e046e95e5a34036ce6711e806993a2fdc97bbb13396ff
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.40.self_attn.k_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.40.self_attn.k_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..835e9151d3b1b34896635a6dd45f0e199efb3a1b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.40.self_attn.k_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cd3191843e7c352a80985e5aef0d00162c629a48c83c26c8ef77c18a5461653b
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.40.self_attn.k_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.40.self_attn.k_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.40.self_attn.k_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.40.self_attn.o_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.40.self_attn.o_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..05c719ecada7eeae6ae0450c23c8296527b2df68
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.40.self_attn.o_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c088008aa2cecf5464903325663234919358099108ed04009011d5afadda29f4
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.40.self_attn.o_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.40.self_attn.o_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..185fc924d0aa4268a42a809c76e42261ff96b40b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.40.self_attn.o_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:83aa9ccbb8232837df8dbc7b388d06a6741295666dbc053476e704253191bf90
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.40.self_attn.o_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.40.self_attn.o_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ee416e7da9c347a1d1577da34da606a0295c8f70
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.40.self_attn.o_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7d07fc5230f05d3ae1f2d30c2e4029076920b1545ad21b08df6320ca28b94c79
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.40.self_attn.o_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.40.self_attn.o_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.40.self_attn.o_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.40.self_attn.o_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.40.self_attn.o_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3a1a4e14e29169883b85c4338e1483b2deda2a08
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.40.self_attn.o_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2480b1484106860e397ec2301296984c43aa7bb095d0ab6eec6c641a69f4ce1c
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.40.self_attn.o_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.40.self_attn.o_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..dfb4f69977c6d2b8d870688668b2c412626af9f9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.40.self_attn.o_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f7e2af6e0e835dbbedcdeebee19751613d168778070e09e39546415d33a7b2bc
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.40.self_attn.o_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.40.self_attn.o_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5fbc189b4021369edcbadc65eaba30fd93a64b53
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.40.self_attn.o_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2ddc2453fdbda06f6d67eed6e999fb2f75e4bec99f1d224413d2e533ac144019
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.40.self_attn.o_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.40.self_attn.o_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.40.self_attn.o_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.40.self_attn.q_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.40.self_attn.q_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e8183752a79fe6e0244ade99c4e8d6fda33106b7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.40.self_attn.q_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:44733fbaab9c2e36ca0624cf8ccec060ab3075c5c5ca2aab193834ad3478e74c
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.40.self_attn.q_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.40.self_attn.q_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6722806ca47562047cd8c574ebf339b95ef0e887
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.40.self_attn.q_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9bc44bfa75f741dc405f75bc53ce792493b8cd9bb40a5b4e73c7d4980f6f6372
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.40.self_attn.q_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.40.self_attn.q_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..dab3168f48522dfa9a81bd545ef850d0f31c9c77
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.40.self_attn.q_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e71492d155ba80060475609d8508e9fc20e4f940801050a5738d74cded642942
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.40.self_attn.q_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.40.self_attn.q_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.40.self_attn.q_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.40.self_attn.q_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.40.self_attn.q_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7e88d4642d2ca3f4a916747ebf6cc018dd7e4a90
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.40.self_attn.q_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f10d757608178d0f9695af1a8fdcfbc063999835606498d29094e5caa93853dd
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.40.self_attn.q_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.40.self_attn.q_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..52ad67da05a066f9ba3f34d94afccf01d17529b2
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.40.self_attn.q_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3a6da9c5b5130edd048a89bfa8529106728be664c51d2eca632c0570e28957d1
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.40.self_attn.q_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.40.self_attn.q_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..adac9f6f804e8511a829eb563a6448e71bed2abc
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.40.self_attn.q_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b32539210243ccc6feaf9718493f8cb6a6695b64f716528b01021c95dbcd8f6c
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.40.self_attn.q_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.40.self_attn.q_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.40.self_attn.q_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.40.self_attn.q_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.40.self_attn.q_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c609fd57ed67e35801d330bcba75a02e5b289b25
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.40.self_attn.q_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:33d868f085db185dd1c5efd3e34eab265622da576862cc56531c889ab081db37
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.40.self_attn.q_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.40.self_attn.q_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e47eeac8e37c0d30f5112e231a93210d847264e7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.40.self_attn.q_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5ab250de465c02e0fe301f94890a41dbd8e7dc3e719fba8ecf7c4161f3f89b5f
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.40.self_attn.q_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.40.self_attn.q_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3dfe864a5ee48b9de969c07cd9da678afdcf1997
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.40.self_attn.q_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ce660e7d4d57426947756eee570becd97ad2faed1238a152e776a715b786de83
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.40.self_attn.q_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.40.self_attn.q_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.40.self_attn.q_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.40.self_attn.v_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.40.self_attn.v_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3e9934d07724b976bf9d8611eba59526871c0638
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.40.self_attn.v_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:11f74ade6a7a6092a0a278531681598a478d974f54acac179a64e0ba0194a9ae
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.40.self_attn.v_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.40.self_attn.v_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..619767d0955663076e74bc3ab54babb23cb055ae
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.40.self_attn.v_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ee9837b3d286740fa677f423200ed67b7116553f53b4d4050e29415bd508affd
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.40.self_attn.v_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.40.self_attn.v_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3d702ea773cfc077be644d47c63e6612495876eb
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.40.self_attn.v_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:33d3a4b158d85276caf21b679759038cc07eeb13564c7b0aee686463cddefb9c
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.40.self_attn.v_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.40.self_attn.v_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.40.self_attn.v_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.40.self_attn.v_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.40.self_attn.v_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b71b3c29b5b163dd0949729e91af4347961909fe
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.40.self_attn.v_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f838593d0205e5d389326c12fbd491921d6aa864f9cea357369a0f2aaf792632
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.40.self_attn.v_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.40.self_attn.v_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d7943742bf6f835d67b46a7f9d63043125fb53a3
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.40.self_attn.v_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:25191682dc741c8b1425d25a48f1d2f0ca3531b6412ca2089f4a80818cf704f7
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.40.self_attn.v_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.40.self_attn.v_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..11fc1bafaaa19a0dcd7e9fbb9b1a07ab8de4c44e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.40.self_attn.v_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8343ec2c84aa0a461efe6fe98de8ebdee2abd7681cfb5ef2dce5b71206b991f7
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.40.self_attn.v_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.40.self_attn.v_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.40.self_attn.v_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.40.self_attn.v_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.40.self_attn.v_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..62331f9b4f685b20a99e0cff84f92743c700804a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.40.self_attn.v_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9b52a97414d4baf5836c81fd264bf985f5058da8fc0a1d256e0965c97d731bb5
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.40.self_attn.v_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.40.self_attn.v_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0c1975800c3ecb9c6607356d1d3c20898d5180da
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.40.self_attn.v_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f5ef1580d76726f8789a9d4976b7b29ef1aba6a72296c57b95da9cac0515a99e
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.40.self_attn.v_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.40.self_attn.v_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b12bc01f9ee8a9d6b8b3493a818ed2148429e643
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.40.self_attn.v_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:720f43c0bbf37e01846436181edb3fb0cee8506f91358d79e70d22eab91b875a
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.40.self_attn.v_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.40.self_attn.v_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.40.self_attn.v_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.41.down_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.41.down_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9d46798b32a86c2909647ff38b7cf0591459346d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.41.down_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7f3f587b37fdabdc7d3cff94663fe585f3dbaefc9d930a0fcbfd77b3970d706f
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.41.down_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.41.down_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e55693699646d67c3143609c024a7fb106920639
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.41.down_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:af1a55a21ed51a636d08c00eff739e1273692437bcbac43cc6b884294335c725
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.41.down_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.41.down_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4dcd04fb8b2492c2ce527fcea296550eec7232a9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.41.down_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f053457a0d1f41e0335286162b5e384fde966c05d7ef11e8c07d4f9d9896264f
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.41.down_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.41.down_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.41.down_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.41.gate_up_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.41.gate_up_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e09c5965dae9d881ba93b4fca51165ddf093badd
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.41.gate_up_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5451c4875c207ea15ef1a2983f0f3d30eaec6edbee97d26915c6bc162fa2151b
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.41.gate_up_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.41.gate_up_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cadeab580f72e583120bf5d08f5f33f3ae83fc3f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.41.gate_up_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7d265f0c5fa0b0d55c99a172084b8382bf43d835ce2944448fdfd2b0aad760cd
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.41.gate_up_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.41.gate_up_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cf2c77b773915cd720c1f320cef9dc0da161c349
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.41.gate_up_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:25dffef57cf5f6269792312275a6e2b243ab5a24be8e0faf4353352e018564cf
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.41.gate_up_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.41.gate_up_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.41.gate_up_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.41.input_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.41.input_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b2b93ea367470e81b7fcce492034109ea33fd721
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.41.input_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:171affd24b0ca3f0c26fc2ce864e639036a43a3316a64c74ac56845780ecaa5d
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.41.input_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.41.input_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ef83d5f5f1217b31ba9dd2ebe44ce6a04846f9fb
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.41.input_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:93ca8766681359ec6e7bbf49072d10dad814810a07f3a0c3956f9bb170d1edf6
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.41.input_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.41.input_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b7ad4f10e14c9dc0d165086d4a754897c1a73703
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.41.input_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8be57577635a01227f5d9fdd978ce5e1252acf6a64f712146ec41aec6af81a0c
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.41.input_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.41.input_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.41.input_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.41.input_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.41.input_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..db4d60b32278bb241be8708cdd7b47134a7bc7cd
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.41.input_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a906e116606bc27d5c1b21ec2a6b520cf40ca2ef81e0a614f22b202e6810d8c0
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.41.input_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.41.input_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c314135ba54322c591647d5e4b1a3fdbbc7d0381
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.41.input_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e61776e49c6d618a6dccd68af64f54a436f3ce4f7070be2c24a4f34dc10c5c4b
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.41.input_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.41.input_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..173c9148be73b05bd89f96af758de52f71400f73
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.41.input_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a2bf05785650db663327ff884f447bd9df481c0ecf1a3d5b3f64be252ca6cbdf
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.41.input_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.41.input_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.41.input_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.41.mlp.down_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.41.mlp.down_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7e0b987f4ad8a29d273f32f48de834de53a6e46c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.41.mlp.down_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:af942b865ef1ea505b2e2f71b946ba4589e52a8212a6175b2cb5c2ebc13b2f18
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.41.mlp.down_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.41.mlp.down_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..36b992deea72ff18fa02e98a90d4646ed3be02f5
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.41.mlp.down_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8c7d14906f7975ac4818976285f17df84450d2810b2a720b89ff39a5b5f869e8
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.41.mlp.down_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.41.mlp.down_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..dda058f5053e53c0cbae76f91bef8675fec91687
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.41.mlp.down_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:73c5743b78c078cb5dc94d86199bb8c80ffb3bb0370899d63a60007fccb88a74
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.41.mlp.down_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.41.mlp.down_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.41.mlp.down_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.41.mlp.gate_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.41.mlp.gate_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..bde7811433cd2c50f03544467bc2c4d0a66768db
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.41.mlp.gate_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:485be2d99bdac0e665a1f0fefdd7825c9190655db29b9ec0acbfc3514ddf3fdf
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.41.mlp.gate_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.41.mlp.gate_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..de87b6a8c9270b362b6b64f5935d6aaeb2da8c04
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.41.mlp.gate_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4875bbe365fbc5836231d727804190b132481ef4bb313fcad9a69f8db327b8d0
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.41.mlp.gate_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.41.mlp.gate_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0377341ea02261cc0f6b1741b7199c990e7c6e5d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.41.mlp.gate_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a4852be47e2e41f9d62e974aa06cadf9b3b78466fd194d8057985e827a78e02
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.41.mlp.gate_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.41.mlp.gate_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.41.mlp.gate_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.41.mlp.up_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.41.mlp.up_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..707daffcf20ab538b6f2036762e1d6a9ab33cb44
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.41.mlp.up_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7f300b1b9c25a05e55bdf2a095a75d969e1ed6edd1a2b82b614c87edee772d8f
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.41.mlp.up_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.41.mlp.up_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e06daf6fc91372ac659eb5d24087d116adad1868
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.41.mlp.up_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f7cd9f510e1391d6d90a5f57b3258921197d85e49e5199941e23f1fc7b8e3608
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.41.mlp.up_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.41.mlp.up_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b4c00a90b541e3dba42a24b62498dc592922b0af
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.41.mlp.up_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:25b9d67d206347b6e2a8b28551c80b62b0a3a17ecc255f3ecf7f6a13b13195c3
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.41.mlp.up_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.41.mlp.up_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.41.mlp.up_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.41.post_attention_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.41.post_attention_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..911fccbd7b67c300e19e1425957e8f753c083b81
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.41.post_attention_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dbcfa1ae8a19625748f57e971fd5746388334f9813375bc92218196027b24f06
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.41.post_attention_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.41.post_attention_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3f43593fa35ebbb832cc02ec31e266fc0b325812
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.41.post_attention_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aa69736bbfcb641fdf73ab047a3d9dce5271acee0f2f466871fff7e7c0070bab
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.41.post_attention_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.41.post_attention_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9f8029ca423f7d756ff96769a6c5d52a017e66f4
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.41.post_attention_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ad2f42657b312c750e9fc9b0a572a78bea032b5e6c8171973412a17e7de3b604
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.41.post_attention_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.41.post_attention_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.41.post_attention_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.41.post_attention_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.41.post_attention_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d7c56e79a44d3150e9e1c77620af7ba09702ff4a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.41.post_attention_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8e1d6c6f72875ab68d422343fd9975822f3bb8b9b0e86c2dfdd69612e94c013e
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.41.post_attention_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.41.post_attention_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cfc298f6771a36ab9a1ab3818c774e31d50d0889
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.41.post_attention_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:00e373c1e9912828232b15c86602dce9a98d1012d2112ed253b61731909e7f47
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.41.post_attention_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.41.post_attention_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d53afb4129d151331117cb0e52edbb50fc5e2c7f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.41.post_attention_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:40167c1a0fb6a0800c9dd73d2d813383ff7b262930c4561f9ee96631ced246f8
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.41.post_attention_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.41.post_attention_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.41.post_attention_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.41.self_attn.k_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.41.self_attn.k_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c031b80ef68120bfd67be49dff4390319e45af41
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.41.self_attn.k_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a6e9364db8e6ce0e76dcace230e4e6c6a97f88a5c47a7868e470bbe2cf6a7164
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.41.self_attn.k_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.41.self_attn.k_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7da8fd8e1edb5043d2be0d37bed20bce98b80c49
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.41.self_attn.k_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4fbb4cc5ed8dca30ba84e0808fdc3ac335b7ce7c0a5596a95eaab52b07d79c93
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.41.self_attn.k_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.41.self_attn.k_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9f9a75df9c3bb7bed7ade1ba374cb7a4962bf150
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.41.self_attn.k_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ff4839f8e57eac56051a6bfa5c865dbb0b1d40cec54ec2c15913d87a5cd49132
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.41.self_attn.k_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.41.self_attn.k_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.41.self_attn.k_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.41.self_attn.k_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.41.self_attn.k_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9ca4a5627fa060aadd6b63644677e403503c7fe7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.41.self_attn.k_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:34c8b1defc18fa330b4626516f9be7f151af00aa387c608570bfa16b4b445258
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.41.self_attn.k_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.41.self_attn.k_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b0ac242211bb48dc51ac404273bbddd56810900a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.41.self_attn.k_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8f0a1a90c73d5d711b4e8e1de47fe50d184301c34541f956d6e6050a4a35742f
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.41.self_attn.k_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.41.self_attn.k_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..20878ce689f425a770723c033430612367a22a2e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.41.self_attn.k_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8a40ea1039eebe5c7572ca9bafadede065dbdd2afabc0a7e7ad262180991e0e6
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.41.self_attn.k_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.41.self_attn.k_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.41.self_attn.k_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.41.self_attn.k_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.41.self_attn.k_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..eccb6c7977a03045a0e56454ac03f019fe24d172
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.41.self_attn.k_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca4ae468c3ded3148593d1a3794af7367299c6a417cb3b51b13c0065040176f9
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.41.self_attn.k_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.41.self_attn.k_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c03beae3042829560f15284ac93a6843c590e801
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.41.self_attn.k_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:55fc612c54b0a800dba73c2872643931c01458898def77a4603595271c08b575
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.41.self_attn.k_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.41.self_attn.k_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8d98a14518685cd2fc3d2900caf2e361d9b6fcd9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.41.self_attn.k_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:de1612ff4a3323cb4a55481a702da7715833a194a949f6267f7b8b740f90ad4a
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.41.self_attn.k_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.41.self_attn.k_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.41.self_attn.k_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.41.self_attn.o_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.41.self_attn.o_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2b6dce132e459cba208d34b6564ce21a03e94bcd
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.41.self_attn.o_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d733f41476fa80625b720847a400af89496c60a2b4cef7b024a0ce94dc7122d
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.41.self_attn.o_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.41.self_attn.o_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..438f58a84086e54312ddb0a5edaa72d1326475f4
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.41.self_attn.o_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9557610b44aaa23ba81e03230eec8017315b7a73b436ade62e2099969b0fca8c
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.41.self_attn.o_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.41.self_attn.o_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8f098a8927e3731f627b5192e513211618d1292c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.41.self_attn.o_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ad64ebbd4872389290f3add4fc124e523f8fe23d2a570c6a940f7a3680ab89ca
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.41.self_attn.o_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.41.self_attn.o_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.41.self_attn.o_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.41.self_attn.o_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.41.self_attn.o_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8aab5f2ae033408f0b1b06f4ad40bbbf7e8e5f6d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.41.self_attn.o_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:72eade9c7c5dd4b12ad4b05a0fe704bbe15265a69f2c3e6d9aa054f662121b85
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.41.self_attn.o_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.41.self_attn.o_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cbc0a8a3fa8d7c84c42477295743e41030a40c4d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.41.self_attn.o_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:66e24747bc614e00964fc5f0dadda091bb36d7469e8f74a0bcda999b1a2c8564
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.41.self_attn.o_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.41.self_attn.o_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..89cad7bfe039695085ae30c1f12882cfed9a4912
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.41.self_attn.o_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:48c821881878ec086e08662b2f4cf72c3824df860586c6246ce497382808543c
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.41.self_attn.o_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.41.self_attn.o_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.41.self_attn.o_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.41.self_attn.q_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.41.self_attn.q_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0cd6cd0bbbd8f67c996668cbde2cae92dc6bee8e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.41.self_attn.q_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8c6128c6a35893dabe63400e906f25e530b8f7f84d350fbdaf26429ee5ae2fb6
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.41.self_attn.q_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.41.self_attn.q_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7a7595c0a1b7f1331ade4e8aa49640733e7a58c5
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.41.self_attn.q_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7752048bcac2106ea77ab9d26893aaafdf5afe8afa6467d3954cd860edb9625c
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.41.self_attn.q_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.41.self_attn.q_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6b40449a1c853dde41df91b9260745ebaba7c5fe
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.41.self_attn.q_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d94d698a4c2fa68bd7dbeae7da3bf727c28ccad6b7fbdccb2da9f1ea9387344c
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.41.self_attn.q_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.41.self_attn.q_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.41.self_attn.q_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.41.self_attn.q_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.41.self_attn.q_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..141e634470dbbd3ce60a558c4c622df6fce18b6b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.41.self_attn.q_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:343ab523a8552d265ba4308030f39193b6fad345ab89de3851eb67b7284ef8af
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.41.self_attn.q_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.41.self_attn.q_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..62cc1b395708c9fab99c044ed1cddbce247dd043
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.41.self_attn.q_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8f4d9c8c08b0a46d67ad90b27cbb0878964d658f11b44f5f2d864c4d1d2b4a10
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.41.self_attn.q_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.41.self_attn.q_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f56535785790c5f47f0d1958390664d262272fb1
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.41.self_attn.q_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d147ba6a0a581ecb3525ca7d8124cc0574765d8e40a4e70d9adc64716834202
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.41.self_attn.q_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.41.self_attn.q_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.41.self_attn.q_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.41.self_attn.q_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.41.self_attn.q_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ead1b946bc76fe4fd6fbb11daf64e990102c4798
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.41.self_attn.q_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:46ecf5f0360b5402cee63e6d1603fd54cd42e508e4eb2d9dc2c45227ea1b77ee
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.41.self_attn.q_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.41.self_attn.q_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6a26ce7145838c3bd3bd24e2541ddd0d4fd2b8ff
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.41.self_attn.q_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a1098bcea071d6d6beb6951c0b3e2e44300a9cf42bcf59d567a78421f5c0203b
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.41.self_attn.q_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.41.self_attn.q_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..25f1ad74e29ef0564a98478992eec622f95e09c6
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.41.self_attn.q_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b7d52f4276b453a0b420e0fd81283b2f4df4ff6a42b081dd832afc6a01d5dbe6
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.41.self_attn.q_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.41.self_attn.q_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.41.self_attn.q_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.41.self_attn.v_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.41.self_attn.v_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..91f11a16a852b148959d0ebf006c53527decb234
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.41.self_attn.v_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:77e24817e6c6c3f07f4b94df52673cb4750f07cf26d21a54e8787f339223858f
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.41.self_attn.v_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.41.self_attn.v_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..620e76f8198179735ac9929cee042f1ce36341c4
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.41.self_attn.v_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:54bdee5d4c7d4b40ffab0806f817ed9092eea672e009d7f0721de60557b0163c
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.41.self_attn.v_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.41.self_attn.v_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1860ee42e10cdab5af3f617de49bf23c230dd581
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.41.self_attn.v_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:30f01dbd5221750a8f1f8ef642d9f9216713191c280c548b45628e425534325a
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.41.self_attn.v_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.41.self_attn.v_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.41.self_attn.v_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.41.self_attn.v_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.41.self_attn.v_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0e9ba478994f307fa6daf0d71a57ab8ab17f3077
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.41.self_attn.v_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c97b0e546173e1675ad25d5078aaef3d27aaf94e98f0a51a9c328691e2e130b1
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.41.self_attn.v_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.41.self_attn.v_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a85960e114f9157125fc7ccba713c6f7b7dc143f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.41.self_attn.v_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:900ba2d265342c8bc03a13979307b56e060e654af5406f2a5c68d11b4d64b5a4
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.41.self_attn.v_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.41.self_attn.v_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..34cd0a524b365f89fb5d54adde49858545971da6
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.41.self_attn.v_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c5bc2b63af464b311f53538b4bb23266e8a6f2aad8bfe964310969ceca10f4b2
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.41.self_attn.v_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.41.self_attn.v_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.41.self_attn.v_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.41.self_attn.v_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.41.self_attn.v_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c73414cff2a06f94b39449784cde01c547b69240
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.41.self_attn.v_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d72f2ad8d17ad9f4ef73c60f2940cdf48be0205a792aba09c325fe09a84639c9
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.41.self_attn.v_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.41.self_attn.v_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2f60c52ff2dfc81e01748c817c459f2dbd01705e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.41.self_attn.v_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3e5e161cf301d17d8c4cb18b2f8ef718a6d169a4e038a8639b6988ae44de6b43
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.41.self_attn.v_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.41.self_attn.v_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..93b1d840081a0f62bf7d3d1d5ee22158506545fb
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.41.self_attn.v_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f47a081d51bc343e7e8c9abe2085f88136deacfde94d3c9ee583905e82cb50b3
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.41.self_attn.v_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.41.self_attn.v_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.41.self_attn.v_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.42.down_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.42.down_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fcf2514fae250264db50991bf2e325c1eafbf17d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.42.down_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a03c4aa5e975f02fc2838a0cb8542d9bb97e79c0f6b868b752bb113f0f0a362f
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.42.down_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.42.down_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f96f501c85c0975e32fe12bdf5b803b6403e93ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.42.down_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:970a6290c13ddd4118bfdd36a791311d4e42c66d18b9541b8a3baa6721ce500a
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.42.down_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.42.down_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7da130a17669df536d499cd1b046d0b7ede91dd7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.42.down_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:90bdcdc7485ae9666ede0ac4eeaddd11d2169c9940f455fd9aa9abf005f9859e
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.42.down_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.42.down_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.42.down_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.42.gate_up_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.42.gate_up_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..354ce348dca885cf79373faefc818c7c087f6874
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.42.gate_up_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d77ad89836b1c0668bea14f73fa7d043c77983d7a78a4468453b83e075896b8
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.42.gate_up_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.42.gate_up_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5d9639e963a9bc3de55291cff102f1ac8382d6c2
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.42.gate_up_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:061064b31e88e563c8e9cba89ffdee0832ab41f16ef00eccf8ad8d35d2f8cf1f
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.42.gate_up_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.42.gate_up_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..090871f8360a251cfc1e0524bc26c01ca3768a2e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.42.gate_up_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1c4d82e0454915904fabde311abebc3957d19b85487672b0f09c3ecaa7b41229
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.42.gate_up_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.42.gate_up_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.42.gate_up_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.42.input_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.42.input_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8871cf2d903e3a3381df0d0ad03e518fb52ff13f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.42.input_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f28dde5c92b40dde4605a74d12b42c6912d8f6851608763387071e36a357a4ae
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.42.input_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.42.input_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0fa34e97e2df4163c79b3851f2aa70477ed935eb
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.42.input_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2ab93c882afc94322f1a025fe51f9978bbd952e538b54645a36dd0cba05fa799
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.42.input_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.42.input_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..433d4fded32b40e5bfbae621650a44eb32177f4a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.42.input_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:14472de88b61447fcfcca20aa9d494d369ba1712bf9d2bc49df8953c12af84dc
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.42.input_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.42.input_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.42.input_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.42.input_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.42.input_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f2b6789fba6f9e5b8bec7134010f1a6f5591fe28
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.42.input_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:da503a9cdc48a7d520334706fef9bcf8f87d0916fed1a779bfa09b31f18c5a23
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.42.input_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.42.input_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a58fdbcbb1738e5cd3f0909e367b20b32054cc9d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.42.input_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a14eb22873b2f51ae31ca73db6fe021b7ed1b06c73fd6a8c96d4527ff6fb03c1
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.42.input_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.42.input_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4561ac2742cb440cf253dcf79d3ea9c2a8944b9a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.42.input_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8517d9cccb0aebcf8e2e2558403c85abd297f734605e9da048eea6c7a5d0fa12
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.42.input_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.42.input_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.42.input_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.42.mlp.down_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.42.mlp.down_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..193cc13455fbb0fc25b8e89f0a7ae0bcb6f6c8fc
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.42.mlp.down_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5bd5088c3dd1fe718f00e047c8717b6448822ccd611c46db9686b103fc84ba60
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.42.mlp.down_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.42.mlp.down_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d94aa8d1883af6540ca9273a22de0a48a238dce2
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.42.mlp.down_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d572a2d9f1a7004b9566682c8c551dfe5bb4561fc087b6b33298f28f6db63681
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.42.mlp.down_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.42.mlp.down_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..99516d3e3cfffd4a9dd6425ab989d3b18b5d3b4b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.42.mlp.down_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:68d3f170e4705496d7fca119d276b39299c427b045ab3fc69cd7f359c9a829ac
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.42.mlp.down_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.42.mlp.down_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.42.mlp.down_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.42.mlp.gate_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.42.mlp.gate_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f0c72eb005e3a2f73db10f24f001bab1e8e9893b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.42.mlp.gate_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:565f13e7e69aee1d8659f545e7bb3698b4405b5030e6b6ac34e7b3d923958b1d
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.42.mlp.gate_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.42.mlp.gate_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a1feffadf9d60d90b3f64ea54f1f1617405c38f1
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.42.mlp.gate_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:96a59a52ac6a709f47c854b106c3bbc88c8d24e114bbbd110b94b059bdfc0951
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.42.mlp.gate_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.42.mlp.gate_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..02baca67280c950a7e863674e521986de170ceb7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.42.mlp.gate_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a0a5054e77fe8e51a75286c221c7c76d1d1c63184918c61be32e9fcaef09d7c4
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.42.mlp.gate_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.42.mlp.gate_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.42.mlp.gate_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.42.mlp.up_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.42.mlp.up_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f952c8c7ff1f0d2cfcd5fb538d6a87a4c7424000
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.42.mlp.up_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c9c8581b17d09cec6a64fb42855414d415b53ae6e4a21e656be33c20b541d11b
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.42.mlp.up_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.42.mlp.up_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e2f75da1cf07fb23f912a7dba8e57af7137431be
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.42.mlp.up_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:58d290f2cdfcf38343d6747fa9c7548a27e5248806d789b92752ff3217524284
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.42.mlp.up_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.42.mlp.up_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3c8272fdf1419f2871b589f21e83671183a6ae9d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.42.mlp.up_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dc4b5eb1a2f684e0d591223cb825fb87c1a7f96ca8805bcb1cf42f0db9692287
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.42.mlp.up_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.42.mlp.up_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.42.mlp.up_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.42.post_attention_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.42.post_attention_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..db7beb47dceeae4ff6d2d2a9f733eee11f4abd65
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.42.post_attention_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c7f4b695b72091b0a13e0c67d75bcb77dc19e8097514a78704257b0077939016
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.42.post_attention_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.42.post_attention_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4036cc74ac9cad365a14f3429803bb90026457c2
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.42.post_attention_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fca05be783cf443785d8e6e4051d4d193f584cd8b214cc2712690a9c86b682f6
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.42.post_attention_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.42.post_attention_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..88927a4d188e15d012d7c7e919dee7faf68bb956
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.42.post_attention_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:48e91cea83a94d8d783e9c1d2852776bf893df82cd39360eac30908c45d131a2
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.42.post_attention_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.42.post_attention_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.42.post_attention_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.42.post_attention_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.42.post_attention_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b09ce97a0d0da33735521f9e86d7995db36f0ae7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.42.post_attention_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c43d2954b91470f32af3e800f3ef10338f1fae3dc3e078134279325638eb1e1b
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.42.post_attention_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.42.post_attention_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3d2f138d5eb6bd7dc8c9fd8130b8321af78703e3
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.42.post_attention_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f5fc5e116317a1862e5cd340dd88fa4324e6437af3ccd13be3fee50c698d4dda
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.42.post_attention_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.42.post_attention_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..302bace4b3b626ebeafcfb0f97d1041a8c04f6c5
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.42.post_attention_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b1e5ebccd36012aa35c03cc203afa5dcc019374adf7cd2ea03dd5a3ef6ddceaa
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.42.post_attention_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.42.post_attention_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.42.post_attention_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.42.self_attn.k_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.42.self_attn.k_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..25bb074327bd6e3b33cba491191d5e03ddca825e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.42.self_attn.k_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e97f31609d3b9a5cc2d1fcb3a01e5243bd1eaf56ad5a51cdf08d1c6aa813e4bd
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.42.self_attn.k_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.42.self_attn.k_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..28b2447cbf83eb7ac23ba8e41ffc916d4fbb9992
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.42.self_attn.k_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7cd65508083ab5da549fc4fcc901b8a571533f1a2abfe4aa7359db08118154f3
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.42.self_attn.k_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.42.self_attn.k_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8d25b351514abf319bba7ed43e059130bbac8dda
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.42.self_attn.k_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d9f1b027fb8fd49249756e27aee9971f39add9f599be7f69af1d4f77d40031f9
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.42.self_attn.k_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.42.self_attn.k_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.42.self_attn.k_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.42.self_attn.k_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.42.self_attn.k_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0f24fe1b4ee839795755e947a2d6ee72b7c6a42e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.42.self_attn.k_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f790f87fe6047956fb31b4d9e1cb6a1b692ad7fa060645282e39b62474619302
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.42.self_attn.k_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.42.self_attn.k_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..53ada907356c794541a577f5a1a1ec1958675783
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.42.self_attn.k_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8e5ffa18da6681b5fe17d20a200b5ba4048cb1e47027deab53a6df3c8d3d40ce
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.42.self_attn.k_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.42.self_attn.k_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..99aaa9bb8d8c94fed2f56ee62628f30e0c4578a1
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.42.self_attn.k_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:83f86af51956ebe4ac2d48cda01138f8db902a69ee5b38937a7d3b4bc848d922
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.42.self_attn.k_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.42.self_attn.k_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.42.self_attn.k_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.42.self_attn.k_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.42.self_attn.k_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..44b7c57a9895d9d9adc52a8e031ae04866a56286
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.42.self_attn.k_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b88e8822be0afe83fcdbfb98438f299e3e70192913536ff8dbaeafd41542d2ee
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.42.self_attn.k_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.42.self_attn.k_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8db56995237be9527dcfa204a8bd5b6a6e03bce1
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.42.self_attn.k_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:24738d696e1c5abd060df6382bef2cf27ef2ce56da7325f7e7707d1c9c86867b
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.42.self_attn.k_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.42.self_attn.k_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..399bb39df87ada52953ce8fc7a7cd11e4604e243
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.42.self_attn.k_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5ae68899e6b344289341b13ca22f467c975196325955fe76545906d1c65e3428
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.42.self_attn.k_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.42.self_attn.k_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.42.self_attn.k_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.42.self_attn.o_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.42.self_attn.o_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1e19f47063c3f8fd9afcfa56aa24c790a9f91937
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.42.self_attn.o_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2f7b7cb1651f34855b69ac5cc5e2a9b7278244b8a5eafd27db5f31954c4b4b7b
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.42.self_attn.o_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.42.self_attn.o_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2b09e80910f193d7bb6e788d340a1652ba1d9669
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.42.self_attn.o_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e3578ffc1ffdfed78b9e883421b1135a6917bb889503d34a9586042746f511ad
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.42.self_attn.o_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.42.self_attn.o_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ccb9352d09c7d2437873fb0205a42c96b09b0e6e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.42.self_attn.o_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4c0434cf3ffac6f2ebda499090d7c5747edc7075f4df3c769fdb7ac9cdfb320e
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.42.self_attn.o_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.42.self_attn.o_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.42.self_attn.o_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.42.self_attn.o_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.42.self_attn.o_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..632a810d49330f2454e8a084c461d71ce3492369
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.42.self_attn.o_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:78f321df9d9ab8875a353a34990a57f5c90b21285d792f53bacd47173bbd30e7
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.42.self_attn.o_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.42.self_attn.o_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1ab6c34ac80c89ef08afd3f3fd2c7feec2976e05
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.42.self_attn.o_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6c54f88a5ac3bc7668eca8ebfc9916dd8292f6b69036040e63c684d2d0aff7f1
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.42.self_attn.o_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.42.self_attn.o_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5735d971257343d5259c99a20a9b3658b886b978
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.42.self_attn.o_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d1963df2012a4459ff56034d79038b6c816b7dad5d48f465c5a2ccedea01c6a0
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.42.self_attn.o_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.42.self_attn.o_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.42.self_attn.o_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.42.self_attn.q_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.42.self_attn.q_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..778ebca14e3741aa95d0869d2b3aea83dd1a7892
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.42.self_attn.q_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ed3049bfc41099d4415750d76cfc783a66bf3114507d7439e9e80faa2dced135
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.42.self_attn.q_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.42.self_attn.q_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9f00fdfa8ebbefefe8b866bd1db69d4ad08037af
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.42.self_attn.q_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4f56964dbdd15bdfc1dde2e29d0b377156399fba709b04930e654579675c7f8a
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.42.self_attn.q_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.42.self_attn.q_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1d74e5c3e71b7dbfc6e8519c3f4966fa12351dba
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.42.self_attn.q_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3dd976772eb5fd24181dde317dc540304a02db71878d1c0764d0989ed26ed84f
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.42.self_attn.q_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.42.self_attn.q_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.42.self_attn.q_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.42.self_attn.q_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.42.self_attn.q_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..83222e7273c2701f7bda70fca8a98968453b71f8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.42.self_attn.q_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c2f49357c53b1d33b93a1381f06fa94da352780600c498c04037e4881eaa0c6
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.42.self_attn.q_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.42.self_attn.q_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..00a775711e846c5010a7b5cb77b3c71dcb2bd739
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.42.self_attn.q_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:544429cd68835e0b522ee8ee556d3ced75056828aef74c7d74680454add05b54
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.42.self_attn.q_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.42.self_attn.q_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fe8b2a9593f5550e3f3b683a3074401a4afbf9b9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.42.self_attn.q_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b09f278fa986171e99891230f937c019e08aaced65201025d2e2983f94795736
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.42.self_attn.q_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.42.self_attn.q_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.42.self_attn.q_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.42.self_attn.q_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.42.self_attn.q_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3ff39e989c230c48e43c170dd76810749977ddc7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.42.self_attn.q_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3a86f5e89dd707787b5cf7d1627a26edbd6641216a9678c54afdd76652bad22b
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.42.self_attn.q_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.42.self_attn.q_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c2805a7c205c76531880a9abd05c64fe983375b0
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.42.self_attn.q_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:66c5f1b646b63997d296b35ddbf17c92a9fdbc572dec112d467fc79d548508fd
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.42.self_attn.q_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.42.self_attn.q_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f0bf3dd851e2434ca921d6fd58de21b2b45536fc
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.42.self_attn.q_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0e98b177094440baf97835a99af00476ebe10593be97f140f2f7e26fb5f07db0
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.42.self_attn.q_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.42.self_attn.q_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.42.self_attn.q_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.42.self_attn.v_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.42.self_attn.v_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e2e572204d0f191339d96b9832e30a23830dbff5
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.42.self_attn.v_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d69b5ed1f3bcb21e375b9876ac152afd65ea02476a7164ad3da1b01763d7748
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.42.self_attn.v_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.42.self_attn.v_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7126f43f6f49c11e5c30c30fb044050a47dda60f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.42.self_attn.v_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6efb866ae70246335454fff57a07882f6ddafce195a826d3aec9c4fd98624f0b
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.42.self_attn.v_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.42.self_attn.v_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..678d9d8a38bbc98732dd5c6b7e488036011c7c09
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.42.self_attn.v_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ee4793c8ede9c9205c5360b9b27a0949c05890d7a87a6007bc19e28f5692ad3d
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.42.self_attn.v_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.42.self_attn.v_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.42.self_attn.v_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.42.self_attn.v_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.42.self_attn.v_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3dc6c28966713fd63b3250e02898c3f41504bd47
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.42.self_attn.v_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8ae200122aa069e3ac198b711f387a24a9ef3a93ade91e46e245c5e5ac236aef
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.42.self_attn.v_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.42.self_attn.v_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7e346e89f6a4e61754c6b5e94271ddcef8670419
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.42.self_attn.v_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f98d0ce007d98ef2ede7ce7373a83dad62054e0d07b05292d7f7c577b7cc9b3c
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.42.self_attn.v_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.42.self_attn.v_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4ab496ad78e4ff5d1f838e38852db93373edf771
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.42.self_attn.v_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3d2bb91888c199f3ebef45036ac2ee224414a44f55a8811be331b58cf41b1fbf
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.42.self_attn.v_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.42.self_attn.v_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.42.self_attn.v_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.42.self_attn.v_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.42.self_attn.v_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0c251df54f9b37ee666072fd9df75e1684d17d0e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.42.self_attn.v_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:17660434764e1dcd27027fea63f1574cfff17d29b7a96528cd52dfa47a48ce74
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.42.self_attn.v_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.42.self_attn.v_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..edf4eb2407d8719603157e88a0bbcc3fed940525
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.42.self_attn.v_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:19d5abc75fcd74606597ba2ab1d7f367fb9a256506312fdf411e76d371ee5c6a
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.42.self_attn.v_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.42.self_attn.v_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b384bc59a98789507f7ed3db68c30b11b6270806
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.42.self_attn.v_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a6295cb111de29a84ac82743b5cf76e76ce5142422b125bd39370e4baaeb75f4
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.42.self_attn.v_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.42.self_attn.v_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.42.self_attn.v_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.43.down_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.43.down_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..63b2fb9a94118916a8803239677001c6d2ae16c1
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.43.down_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:90bea8c0fb0a82406307f0ad3b12ed95b38435f557fba045a2bffa7400576f04
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.43.down_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.43.down_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..06ef7f5e20069200ca825c8da4ea4e583eaecc03
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.43.down_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8b14b3a2e83a0ef07ff0f18ea19f9d53925c199f4c818f68b1f4ee251e2ea3ee
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.43.down_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.43.down_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..200d82ba82b5eb7afe73ff553b25a57073a12c69
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.43.down_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6222f49389bc3950449206f905254fc09944a480b07d7cf003e52b6540471915
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.43.down_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.43.down_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.43.down_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.43.gate_up_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.43.gate_up_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4ed3081fc574cc788ea9da7ab97ff568c145bd47
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.43.gate_up_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ad6ebe1958188b44b58fc9d2b413113b35bd4a63e7135614963916aa853b85e6
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.43.gate_up_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.43.gate_up_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..54ef0c26b76307cbab3b74f448911ebdb907f8aa
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.43.gate_up_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d39b1c9cea8126ae37420f28877027cd41d694bfdbe62c799284fcc5cd8dd8be
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.43.gate_up_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.43.gate_up_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1d3e4c4200902661ce007d37e5fb9ae477471fd9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.43.gate_up_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:adaf4341920efdc7348747884346442fff9aab01bafa21a68a1b39cff30eb0cf
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.43.gate_up_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.43.gate_up_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.43.gate_up_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.43.input_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.43.input_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..208a95994a33c73a8471e781000fc8437440387c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.43.input_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6ed4e55d357d20cd36fb75da24240e8326c2ba742cd9d84a67a9d3bd7f4afa3b
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.43.input_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.43.input_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7661cfb0ddb8ac083537723f55467e657911e59b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.43.input_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:916f3d710c3d93cd1b973818142a27e3c908c028e911824548154480ab7831b9
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.43.input_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.43.input_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3a5dc8d47836fcb2280b7e1232d2dc62d01f4b1b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.43.input_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5ee8215c880173993b62e297c1a36277d5ad7c0e935d4d29e7273e09c746f7f5
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.43.input_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.43.input_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.43.input_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.43.input_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.43.input_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7d470dda28c6a1d3f3dae4ee34e18b7518f4a3b7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.43.input_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5ba139b66c96efbfdee1458061968664ea4d87312f58bac2f4a8a2012b1bc7cb
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.43.input_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.43.input_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..de54bd965ac236234b6f7c0760a2b2fb213055d9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.43.input_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:52825e50b399d795924b71283de894f824d2a943cb968839ed8d252219c0f124
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.43.input_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.43.input_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..84bbc4f9a72e06c840f776015cc754a15c05c7f8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.43.input_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:60b6f5eddea7ac06984f74a219ad270d58f41e19b7496df90b1b21ee7bb27002
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.43.input_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.43.input_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.43.input_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.43.mlp.down_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.43.mlp.down_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d45263fc38f432a584520a8b2975cd5d24bab5ec
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.43.mlp.down_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0ec619c2a9bfc5afbfd7de80852edc133767b95d177cc95eb81955bf9a3784c0
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.43.mlp.down_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.43.mlp.down_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6da09c4c8a114ad88c9181ad9b15c0b5e7514728
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.43.mlp.down_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ddcfcf240209f224d7a774a6928a9cf440d6c10fb27fe466b708100f31ef332
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.43.mlp.down_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.43.mlp.down_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c62da022e70147add8e47bc589bd5f03235721e5
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.43.mlp.down_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2c35b199e9a00cbae9975f41f323234b2e6525fc5e8428a73f4da3f0c5f3edb9
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.43.mlp.down_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.43.mlp.down_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.43.mlp.down_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.43.mlp.gate_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.43.mlp.gate_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a21aeebbf7be8fdac93b52d8bda20397384c278f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.43.mlp.gate_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cff980e5d5949073d2adad1e7843087fab1a9348cbf801799cafd98765900520
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.43.mlp.gate_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.43.mlp.gate_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5f2aaa802082515287f6588b7aabfc0424a3f430
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.43.mlp.gate_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:db0270df68d49178216f978a0e003b15a058ebead6a3fb70f104f2627f8883a2
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.43.mlp.gate_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.43.mlp.gate_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8d4bb83e1641216ab7b0179814cbe87968b4aff3
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.43.mlp.gate_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2e5b7a8f8989b061fd77b597dcc7d0c18a3d74d84e3450b66ce83b050d44196a
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.43.mlp.gate_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.43.mlp.gate_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.43.mlp.gate_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.43.mlp.up_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.43.mlp.up_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c480590d63a929d3480fcfe97eaa374e5320873e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.43.mlp.up_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:477f16d62127e3e8cc0fdc366aeb877dd09a1ec629f006008f15038c1b15270e
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.43.mlp.up_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.43.mlp.up_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f9a82e84f7c118895934e807e8776004c132294d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.43.mlp.up_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2675b96f21d16c1872617d9d002c3c9d39d27701d9ff2fdd0aceecf9a58aec95
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.43.mlp.up_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.43.mlp.up_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5a069c30f633ab42de58b7352217e9a6163f8252
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.43.mlp.up_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4057ac01ddb37173eedf65d0cda575a361eca99c08f0ae2605d28341e8d3b897
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.43.mlp.up_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.43.mlp.up_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.43.mlp.up_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.43.post_attention_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.43.post_attention_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a2644cf4f345f6529b72af1dd110f723bd28d37d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.43.post_attention_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:75cd35c4ddb40ed4a6baa96d13a80e7e5060379460b54ab01fae17246bc2f903
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.43.post_attention_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.43.post_attention_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..58bf9727c12aa8bf5d0e6909eb6970523ee07227
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.43.post_attention_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c3da5d732f19fccf487d24aa0fcb75618f6eeac482d91479bb2f1fe7c381494d
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.43.post_attention_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.43.post_attention_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9ecb6cf7de4a567c51c20a7f30d370e737fdaa27
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.43.post_attention_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8d59d691f3c3d3f64bf94942e096df0cb2cdcc982820b00b202026abfe21bd93
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.43.post_attention_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.43.post_attention_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.43.post_attention_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.43.post_attention_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.43.post_attention_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..010731808198e98c42d473cb638031d98dd43962
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.43.post_attention_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bd9d1f164b8f4af2c9a125c2ac018fd5acb368884366b3ec33f878c1c62fe73e
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.43.post_attention_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.43.post_attention_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0f5e7309a8e76b8b68330f3422f25444fec5dbe0
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.43.post_attention_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e9fc1279ce7d02d7d7ca8755ee21c84faf4cda34b937ff46d639242f9d4aa7d7
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.43.post_attention_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.43.post_attention_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d317c5b34397c5504a0db82cab02d93dfb2d3aa0
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.43.post_attention_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a72fe7fe2f802fa17f353cfe48f0b7892a1cc94ddb2c9d349733b39650127684
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.43.post_attention_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.43.post_attention_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.43.post_attention_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.43.self_attn.k_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.43.self_attn.k_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7efc5a4a0ad8ce674eb559aab3bcdcf7f45a2c24
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.43.self_attn.k_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ce9c9101e8b3b1118e937d1ad44741797930e12d67e9f9f2451821047b7f1681
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.43.self_attn.k_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.43.self_attn.k_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..091ce84fe8e3e37ea2dbee75f0009ac71f53049c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.43.self_attn.k_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:51f6a5b279b7131706dd21b73cd10d0f1d42348a1695820d43ba8e5fa54abb55
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.43.self_attn.k_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.43.self_attn.k_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8834da807993ac614b71c8b5af94d68c8f63a900
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.43.self_attn.k_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fea95054052417e437023850f0d5350e0657a3027bf28134524c8a738d5bd148
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.43.self_attn.k_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.43.self_attn.k_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.43.self_attn.k_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.43.self_attn.k_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.43.self_attn.k_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d3fc7228f36c1fa0539952a601d921316af8899f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.43.self_attn.k_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dfeb40a81f718a0c45959783daa95ee765f727d3d4fcb5fab596bcfc6325c96f
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.43.self_attn.k_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.43.self_attn.k_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2e7a4fe7ad33230c6991ed57da9e03a117f26902
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.43.self_attn.k_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bd130e36dd3196ac01cdda5750c86d4b651336ce5b565e2f7b1d9b7f0a387f72
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.43.self_attn.k_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.43.self_attn.k_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2c96347b8f06934c09a7153adeadfce0028153a6
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.43.self_attn.k_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b9b1549b2e58713060a31a1a9cc893d2db31242974fa196af039c7c066f09a62
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.43.self_attn.k_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.43.self_attn.k_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.43.self_attn.k_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.43.self_attn.k_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.43.self_attn.k_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..678c7f389371c31ca31f21d5bfe8228289bb1405
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.43.self_attn.k_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b087ed99ef11eae72ff583c6618e9ec94def2b7167942e8b3fb2af18e5cc5405
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.43.self_attn.k_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.43.self_attn.k_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..62fe27e7a3848c3bcee67c1f887e566e75a920f9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.43.self_attn.k_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bd6c0d492216e417667c2ebe96e8e24eefce74bec2a5053a3332992bdfe8197a
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.43.self_attn.k_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.43.self_attn.k_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fee64ad7829f88a8b58cca2547f53e6d3b58386a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.43.self_attn.k_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e1f414dd010086f286cbd8ff9619d8973cfb14aeba5c11c9b2d7b5806bf3cf78
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.43.self_attn.k_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.43.self_attn.k_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.43.self_attn.k_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.43.self_attn.o_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.43.self_attn.o_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..980e32faf0939910e4478f00aa4f9ea408d8376e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.43.self_attn.o_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29f953927bd038a34aa4011688d6633ed4043c4f5a6eaef9b77f2b79f9fff773
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.43.self_attn.o_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.43.self_attn.o_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fb1ae9ee8c9f5357b79c9010aed2f2ab86716c65
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.43.self_attn.o_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a91931f3bde2e80979c84641bf393ad64cee81b1cbd4a305db77fb7b74636375
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.43.self_attn.o_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.43.self_attn.o_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5048827316e1faecb9b6577c3014dac426c7643d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.43.self_attn.o_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9656ea28bee5f670c5a91e0dd9a0fdce3d31d7c6fb5bd356cbde95ad9016f6d8
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.43.self_attn.o_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.43.self_attn.o_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.43.self_attn.o_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.43.self_attn.o_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.43.self_attn.o_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fe6d9f95c04f21ce1cdd7b1f02ec974f35b92d8d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.43.self_attn.o_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d11710c86284a773f312f6d3cf125fbc493826bca3ab96c9a292bf596d7df7e5
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.43.self_attn.o_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.43.self_attn.o_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4f1f94bfbcd62ab8a2b4cd1919798b3aedfed1be
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.43.self_attn.o_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a9538b623af32339f4e743f964f7272b0538d5ddd2867a2e87e91e0ed182153b
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.43.self_attn.o_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.43.self_attn.o_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..79508463000d7c56d0537ebf13fc819cf1bc121d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.43.self_attn.o_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c9302aa75a8284c54896409a1e78b11749382d5161a1aed4e9ea8cbbb1f3914e
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.43.self_attn.o_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.43.self_attn.o_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.43.self_attn.o_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.43.self_attn.q_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.43.self_attn.q_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e3c557535d04d57338cca5de72e361d8d6f53291
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.43.self_attn.q_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b462100ea156a5a4c308355f4993eb6d08b31a72a8ba7db48aaaba982173014e
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.43.self_attn.q_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.43.self_attn.q_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2d6ce6265b735d18175d5b32437c3a7f51b2749c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.43.self_attn.q_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f66482532c240b1ada637dfb6d1ba386dbcab5cb2fc80c33b7ec77028891894b
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.43.self_attn.q_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.43.self_attn.q_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..50d54663669ac6d02757cadac195c82c0e08a20f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.43.self_attn.q_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:38f07054835697b2f13bd424d71442b1baf86d1ac86ca79964f8141a3bf95250
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.43.self_attn.q_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.43.self_attn.q_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.43.self_attn.q_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.43.self_attn.q_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.43.self_attn.q_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e1acc836142c267648a2adef42bcc44379d44efd
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.43.self_attn.q_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a47c5d4f49e9a5351a62853a92d29d9c325b13e8aa9b692f942e9be56ea805e7
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.43.self_attn.q_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.43.self_attn.q_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..435460d4ed7e8df03f32165d21ea3d7522917a4a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.43.self_attn.q_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b26614a1f3ddb11ca69389c96c1c9da4762c1b5f80380b23b79ea84fbfb1f7fe
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.43.self_attn.q_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.43.self_attn.q_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65d1aefcc4cd7494c3daa85b6e25a98f0d42d315
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.43.self_attn.q_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e20e0e369a435bc160ec6cb98cba5ee057ce1c0e84247198175e84cc9d040ce8
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.43.self_attn.q_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.43.self_attn.q_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.43.self_attn.q_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.43.self_attn.q_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.43.self_attn.q_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f7a39139f47eaa88203a13bc59d9b1d7a3f24970
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.43.self_attn.q_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:71f0d9232c0eff705d7fb106801f7071d7a62d360e3dd2d96c4dd895250243cc
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.43.self_attn.q_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.43.self_attn.q_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e97929a6ff37637058507ffd9b64d6de2bcbab4b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.43.self_attn.q_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8a270a7280450f4b21468146aee9367aed7bfb63306637c858dede67e2552249
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.43.self_attn.q_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.43.self_attn.q_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6697305b173cfa1decf13f1e4eac46f3b05b8734
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.43.self_attn.q_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d1862bdbc6d50716aa60b771352bcdd35b18e8160a72f61a880df82f7cfb7169
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.43.self_attn.q_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.43.self_attn.q_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.43.self_attn.q_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.43.self_attn.v_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.43.self_attn.v_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1c6e7ef9eb169db24bd5eb357401dfec86ae6442
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.43.self_attn.v_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cd520ce68fa1b08614608f2581526eab24be13e9b68433b9b8f5e52f97defff1
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.43.self_attn.v_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.43.self_attn.v_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8f22d5047fc0d65d3b50351099f3a0b8cf9293ef
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.43.self_attn.v_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:18da84fa87db8a3297772856b2d7a7b7bb1d9c677f61c3f8f340313528a03a6d
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.43.self_attn.v_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.43.self_attn.v_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fc0d642d20cc7cbd779fb35a677ff807afa48361
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.43.self_attn.v_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a90a958fce00337d5766afcb2419a61272f6213347bb61d7e1ebd956c18d5fc5
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.43.self_attn.v_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.43.self_attn.v_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.43.self_attn.v_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.43.self_attn.v_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.43.self_attn.v_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7405619724035aa897d8c9c37653d1b15343b75f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.43.self_attn.v_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f524bbe79149bd5845b89d0b185d6424800a192588c5752b654598b29ef1cf7a
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.43.self_attn.v_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.43.self_attn.v_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5974c6f49e439f6a95fb5a990567384911d6b97b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.43.self_attn.v_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ed95efcfeb9b05cafdc8e611139c8a3ef3410faaa9df728f486061e95b6cbaa
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.43.self_attn.v_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.43.self_attn.v_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..23ef9f6a3cbbaa1a9131db09bfa769c5ba918883
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.43.self_attn.v_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:93636b6c7234a6c4f2e56d687d92e5ac3743dd444ff09c68da51e20f18164295
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.43.self_attn.v_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.43.self_attn.v_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.43.self_attn.v_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.43.self_attn.v_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.43.self_attn.v_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0fc026f15d22dc65634fa41b0f9216c5304b90d8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.43.self_attn.v_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:94b0b2724c22ef5c1db2834697be786292dc055f1e513774896e78fc64d08172
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.43.self_attn.v_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.43.self_attn.v_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9736b29a6cebe3165e9558698c771ab8a1ff3c63
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.43.self_attn.v_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aa1a871425b327d3b8328df499fb0743fce3e9775e13ddc5405cb7fdfc0c175b
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.43.self_attn.v_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.43.self_attn.v_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4e6621799131b2525122774733875446d9874b45
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.43.self_attn.v_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6bd2c0bfeca513f9f07ac2cafd5b32426ebfab55ff9ee18ce91ed75822b9c09d
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.43.self_attn.v_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.43.self_attn.v_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.43.self_attn.v_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.44.down_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.44.down_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..82ee6cc1bf4d65cd6748fd2b96c476c80edfd86b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.44.down_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:403b802f4b3be4d495513e2481a9110ca1258c8ddfcc8186d5b93aaedc235b78
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.44.down_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.44.down_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2e2f683eff4d50ca0d361c983f990a924439f7c8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.44.down_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:44d923fae88a216096bbff56d21a1bcd7ea15abd7d2927e7271f2a162019b083
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.44.down_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.44.down_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..14c936d420fae742a70a0c0d89ea187273cb2cb2
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.44.down_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:df1adf03506af41a67e87721718b734780687c7e8a4af7f4153c1c1f77a7c403
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.44.down_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.44.down_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.44.down_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.44.gate_up_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.44.gate_up_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2ca54724e54e14ad37af0c0095bb6b0742a8ca34
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.44.gate_up_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:99f2a8a6667568d7e73205a8b4567f4c5a6667a5677dd8b69d4a79a91093cd10
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.44.gate_up_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.44.gate_up_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8e1606cc76b1bd64b65c4d42df31252df35adfd9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.44.gate_up_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:333f47f0d51a77da2a9eb222f8910ac930db758efb62f884868f02b2628b6cfa
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.44.gate_up_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.44.gate_up_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c90981f633d18d9c0dd666d325bd2fdf78ba9efa
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.44.gate_up_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1b523fff8406d99b0d10402739da3eb90fc3caf9a68b962895c8159bc94d629f
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.44.gate_up_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.44.gate_up_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.44.gate_up_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.44.input_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.44.input_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fcf88c815fc7ab6205f110844326c898dae54781
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.44.input_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:00d1ec51965de49080b5cd5d28e6be8bb722a07ff2347796e4dd55ab10006626
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.44.input_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.44.input_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..46642810519f843fe5c3e0ea881d8b4392549a3a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.44.input_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3912b0f4d26a48c441805f25d3e3ed1fe5b5e7e1385a28ba16364353831cc1b0
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.44.input_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.44.input_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..db4970e07e37b0415fb1801f55b06c63a1a3c75f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.44.input_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f2f433287cda897d96d8baa4e1e6e6ee7017f786b8961ddec0194d8a67e7d862
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.44.input_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.44.input_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.44.input_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.44.input_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.44.input_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..82461c16a2a01bbfcfd99129e563f603ed21432b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.44.input_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ac4ea812c7324a226df2b089e5a8be150ee9eb3f32ca58c6f958b13b0a52ee75
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.44.input_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.44.input_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6fbbd779548e1c310dc9532e4531612f1d62e9ff
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.44.input_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0021e7830a6878c5a0aed4cdf7cb00aded3111a73be4ed34f4c45c0feb62e783
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.44.input_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.44.input_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..16559c60055f7d13fe84c33cf905976abfcba401
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.44.input_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:24493d5f8da08298c7c51a5e4e63b4848bb4291d3af8388e77124c6b352eb4c8
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.44.input_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.44.input_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.44.input_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.44.mlp.down_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.44.mlp.down_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b7bfd29a00e796888e4a19482d4ebd74572f6421
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.44.mlp.down_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8b228be690d5afc06a01b9ca8b0ad675759113b9e55b6002fd9ad3d035ada9ea
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.44.mlp.down_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.44.mlp.down_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4dc33c64be14e34526a3d3a666a21d3d90e82c4d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.44.mlp.down_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:73e9022037342c4cd2cec0108d2629554dfa2ff463bd017ebc5af9403afc73d2
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.44.mlp.down_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.44.mlp.down_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..da15df0f0c566cd9d76c39e79d430d488a83c72f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.44.mlp.down_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6297a018712f96beec7ef824d14382ad9dda33a6a3e33bede60bf50a48644f4a
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.44.mlp.down_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.44.mlp.down_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.44.mlp.down_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.44.mlp.gate_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.44.mlp.gate_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..253005d3127f20861baf870b313568c07036ccf3
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.44.mlp.gate_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bd0e3ab6b5daf48996db290438a16e9df1b79aecdc1885269785d42a3c95925c
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.44.mlp.gate_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.44.mlp.gate_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fe7514c461682efaa1b10bee01fc24d2c6a40863
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.44.mlp.gate_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a5551b9fafadec768800d27493c658ce3d2b7fa0ee81594b8de281a585ea6dd1
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.44.mlp.gate_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.44.mlp.gate_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..35d68aafa322a76d41ca38ed9e33910d7b0efaca
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.44.mlp.gate_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d6d849eb178509f57ee680d1dedd001049ad62c0f57998ed6d0c512410f60f22
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.44.mlp.gate_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.44.mlp.gate_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.44.mlp.gate_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.44.mlp.up_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.44.mlp.up_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c2b622280a857ffdbeb7745b14a72a6301a18344
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.44.mlp.up_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:33d93b62bddc140c9a15da6dc329640638178dd8744249533175a320340e4bef
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.44.mlp.up_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.44.mlp.up_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2b3fd964853324e3673dbc3223b0efc9d08fdd4e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.44.mlp.up_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f439604241d4d91f506de9da5a8bd4cbc19bbc7299c2569fdc1905ec6c1aae56
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.44.mlp.up_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.44.mlp.up_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e991833283de202bd80e399b04b344ede072863f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.44.mlp.up_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4c93a0cabd7fe49ee94e26e257719b1fb5d183ef0ee6e24735db46e8cacf88cc
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.44.mlp.up_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.44.mlp.up_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.44.mlp.up_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.44.post_attention_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.44.post_attention_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..df048296bc4a79f6a48d86d5ea6ada32239a760e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.44.post_attention_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e99107669bb635022b627dcaaf79b191c20521c7757cd00068e700fa8c32c363
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.44.post_attention_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.44.post_attention_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..346880bf66a14cf1a087539d62c7812a45029e28
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.44.post_attention_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e6c243debe626b62b28f0fa964d6a9973e93a7da07052707bb73856fd380c7a6
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.44.post_attention_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.44.post_attention_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8657968c0e40b398a820451fbef89952e823b575
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.44.post_attention_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6542026f6fb07c1ca76b9484011a61c0daa8595c8f536357dbc5dff89273fa9b
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.44.post_attention_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.44.post_attention_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.44.post_attention_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.44.post_attention_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.44.post_attention_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..06fe411edeef80a3cc9c9812d06928f95bc422dd
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.44.post_attention_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:57b7bde2fc2c94309a87075965a47068d3f4077657a5daac97f9fbebdc83ab95
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.44.post_attention_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.44.post_attention_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b3ba7d43ef72eba22ebad1adf04e731c86f7bbf0
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.44.post_attention_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8e33a9984901ae96d09db6fd19db86b6bb3a2ae058986475202beca3ce46f993
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.44.post_attention_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.44.post_attention_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..545d7c588769ebfc4f2e2148fcc3d92b16f059bb
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.44.post_attention_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6fb711ee2f5d76e3cdf772e551c314a3d9edb6a85bc0e4e3f6780b0c74d2a5b5
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.44.post_attention_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.44.post_attention_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.44.post_attention_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.44.self_attn.k_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.44.self_attn.k_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b308b80af1273be647bdd383a3ec7d622f6a5a4f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.44.self_attn.k_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8044dc45bac1708e9d022ae107ff820a0731565867f16216ce81a3508b4071a6
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.44.self_attn.k_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.44.self_attn.k_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..72986e346edb9448e40e799a48d46d36d5544c6d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.44.self_attn.k_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cb91c78ce53828163787e75e9331674ce5151ad722d416bbbcd41440ef206552
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.44.self_attn.k_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.44.self_attn.k_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..44e93c9f33037cd594a7a72fb97cc65030c3d22e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.44.self_attn.k_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:33dffbeca2c5bb6db746b3eac46f37f9a7434331bc526fb9c020aa482064a52c
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.44.self_attn.k_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.44.self_attn.k_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.44.self_attn.k_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.44.self_attn.k_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.44.self_attn.k_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..28cf81bde65570a62cbead2a9a65c386d9840476
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.44.self_attn.k_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cd24136cf7741aa6e572d5fd7a78358f01a190fecda90769e7d6cb3d22af4144
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.44.self_attn.k_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.44.self_attn.k_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..78b9c4bcb3de96baef81ded46600791e27df5aa5
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.44.self_attn.k_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:12d0dfe66f3a4aa0020ab6fe5d27f4f90d702b772546a139c94d3955e844cb2f
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.44.self_attn.k_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.44.self_attn.k_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0039db45fa4849692df5b2161f8cb47165959ef2
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.44.self_attn.k_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:36d893558ceb01206312087a11ac47b6c18727f005a8ffa9ca95d3aae2a2fd20
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.44.self_attn.k_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.44.self_attn.k_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.44.self_attn.k_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.44.self_attn.k_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.44.self_attn.k_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..838eb81f67a8667ea08a7cbe036dd5535cdef07f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.44.self_attn.k_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:74c9f981c2f36c4128db2e279d73e3b675cd5f90246b1c2e06adde99b9b877e2
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.44.self_attn.k_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.44.self_attn.k_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..755d7937c83aceb6ab54b77bf88680b9b7820a83
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.44.self_attn.k_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7b06f027e465f2c7b675b76f2a43bd1a4010eaffc9e40aa90e78f3827355c5d0
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.44.self_attn.k_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.44.self_attn.k_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2a0004df0d44eea5e7ccb5f54861f8c75377f0ac
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.44.self_attn.k_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4fd5f5cfec763edbea068bef55161e0d1cf319f5c7e875f5ca9e382702484c3e
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.44.self_attn.k_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.44.self_attn.k_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.44.self_attn.k_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.44.self_attn.o_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.44.self_attn.o_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..42ef159e9d804b7c5575118d5fd48232bdf92264
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.44.self_attn.o_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9723a85a66c7249c1907f95b4116cc257c2801fdf81f42453b1138157a78c05b
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.44.self_attn.o_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.44.self_attn.o_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c73698aaa11ffe9ab88643e0ef74f728d60a069a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.44.self_attn.o_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c60b9b4858d6f4757cf4107325aed886d1b01337dfca1e1916fbb611060cff5a
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.44.self_attn.o_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.44.self_attn.o_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..883ec1f9b88bd79dbc26492d6fbcac5c1ca2538b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.44.self_attn.o_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0ae7a0b2d4a18471f9e87edb84a41b06b21c475a11362457b42f9076b1af5590
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.44.self_attn.o_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.44.self_attn.o_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.44.self_attn.o_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.44.self_attn.o_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.44.self_attn.o_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2196d43d4c43689c874e91df2822aab338004368
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.44.self_attn.o_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea2ae9f4b886b6cf96f7ef71ebd7e516302bbd146aebd30738db7dc6346e67d4
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.44.self_attn.o_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.44.self_attn.o_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4b5ea6b494cd55eed401df2cf05f8389ba7644a1
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.44.self_attn.o_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:995e4a1b772be4cafe6ecf6f04c52aee8b17221e80c44fc711273d2ab49876e5
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.44.self_attn.o_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.44.self_attn.o_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2f46238d29dd975bbb5d99be882c8369386fa15c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.44.self_attn.o_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d042a665376bf8cdd72ed3c4b0c17aa60c10e84eff323c34231feb3f55eb10d4
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.44.self_attn.o_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.44.self_attn.o_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.44.self_attn.o_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.44.self_attn.q_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.44.self_attn.q_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4d0a8cf7e4cca62e2ed676134d3d5ea72b465dc8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.44.self_attn.q_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:441268c16df5d540f09226bcbda7a304ec2cefe4b643eb67349c0e2c6b9ca681
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.44.self_attn.q_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.44.self_attn.q_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..31485d10b902c78e32d252b75fecd747b411ff65
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.44.self_attn.q_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a3ff58b0e65d436bcd0affdd68763a60ab54e4d0ef4f65855d204aece7fffd3
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.44.self_attn.q_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.44.self_attn.q_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..03a94a4ec62d2fd64c9b6fdeb0d8ade8a33aa12e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.44.self_attn.q_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2e4c43c58ca41d0ebe06b629147a42c56d649b90c29b53ddb74feaa728cbfc14
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.44.self_attn.q_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.44.self_attn.q_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.44.self_attn.q_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.44.self_attn.q_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.44.self_attn.q_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..13fce543347157bc2a95e06afc2588cb02efdb66
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.44.self_attn.q_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9b3a44e17348943e40fbdb326d40128500944fcfd30e1643ce46a8d78545b30f
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.44.self_attn.q_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.44.self_attn.q_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d0b85bace4aade5a8c0a9f12ed842fa2edceb007
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.44.self_attn.q_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:345cee87173eb98ccd35e35544f0b15734a25930f5976c7721fbfa9bd2c9d803
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.44.self_attn.q_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.44.self_attn.q_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..690521c70d8753f40036386a136c13426420cafb
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.44.self_attn.q_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:612e3e86f70cd8ae62a0701eb06954e98f3f7218249dfa43557bf485504a540f
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.44.self_attn.q_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.44.self_attn.q_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.44.self_attn.q_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.44.self_attn.q_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.44.self_attn.q_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7a5353e9a307fe4532c5cca7ca75b3435b2f0aaa
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.44.self_attn.q_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4d0cf0ece46b7570dbc1da29937a5aec1dd0c3608f9f0b524070c81dad8a6edf
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.44.self_attn.q_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.44.self_attn.q_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..76ac130a0e9896feed5427b36f4677a6c3790587
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.44.self_attn.q_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e6e4fef40bc254cb4d7c20fc40be5bbf3b4da2bee5f95c62ebda148e8efbfc11
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.44.self_attn.q_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.44.self_attn.q_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..abb8e6759e7dd16d94d1c691ef740039a25bc35b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.44.self_attn.q_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e249257faa2131d974891d6d3fb1e154daa5c21158cc7cb53e646912d3e7a247
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.44.self_attn.q_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.44.self_attn.q_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.44.self_attn.q_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.44.self_attn.v_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.44.self_attn.v_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c1c517f51751deea9afcf5a3afa0a117d9e2404d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.44.self_attn.v_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:98fa9265240e0ff50caba1fa784e794ce1948a940f1d0f639d7eb9e27fb448c6
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.44.self_attn.v_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.44.self_attn.v_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fe9df8a9ef149fbb609489de491cc066f75f6f93
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.44.self_attn.v_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0e6dbb78de67dea5d033bd3a571f81ffa46113903c703cd9eb010f13249bd1bf
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.44.self_attn.v_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.44.self_attn.v_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d579383e4ff6314c5003bd6e0c5053385fc9d31a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.44.self_attn.v_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3d29b30c86d4950c57e32b7c924586d22f034e256ab32802b88f7c34dd169c3f
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.44.self_attn.v_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.44.self_attn.v_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.44.self_attn.v_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.44.self_attn.v_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.44.self_attn.v_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8941b6ca8e7b1b986d116d585b57f8d86cee45e9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.44.self_attn.v_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5a745318be0b181907c526d69728613a5757b27518eee6074196a65bb3228bb2
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.44.self_attn.v_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.44.self_attn.v_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..28943fbd23b8c1d0177213db8df621038484b224
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.44.self_attn.v_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c535aaeaef7b910658c7c2126c5c55e50da46a7a6f1271714728be8b05f63a82
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.44.self_attn.v_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.44.self_attn.v_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c16bf5600eafadb3b0cf4357723022ff55baa2c7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.44.self_attn.v_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c28194bca617423eeec93675eb4e8aa1a275f58dcf0573ea387ab21748232215
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.44.self_attn.v_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.44.self_attn.v_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.44.self_attn.v_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.44.self_attn.v_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.44.self_attn.v_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8f30d00ade2fa43551dc407fd22fe007b57fe306
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.44.self_attn.v_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c80e5f50c6acab268f134daa9b1f362d9ac312877cd497aea1ffea06f4536e3f
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.44.self_attn.v_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.44.self_attn.v_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..09a974a11464fd23814f03cf14d69722e0d866aa
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.44.self_attn.v_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c1feb4583ef1889e121039744e32456fa8aeddeccac450e5583bdf29a83ca452
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.44.self_attn.v_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.44.self_attn.v_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8d1a6da7d6080299f05f60c1cad6cda7f1705fe6
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.44.self_attn.v_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:51031756b7506fea1a8c9ab3960bc761bc30a9921638e1d7fb260e5dc16fc52a
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.44.self_attn.v_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.44.self_attn.v_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.44.self_attn.v_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.45.down_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.45.down_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..14c462088299729e3ff44068813e8e8e7110b0b9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.45.down_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5eb7195139ca89b7186c64c0a16397a16fc77489e5b497b8322a669c4dc446a0
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.45.down_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.45.down_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e04b94612dd7dd76edd6919b5930f2d9e2bb8bb8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.45.down_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8dd551267f743faaabc943022049ef4e5c60dfb205ee17ad583ded38d505f847
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.45.down_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.45.down_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a6a0d20152f49bc440191b3fe799019647f9d150
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.45.down_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3513f21606baf085e6bb8a8262a3b5e80475d8d97b77772d93e3a41dcff879c6
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.45.down_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.45.down_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.45.down_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.45.gate_up_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.45.gate_up_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2105220ac4a7ffe1fa09747a62e703b3f8ee35e1
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.45.gate_up_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e74ff9a7142ac10d89264d133544dcbc8806eb69acf15e9433fd95573aa73c8c
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.45.gate_up_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.45.gate_up_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ffcc31d30671563ad4682bf560fca27630e6972e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.45.gate_up_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:929453b714adbe088d617093b91c132d6564f564f3d03ce11d31f2c10b978250
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.45.gate_up_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.45.gate_up_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..36de6c8dbb4021c8270b26830e29b761f183abe0
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.45.gate_up_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:15892930006033fbc998aec7554a3532d64e9494d9b8fe884c365350f56e032e
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.45.gate_up_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.45.gate_up_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.45.gate_up_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.45.input_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.45.input_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2d6405343e90c3cb41a541632ac5bdceec4daf4b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.45.input_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:274a76d4bc7e3210e543395f4ec9adee8c9e9fd509987e7ea61542f2b8bfa7ae
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.45.input_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.45.input_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..830140ca49989f2c3335d87f504ed21308eb2f2b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.45.input_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e362f5a9676acca876dff74aa0ca3d589b3162be6651a654995d7c4a8da57ccb
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.45.input_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.45.input_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3384d005dda01986804539088a1441b23d28b1e1
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.45.input_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bdd70734e6bd75c4737ae9550197986bb9d3f68b0e8f17bf09ee5dee8e087b42
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.45.input_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.45.input_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.45.input_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.45.input_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.45.input_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..11325cc31f8527196d5dbb472480a36622b019d2
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.45.input_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2d17fd042c47b6e214e2d402ad3178261f0cba03ba226303b2412c2b07b239c4
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.45.input_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.45.input_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a2bdd5806a05c27d689cafc251d1654dc1a15ca6
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.45.input_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5b05a10d74cfaa076db71bd08016da794188422069febd414a2279af5de683c5
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.45.input_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.45.input_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..84e355e8954caf11cd719bd7015fb963c72e1e00
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.45.input_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:158f178529b75ced9794622ce80f398754368cb5a5d06ebb4f623e31e499878f
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.45.input_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.45.input_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.45.input_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.45.mlp.down_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.45.mlp.down_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6f73ff59280b5b4e3adb21d4268c66c448680843
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.45.mlp.down_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:32f5abce72cafb87023c3674a6256fed9c1a32d5519eaa9d2db4d4bb56e5f853
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.45.mlp.down_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.45.mlp.down_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e8b33678d563ae5ee0ba75820801d34accd3c1da
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.45.mlp.down_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f4fd44fab4f1706e9b07b14100fe9a144dc4f717236cab2e19f0d5ee8f3af47e
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.45.mlp.down_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.45.mlp.down_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e9df9769c6442bf5138391dec35f59cfb8faa0a5
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.45.mlp.down_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4644129e51a052b58862698d00ee090ed59c8f944e1baf811cd79225764a6bc
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.45.mlp.down_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.45.mlp.down_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.45.mlp.down_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.45.mlp.gate_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.45.mlp.gate_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d16e7c6449faad45cd1c03097e4b7a3fb32377ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.45.mlp.gate_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f959d878c1af5a61b4b0f1bb58ee11b5c4b7aea74ec7e51b39e3c2fc43e7c4a3
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.45.mlp.gate_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.45.mlp.gate_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..20c31319f78cc2657e79999dfe51da9799d9cdb9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.45.mlp.gate_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:12c11e54c62d4a2b7918dbb8bc1d38c86ff1b606c2d62bf8ed82470751e78745
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.45.mlp.gate_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.45.mlp.gate_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..bd8473d654242d303fb7b0d62b191346892145dd
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.45.mlp.gate_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3c8d02cf3b1f8998faa36a43fd583b96f7dd9db8c0ba98ff6e065385bb17a246
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.45.mlp.gate_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.45.mlp.gate_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.45.mlp.gate_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.45.mlp.up_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.45.mlp.up_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a251210e66ea283110f3b60067438c6854d7ccc8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.45.mlp.up_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5807003acd8ed962af9268dabd0285ee3672e772bc8d0c9ca7783a98750a5fd7
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.45.mlp.up_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.45.mlp.up_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..26f78820ebe84090e8d151a104d441fa6366b234
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.45.mlp.up_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0bbcf2ed792fe81a54dcea0d6f1d1c8f5d719c22ad02967b1c81c3233862724f
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.45.mlp.up_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.45.mlp.up_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8a747c283065b06160813835a12182fe5183db79
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.45.mlp.up_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ce516125c40ddf814b04e3af27d8fabfa2da0a957465d0ba71d1dc792da4052f
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.45.mlp.up_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.45.mlp.up_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.45.mlp.up_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.45.post_attention_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.45.post_attention_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..eea146576ad38c4c0f6d55809cef893b33643b83
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.45.post_attention_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:731a4a64b235cf10730abab0ec31fbd39ee03756d2e9c8bc70a7cbae6232416e
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.45.post_attention_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.45.post_attention_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..88ced21ac7b977e102058a64f1663e12aa4d77b0
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.45.post_attention_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f32996f0967e5db7f9c6473f847e7e0f1dc710465da5864838b2bc4ad85089d4
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.45.post_attention_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.45.post_attention_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6fe73fc181be094993044078ebf79a63d4490003
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.45.post_attention_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f0ab24b5407541b9162865fcba3c30cae1cf4bce1f4a1dc691b9181d30f0cc57
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.45.post_attention_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.45.post_attention_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.45.post_attention_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.45.post_attention_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.45.post_attention_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..94b3aeef30c73edfec072fbedf5e96534326e6fc
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.45.post_attention_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fa72b37ac3a357cb5dfdfeb4a46cca3d3166250d6b542f900169b79be77a5a69
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.45.post_attention_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.45.post_attention_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..973aa822b556e321fc287a04096fef23cb92f61b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.45.post_attention_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e4f5886ea4b8f31da5e130b0c3347d2eea259ff849cc6f493da36f7a35aa56a8
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.45.post_attention_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.45.post_attention_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..06424939a5a838f61a0b0ba2b570023ac4924182
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.45.post_attention_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dcb2791a6340fb8c027d41f3d1759b1353784bcc6eca2b0a9fa1bb011019b0d5
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.45.post_attention_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.45.post_attention_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.45.post_attention_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.45.self_attn.k_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.45.self_attn.k_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..31b31ce517e6fbf49276fbf9f1da58d6f486ce7d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.45.self_attn.k_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cf84d6d74e6b84acf059a185319600482d51ec24500ef007ea7d731d26474202
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.45.self_attn.k_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.45.self_attn.k_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9cf0b55c071abc603d65f6e1acdeb4b42dffeb96
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.45.self_attn.k_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8cf263b94833a551b1d157f5415828dd058ae0a56179a2f05ba0148624ee9863
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.45.self_attn.k_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.45.self_attn.k_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..160aa77c28be41d4fdab88498a6c89ad7696ac44
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.45.self_attn.k_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:877025d053899e039c8b4d3d5c7ee406259ce78b2c648fd9db29d48d2bd18cc8
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.45.self_attn.k_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.45.self_attn.k_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.45.self_attn.k_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.45.self_attn.k_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.45.self_attn.k_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cd49c26305a76a459f41ce69168389202f764ee1
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.45.self_attn.k_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e3c2355b07aa5da440d76572536f81d4b7ba872ee62ed5ec45342500b073ed3a
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.45.self_attn.k_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.45.self_attn.k_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..90a24bd6e96b638569598b999129eb62eeaf2546
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.45.self_attn.k_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0e316466cc97abb20057c11cab9ceaa84e8b9a7b2cb929254c228aa4807fa02d
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.45.self_attn.k_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.45.self_attn.k_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4a7f921ab22b1db02d62be77cba5f27b153244b3
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.45.self_attn.k_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7e3edda0bd601946866dce692a2002655e0a672e5eb415b16dc7fde91756f450
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.45.self_attn.k_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.45.self_attn.k_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.45.self_attn.k_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.45.self_attn.k_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.45.self_attn.k_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7ee36ce71492e695067e2dae7ac311b839ce87fb
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.45.self_attn.k_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:63fb97cce12bbfda89097b4a2f060507439d9e4e83bd5546b36d26a370f81c48
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.45.self_attn.k_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.45.self_attn.k_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cea14542ec249e12c6b15066c7ce354c68362491
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.45.self_attn.k_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e0c6eef1ef065a55fa6abac6f639c3ff6ea52c33443a582c3f30ee7e453e2bb
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.45.self_attn.k_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.45.self_attn.k_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cff2c23e232c3fd7c49c8a8a2892515fda5dc4de
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.45.self_attn.k_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d9679164d760bc91e6cd2f44c920366bb54d45a8392303df24d7b5ec2dde234
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.45.self_attn.k_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.45.self_attn.k_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.45.self_attn.k_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.45.self_attn.o_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.45.self_attn.o_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9f81bab28b8a72b2e145da93cdf2403375283906
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.45.self_attn.o_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a265f3f0575213c833b0dcac4554f12766e675ea8fe3b761fc04b565a4d807e5
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.45.self_attn.o_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.45.self_attn.o_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..41805ba98a01c432990bb3f12423470e4a1fb860
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.45.self_attn.o_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3a41302a28d1de149020de9dc69973bf062aa8203e3b27b7275bfbd381986b36
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.45.self_attn.o_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.45.self_attn.o_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3c6ba26c9c5b15d5d3ac8e851294dfd2a566f5fe
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.45.self_attn.o_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:97adb20a6913343f4df1f8a589dc6d0e5e68bdaa7b7a6ef7ac9c0010df18eeeb
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.45.self_attn.o_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.45.self_attn.o_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.45.self_attn.o_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.45.self_attn.o_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.45.self_attn.o_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7de3c3772e1851662c3f20c8e1efa2ab2c0abf3c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.45.self_attn.o_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:95accd7b53b41b83ce4b5213d40cecf9fdbd48c80802667d1ef86a1f1fb83ac0
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.45.self_attn.o_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.45.self_attn.o_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9306a4a6e039778725d46b9ee2497e901f89582e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.45.self_attn.o_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c15125738df37999bd72e2b6937df7a1055f7ef7fc2f29752e2deef9a1bafeaa
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.45.self_attn.o_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.45.self_attn.o_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3648521164ed09e58a9da119c2ab0bfdf8c743fb
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.45.self_attn.o_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:419d89b775d253c347eab3fc0bccde6ce9a93b2f2edd6460d86c14cfa35e0896
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.45.self_attn.o_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.45.self_attn.o_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.45.self_attn.o_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.45.self_attn.q_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.45.self_attn.q_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fcfd06a941fe5a0733f29adea8842ab68b97f4ac
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.45.self_attn.q_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3155a1f30951c59788f7140768e4043e60d75f95ce28e14fea35069851ff8e5b
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.45.self_attn.q_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.45.self_attn.q_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3277b106711c8365adf4d066611907c5a803435d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.45.self_attn.q_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6568d5841cc1ceb1016a8d77e581f6313fc9d475215ec08d665d0fe43417ac69
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.45.self_attn.q_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.45.self_attn.q_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..263a55edf63aea27b95083dd13f0d8620e4471a3
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.45.self_attn.q_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:926aea93837e1c7a96c3a14866b204541d968c4a83901affee74a2ffe168f108
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.45.self_attn.q_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.45.self_attn.q_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.45.self_attn.q_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.45.self_attn.q_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.45.self_attn.q_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..54ad8835e5cf9cad3daef28691508a6b5ebe0514
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.45.self_attn.q_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a3ee7849db444ff1601792f79935563852b30b5a2a2b9b9c3933b3e238beabfd
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.45.self_attn.q_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.45.self_attn.q_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e253050f52b5062faadefc84262de8263baa1825
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.45.self_attn.q_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:916018b3bbe5d9e2120310c7e6c99cf4dfd1ec1a47367f0f95a07f5fda23093a
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.45.self_attn.q_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.45.self_attn.q_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e7fc9ae9e48ddeeda3d9891c0a6c1a3108324c92
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.45.self_attn.q_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e7d0e7f1d8a21b4cb98bba34cfb22eb0959cea6fa09990b14333c460bfeb7733
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.45.self_attn.q_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.45.self_attn.q_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.45.self_attn.q_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.45.self_attn.q_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.45.self_attn.q_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..30d61f07eb36c95df1174b176f96210e7d839b85
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.45.self_attn.q_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:13f09d013192c444c6df2256cf943e42c4a43daa0d152e3c15899936163400be
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.45.self_attn.q_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.45.self_attn.q_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c6a38ab251349131f2cc6253ceb415761242acde
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.45.self_attn.q_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0cada81239ebe6cefc503fa2c28d0d7c6a9c4b18779e00b5d956c92f7b5bc865
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.45.self_attn.q_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.45.self_attn.q_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a70dfdac504eac0efcea1b3edc6253d17db495ca
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.45.self_attn.q_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b30d6409db398fe8f87f76922bc8add4172fd56abf78e553ba5a3963bbd67871
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.45.self_attn.q_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.45.self_attn.q_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.45.self_attn.q_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.45.self_attn.v_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.45.self_attn.v_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a86e4176eb8b0815937cf5fdc1571386ef41841f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.45.self_attn.v_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:56517a91c8845b5ef728a288f1dfbb47e82e28927aa7ac003fce3c812af98a68
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.45.self_attn.v_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.45.self_attn.v_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d51f60106fb468c660755244946f56977d7906d1
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.45.self_attn.v_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:62bcd3ea308f08e3fcf0f0aeb2d87258eb503bf063b54f712ff7df897bd09f24
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.45.self_attn.v_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.45.self_attn.v_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..366014ddb9b64c3b847bd40c16299df0f67b1050
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.45.self_attn.v_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:baf640bb3bd05f15c1149db67391bfa17ad396cd522d241e86cb3d9556e5e5e1
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.45.self_attn.v_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.45.self_attn.v_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.45.self_attn.v_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.45.self_attn.v_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.45.self_attn.v_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..042d14b8bcc27a2a3ff1ea3c03365f8fbb118cb8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.45.self_attn.v_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:af5a254230d269b1e0790e0b9419d125d2fdfb416c89bdcc74695e55cbdb008d
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.45.self_attn.v_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.45.self_attn.v_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..db95c0b2fedd4b8d51b290dae6fe7b0d20edb9a4
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.45.self_attn.v_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:06028d8592461e387ba839a24ba86c8573aae6aea1c36846f22d8ba0bc2b85a7
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.45.self_attn.v_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.45.self_attn.v_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fcdd44c9636a439566a44285221d7759c22615f0
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.45.self_attn.v_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bfa6d8bb1239685fefa0512755feec33dad1e6019c86b729c306df982302c005
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.45.self_attn.v_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.45.self_attn.v_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.45.self_attn.v_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.45.self_attn.v_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.45.self_attn.v_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..34a697af72583877bb06cff49886bfc69c964073
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.45.self_attn.v_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:908b84848c69df27b91940da64dcd46ac5563ade355bcd2321712b650112377e
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.45.self_attn.v_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.45.self_attn.v_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5cccc74eeeebafd28b73efaed5111320d79266d5
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.45.self_attn.v_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:779b3bacdacc2aeaecf7de405bb73925feb534920db12a86da43e88957db36af
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.45.self_attn.v_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.45.self_attn.v_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..78d4590f9a38c7e52bc9d784fe78267edadc9118
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.45.self_attn.v_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:901e7baff39e551c88a6941853c21a7e9282d3326853a55b5e720c9c862e777a
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.45.self_attn.v_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.45.self_attn.v_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.45.self_attn.v_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.46.down_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.46.down_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7f8ec3bcb7f487c8bfc1b14a309e23d330274c00
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.46.down_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c1037b5126bba7195d0ba0164c6d862bd25784c2469fe8cbb4e758294e30a0ba
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.46.down_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.46.down_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5c2e81b5369e57ff5ba8ff991dda059fcc1f27da
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.46.down_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9123594e9c193c34e828691d738b3622305d4c3ce926eff6998dde57524617db
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.46.down_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.46.down_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c01a01b8c63d3857156ec192a86c3b32de632914
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.46.down_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a5c2d8abb9dad41475398e35b1eeb83a35e1dcf9933f2f0bbbe1d2eff8da5f4
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.46.down_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.46.down_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.46.down_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.46.gate_up_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.46.gate_up_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..059368b11b6b5cbede44df4c777952c733003179
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.46.gate_up_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eab4bd166c1a0546d0c96b7bbd9fe7264d28a0a7e1c572c0646146e9d06160ba
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.46.gate_up_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.46.gate_up_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3515b20b5763b310f81bd030d69c17cacb8ece9e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.46.gate_up_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:93b11adf866b2840833f33e320c55d4e8c43495baa53fbf23e295c876a37497c
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.46.gate_up_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.46.gate_up_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c8a9fd90bf8f40c489c8fe569a9174a6b23875de
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.46.gate_up_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8561ec6505cbae36b3b4992408d4d6cf2daf3e48b24bf41a03402c49dd82ca7e
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.46.gate_up_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.46.gate_up_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.46.gate_up_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.46.input_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.46.input_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f3e1b6b561eaa3d0bf9feef13effd9e891322f72
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.46.input_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1645b1c7091c4aa125e39f27b8adb40666ac85bf808aa7aa28ae1f17088c658a
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.46.input_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.46.input_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3cbbaf5459c5b3ebf9557dea1a45ee19c446b240
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.46.input_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9f19079bf86270dba6d085f582ff4f8bf4ef60e6732720ff27a81ae34ed8f14d
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.46.input_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.46.input_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d4badd7fb4d7e14e25364ca83b864d68f2a4a4e0
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.46.input_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0bc93c313954d2eb308cfba38b3008e3c647b37df06aa12bd33e2dfe46e3f5f6
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.46.input_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.46.input_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.46.input_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.46.input_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.46.input_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7f03e10081db99089b8a3c9148689b9ce34ae559
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.46.input_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:66e15f60f28d14e4e38ccc90f1c6f4605966f448790676d56352403d22ad0ba3
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.46.input_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.46.input_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..63755b07d2195b987cafeb33dd13b7d798032545
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.46.input_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7ea72aae8cb9336d79563122a89c5a908c0cd6fb7161bff329ce067afe330563
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.46.input_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.46.input_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d718f0debd82ba48f115f5daf500f8b652957e49
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.46.input_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:80982f869db08fc7d69a0645895312c2d2388617193ad589772516805fd91fb6
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.46.input_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.46.input_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.46.input_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.46.mlp.down_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.46.mlp.down_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ded3956b282d417f2088d51481ca919fd03625e2
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.46.mlp.down_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:47eb51d738f41be1673065ebf76956dc8a8f02b55acbc3b0941202c38a55f763
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.46.mlp.down_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.46.mlp.down_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d0e032291f7c4e61d22934a28a074c243f2642a2
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.46.mlp.down_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d94c9e4bb048884f63aa10bfbe7f4cfc4be3301315375d1687e488e30e5eec46
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.46.mlp.down_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.46.mlp.down_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..287cc1d7e350c39ab77cea3eb5a15287f2df039f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.46.mlp.down_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc8f302c28edf211075177de3353ad51990ae67bf726b3b18b4277adbdec5a86
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.46.mlp.down_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.46.mlp.down_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.46.mlp.down_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.46.mlp.gate_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.46.mlp.gate_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9f3a955c03dc79df01d8d1ea6f68943627a500a2
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.46.mlp.gate_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8b791a31a6820af285cf9a7f8d18b8e82539e610417d38e3826f651c788ad527
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.46.mlp.gate_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.46.mlp.gate_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4bd21aa56df02b07d59cd6cef5aa3eb5172359be
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.46.mlp.gate_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dfb84c676981f70f20beddebccad067e42c377df43b28fc5d2435d46b1fbc900
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.46.mlp.gate_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.46.mlp.gate_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a835bf7a130b3c847ebe3dedd20fc716b0b138b1
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.46.mlp.gate_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9f07482da3676b0c674c9e229c487838c9561c1f2dff407c4ce6db0c4000a83f
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.46.mlp.gate_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.46.mlp.gate_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.46.mlp.gate_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.46.mlp.up_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.46.mlp.up_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..58525a01c029ac650be2cb61189514d1755c14ff
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.46.mlp.up_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dfaea97fcad1c80d10f6e489dd9a0f487308fdce976e7d8c153f7cee94b69654
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.46.mlp.up_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.46.mlp.up_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8b6c843752965f3b3074ed8f1fdb2d0c6e2e7645
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.46.mlp.up_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:217d48484e2a15866fc06d3e5f2cbe3a5474ef5dec82a0d8dd600e2e655f267a
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.46.mlp.up_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.46.mlp.up_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..166adb78b09cc145f24b8ddc945f3dcc0bd077eb
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.46.mlp.up_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea6fb999c19991ab946945b97612f2e4db91ffbbebb28430c4958a36edd4be9f
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.46.mlp.up_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.46.mlp.up_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.46.mlp.up_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.46.post_attention_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.46.post_attention_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d5fba55707e32c7a49e1df037feff4856f3a98f1
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.46.post_attention_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a9c26815c93bfbd8edaf4cf5dfee91a873bfee0335ac3f137ea14ecc5ff52e75
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.46.post_attention_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.46.post_attention_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..89a7a60300a2c213d493d389cfc482ec4b441496
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.46.post_attention_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c52350a308395561dcf6e4f2e276c5fd74d639d0b39ac21e928589271e29d6c4
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.46.post_attention_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.46.post_attention_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a7ff2a6d208f8fb11be879d7326811423b046fb4
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.46.post_attention_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3271f0e4d44132d87e5fc0bacdcfb51e2430ab81860d1f3c5eea5e3b5954904b
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.46.post_attention_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.46.post_attention_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.46.post_attention_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.46.post_attention_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.46.post_attention_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..553c762f9ffa3bd680b8793db0eca89526523253
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.46.post_attention_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ff7c8b94c9a8d6ede1ba35ffe82a05c3af15b9c034397d02106743b1cd1f19fd
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.46.post_attention_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.46.post_attention_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..848a338b18602ad72fb7f07a01064e66caf849a0
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.46.post_attention_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d2fca6174855ab46b668fed9e6728406c84f0ee2f9558fd6df82e0a6c6647d75
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.46.post_attention_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.46.post_attention_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a351aa2f589a3a65656f21dc94ee02e032451859
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.46.post_attention_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b71a3ae95ab2795c6fbe2a503e17d33f97059cad0f16f071f7f733aa6e12b46f
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.46.post_attention_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.46.post_attention_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.46.post_attention_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.46.self_attn.k_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.46.self_attn.k_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d1c91d581b30a92fc98be5cf398537f2ed667bcf
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.46.self_attn.k_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe179b48489d9f750fcc9b45758c55156801715a337582b86b702852e66391a0
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.46.self_attn.k_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.46.self_attn.k_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..35d3c8a65d1d943c55832e18abe07c6f8175f559
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.46.self_attn.k_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:22328f6f0d707537579c576282917518c51f7c6485458d1502f1326e75963b23
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.46.self_attn.k_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.46.self_attn.k_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8b5f6bbc99b185b4bdbab3f6027c79e09d56f6b6
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.46.self_attn.k_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:808b3135ac2a5204659f46fee84e59f4b922b52e046a16d197dc812bc56cc6e1
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.46.self_attn.k_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.46.self_attn.k_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.46.self_attn.k_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.46.self_attn.k_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.46.self_attn.k_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c85c45d07f41bc2cf507c1ee88794fa0eae9c5ad
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.46.self_attn.k_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2b9e1dce83dc2e6c0cd95bc7e740e4b2cc1071378ef9b551aa0f219e19dde9c6
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.46.self_attn.k_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.46.self_attn.k_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1b40b8b346acf0abfd84df65f58ec79d520ba951
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.46.self_attn.k_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8cc8be1e7d33545ae2bc2ae31153d6d49c427682d66fccc9c838bde6222d1a9a
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.46.self_attn.k_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.46.self_attn.k_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2f8f450d077a83a9870af5bf39f75144c0dc6a17
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.46.self_attn.k_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0ea09f805f683dabc13177156f48136d3c5b446bb21f58ed0a8b11040ce94ec7
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.46.self_attn.k_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.46.self_attn.k_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.46.self_attn.k_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.46.self_attn.k_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.46.self_attn.k_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..eb04a413e5456c48f2c704e690db48e154632963
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.46.self_attn.k_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cdbc2bdb436a5e775ee5b615a3c77a5d4bfa5ca17fc872802822dc96bdc249f9
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.46.self_attn.k_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.46.self_attn.k_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0f73582be1b9a78ab08b538716f38400c5f8e655
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.46.self_attn.k_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3d14d7a5553fc0bf382efae51672d4add410a1ffe8d0ebb039f9304d8de25650
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.46.self_attn.k_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.46.self_attn.k_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..051c26b1c27fccda2f21b6163a521c318c7c60a9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.46.self_attn.k_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a1461b109d7df3fcb02af0fc41c68b7896311e7a7f93400068b08010b6072b41
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.46.self_attn.k_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.46.self_attn.k_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.46.self_attn.k_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.46.self_attn.o_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.46.self_attn.o_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..811a5a8f0b1377d4566310482c8c46282ddfb876
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.46.self_attn.o_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e8d9d1039c0e0be5e204603fbebee75f7e2d0be832f0ed43d0853194fc4575ea
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.46.self_attn.o_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.46.self_attn.o_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ff2e430234c2c7cd474f3deaee094972205648a4
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.46.self_attn.o_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:79664ca2bd33e6cf8b94cee08945a2cad73ec44d1361e5634bdd742c826b25b8
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.46.self_attn.o_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.46.self_attn.o_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f257ef2edf5bb5a74efbf264975f37a36753cab9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.46.self_attn.o_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8bcc889be7dd3d3b3715286afde8e6b5a8d06ed5ee0a531523751893bb9a64c2
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.46.self_attn.o_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.46.self_attn.o_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.46.self_attn.o_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.46.self_attn.o_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.46.self_attn.o_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a9574271a82ee1d3a0e76e1f9de8d9fcdd31f4e4
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.46.self_attn.o_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cca2522eac280179413bf88e00d9ae0ea6bcb4d41f4158a6125fc371a609885c
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.46.self_attn.o_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.46.self_attn.o_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c447292e9f54c3c1a67287cf1551908361eb50fe
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.46.self_attn.o_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1bdb848090b131a98d5d0dacbf235f232d8492fbf87a2f9555be9ddf9bbce83b
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.46.self_attn.o_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.46.self_attn.o_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f1c25ab3e2b341f3435168b795596c97acad39a0
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.46.self_attn.o_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2b6880b69f93978a006d518e7e31c270a81c45103a6747e631e624ee1d5dacdd
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.46.self_attn.o_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.46.self_attn.o_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.46.self_attn.o_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.46.self_attn.q_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.46.self_attn.q_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2c8b4bfac9706390ba967285921c0d8fee3c70e7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.46.self_attn.q_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b348de5a1b64c031187ff9eef7b7cd3c7636b36908e977a628001dc9a08d541
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.46.self_attn.q_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.46.self_attn.q_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c36d62b214378055c6259ee80b7a377f83ebf6cf
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.46.self_attn.q_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4d34fefecfc404f4195a099b6df1ae57e5f2c87ce197e4461b2b5c9c5376a35d
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.46.self_attn.q_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.46.self_attn.q_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f2b6fdf8af06d6c1bbeea21234e45191f9ead4a1
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.46.self_attn.q_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:47c37c07c5259c2b2128278c7b1b362a9569af2556c0bfafde9ddba66be38984
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.46.self_attn.q_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.46.self_attn.q_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.46.self_attn.q_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.46.self_attn.q_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.46.self_attn.q_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2ab08a236df7edd25aa09d71f5d2e775e3b9dfeb
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.46.self_attn.q_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c503101101d773f0f9fc84065f973de7d48bd56725496ae1f2343ae5a2cadebf
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.46.self_attn.q_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.46.self_attn.q_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..548ee56098eaa053e1c71934f3a8dc7fde1841f1
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.46.self_attn.q_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1d2a76ca9f2c795a1b79b5e3e2f5948bd83b6c6ff3672a84b184e36b128f0d33
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.46.self_attn.q_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.46.self_attn.q_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..08a92e53d6153a9697d1882e767bb42544c32c01
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.46.self_attn.q_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bcaec92a64d7e20e52e9285d4c5396a8a75486208939adda59af8a35261b053b
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.46.self_attn.q_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.46.self_attn.q_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.46.self_attn.q_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.46.self_attn.q_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.46.self_attn.q_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3e546b10e1626e9f39619f4c582877de9828c2f9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.46.self_attn.q_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5b796c1e72980f36c9f1e5426598f83751ee51c23c68d41c87807d18e4ac1adc
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.46.self_attn.q_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.46.self_attn.q_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ee21940ca5bb780cec4034e0724eff3cebab192e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.46.self_attn.q_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:64e52597a53f130dd1605210666e82e8bbb6dd58645f4e289a8345ca50fb6782
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.46.self_attn.q_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.46.self_attn.q_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..eae6ef3fd857d91938497e0b6270c02b4e6d3231
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.46.self_attn.q_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:883a2589252acb5c938fdd634bbce59a5dc6430339c58807af6519cfe94adb3e
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.46.self_attn.q_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.46.self_attn.q_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.46.self_attn.q_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.46.self_attn.v_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.46.self_attn.v_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..71fc50a0e239d3ba6c941b7f2670446dda7e1417
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.46.self_attn.v_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:27fc7ae55cabd6da3a7d97e7112c68472521547de16e5fcd0f418479653a89a8
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.46.self_attn.v_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.46.self_attn.v_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f58826f58d0f29a8ce41f1569ea2e1c1a1076740
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.46.self_attn.v_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:57c688eac57fe5ae2b94e11cb5ec1bce5eae7270787854dce7da438f0c46a926
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.46.self_attn.v_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.46.self_attn.v_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..26b5b263f947dc5cc9f5398173b3aefa81537398
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.46.self_attn.v_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9b7cdc888aeecad780663c3e198db8a8c83ee6ab8f2ed274b8ef691a93fe1d95
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.46.self_attn.v_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.46.self_attn.v_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.46.self_attn.v_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.46.self_attn.v_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.46.self_attn.v_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9568a534db345e29fb191ebc67db0bca42c8a10f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.46.self_attn.v_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c8fda0258a836919ec5de4681741439f19ef36a52b4fef70b71b1dc3fd0e63f4
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.46.self_attn.v_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.46.self_attn.v_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..192efccd32cbe5360565443ac9f9bcf4b2a63873
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.46.self_attn.v_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1bf1011b4b3a24bfcf4ba2ed45e5d30b8a2e87bcaaf8a4c4387f5e4e65d3ea28
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.46.self_attn.v_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.46.self_attn.v_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ecd0e61aa0ff5dd56fb0442650c85d7b27e53cd7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.46.self_attn.v_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d56791ee197a62c8270dd2f92d3af3ddb9c377c8289a2678201b896f4ee63e3c
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.46.self_attn.v_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.46.self_attn.v_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.46.self_attn.v_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.46.self_attn.v_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.46.self_attn.v_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e19be9f52598d3876ff27bf29f8f553633b76d38
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.46.self_attn.v_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0c6df4351945710b2046dabe0ad15c3aca09ab733c25c68f6f4c5dd9c9bef447
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.46.self_attn.v_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.46.self_attn.v_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cdcb5985b27e469622b8abd288f8af0729a6607a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.46.self_attn.v_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1aff087524d3f1ddb9fec345e1e140e13ae16117483a22b8be37a8ec47e96b5c
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.46.self_attn.v_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.46.self_attn.v_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0d44acfb49db7777baf2c97d90d1401bf72f7676
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.46.self_attn.v_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1f7ca35167d71be2ff50b354096642bd9d25310b0c40afa7ac0e1b5248f1558f
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.46.self_attn.v_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.46.self_attn.v_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.46.self_attn.v_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.47.down_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.47.down_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6e80a331c3be1a9cb6b02ee5331d23cf0b4ce373
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.47.down_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d18ae317de0191b63444112b2db909bdd82773a7ea8469ea539310d27d8a4ed4
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.47.down_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.47.down_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..04bc85362d3cfcb29f4f5d1fc9ef9deb61c6b446
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.47.down_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9adecf5a17829616326388489309ec7a22f469ce926a0eaf6780a63f16813134
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.47.down_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.47.down_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7203a210164c1ee9780020f5c5e39380052dcf03
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.47.down_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e713d5ca5788364e284866ef3c5859fa968d76915324abe0f21239134e85fa6e
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.47.down_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.47.down_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.47.down_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.47.gate_up_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.47.gate_up_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8cf7357a365394c05187e9695a05e5dfcd25a833
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.47.gate_up_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5b6a3258c3ac638896d374543499cf9992928aa956444f011d7067ae5edde51a
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.47.gate_up_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.47.gate_up_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d32f776bd0c8ca5ed462b7e83a0aefcc18a7f8d2
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.47.gate_up_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:44131c3dd58629bac04a16aabf2e74f5a253dd7789b92e941a10d3b67268a286
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.47.gate_up_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.47.gate_up_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f3d28ef3f004ae15a56a4bfc972542b04da7dae3
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.47.gate_up_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1f9a7083e73419e35bd0ca3c5d3a03cfcf91f0219f7bc25c26d9334a699638b5
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.47.gate_up_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.47.gate_up_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.47.gate_up_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.47.input_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.47.input_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a3f8b67cbd65cb8d9d1df7b50cb216d5c8212c0e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.47.input_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be402aa4fcfb7242064f912cf786f16c9c6d423f677141463852766fa4f4c7cc
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.47.input_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.47.input_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7dbb12840ba2ad56f097fe84579fd18efc8b7274
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.47.input_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2d65170a16850e3bc7656511003c71542a0f0a928e699d65167d5a08b6a1a9d8
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.47.input_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.47.input_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0acd3f5ed7a315336ba5e12801396e820666be02
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.47.input_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8475a9bb735de7b04901fa7d4d19d5742c2be37638d5a6c1d889308383c803cb
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.47.input_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.47.input_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.47.input_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.47.input_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.47.input_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..59abd8a646f249fd65ac2848debc459f18d5f492
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.47.input_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d03fef6a5bdd2e275ff140bdb562793464f1b8b1da8161f3bbdfaae067cf1390
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.47.input_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.47.input_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..67e363254fde9955c2e00fa1fccf267be7095c84
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.47.input_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3e53508dcca3b7f6fb6216574b94e9e7923236e682056cb4b776653237a5a0fb
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.47.input_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.47.input_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..92e248e39b16fa71577c4d3508ca75f39c256e3b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.47.input_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:385682944eec054eeaef21d8e0d6406905acbbcf4c1a058c8a93d7ac16459826
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.47.input_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.47.input_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.47.input_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.47.mlp.down_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.47.mlp.down_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e9174a778be888f76b8d09e64f698edb05ed16c4
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.47.mlp.down_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9129177dea569b4f7b130c2e06626bd04bc031eb6cc71a8c676b4a1adf88737f
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.47.mlp.down_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.47.mlp.down_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e673a69a15a9b979c495342e58c993423774b03f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.47.mlp.down_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f0debf6b58616565e29aec6e26618da1ca28276128430ec55ca0f3d502fda9e8
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.47.mlp.down_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.47.mlp.down_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c069487e212328a808c27af1b63b8237ddf90aaa
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.47.mlp.down_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f0217b4ad30448ffbb88358f1873c7a335d8455c6283e9b8cf373ecd9c90544a
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.47.mlp.down_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.47.mlp.down_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.47.mlp.down_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.47.mlp.gate_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.47.mlp.gate_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8b55ba33a0bf079781fc721bf112c9f5a9e85526
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.47.mlp.gate_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2e632fd497b314a27ff83f937d7c0916fead0cf6f7ed04381e47c1e85ae0015a
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.47.mlp.gate_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.47.mlp.gate_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..db5b9f35192407c556e63e0109a2ef0d7bdaa12d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.47.mlp.gate_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:331b2b5056ee77913c425b3be491e07a8f36d41aa2531197e84fa0fe515df4db
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.47.mlp.gate_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.47.mlp.gate_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f2483fa0f6b88c66f1c7161f23824270c6fa5284
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.47.mlp.gate_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5b48ec344f6e7dc96af82bee902b54da05bf36bf22ff6223ca3ca69d6726b94d
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.47.mlp.gate_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.47.mlp.gate_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.47.mlp.gate_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.47.mlp.up_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.47.mlp.up_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..aa951b2446bc1154070105014c532ac40c0ecb58
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.47.mlp.up_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8af9c06509cabd94f58fc9afac0a9c94bbef2524c5e1aadb36f7a8f6f1fb3b13
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.47.mlp.up_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.47.mlp.up_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e1833dc4584d29818c933a959a4fc95fb82e984c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.47.mlp.up_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a82126af1b42d2c7947513d8a35854949ed963b5ea58e9baf3233b45ab405005
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.47.mlp.up_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.47.mlp.up_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5e2188e587786a38986ff3ddbf4f1820a9e29d44
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.47.mlp.up_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9f7043b1351d3c3259030412c836691527586bd71ba1a8eba14cb2e04cc1a583
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.47.mlp.up_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.47.mlp.up_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.47.mlp.up_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.47.post_attention_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.47.post_attention_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0f0cef7787e1c1bf707a23fb015464daa0d39fb9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.47.post_attention_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7e244c4df025eeb4a9b7beecd00b51d2cea950bb97aa1fc0399c3b45bdf3184b
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.47.post_attention_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.47.post_attention_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..efc8aee61543b858c873713e4d5c7eda34aad4e3
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.47.post_attention_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b79f482e10225d2db5b52dc30c2087f4c0c8a1c788167385fc7e4a71dfee0a2a
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.47.post_attention_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.47.post_attention_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4815a7aef0b6804347377aed7b37c499ad571d45
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.47.post_attention_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a132bb8cc2093268afcbe50ddf420c0c686cf4e6a84f25943812c2cd54a3565
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.47.post_attention_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.47.post_attention_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.47.post_attention_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.47.post_attention_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.47.post_attention_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b87f16645f629b13653988476692301cc808dc73
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.47.post_attention_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bdbf4c88c9d2633713dca3363efc97027af0be28a260d82c010359a0274b45e6
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.47.post_attention_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.47.post_attention_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7c658d0d09cf1e128082edc07c145fd3e01c3b6b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.47.post_attention_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ee743978949c0d342ef5e1f9b87de0ed87ee1fe83d4c5cde5f09de1b9a074d10
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.47.post_attention_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.47.post_attention_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e212a427f63df1886cf2bfd428ede42e116aef7f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.47.post_attention_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5b18f2bd3c2c555d1e95afdf59ec7bbca396b501916470868fce07bfcac5680d
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.47.post_attention_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.47.post_attention_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.47.post_attention_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.47.self_attn.k_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.47.self_attn.k_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e4fd20cffdfdef4ac7275c1b34576f6c727cac60
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.47.self_attn.k_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f777069d216471224ef6b0080c7de8acda2b51b5480f6d51c12f663f21cff306
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.47.self_attn.k_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.47.self_attn.k_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4e01b9e56cf15050dd793427c0ef682df3d2a5ff
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.47.self_attn.k_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8dd16d0ee9e6942bbb2cf1603fc2b552370089cfcce4d01cdff20aa5f4107c02
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.47.self_attn.k_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.47.self_attn.k_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8c591b2350220ecaa8471c6b95b296d373dd87dc
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.47.self_attn.k_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c0f70de498c1ba4f94e712b0e5711a29017e2bfeef1bdc8de33d30881e03bc56
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.47.self_attn.k_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.47.self_attn.k_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.47.self_attn.k_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.47.self_attn.k_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.47.self_attn.k_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d941306fbbdee11bc6673ce72ec06a1e4a85c1d5
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.47.self_attn.k_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b942b6e05614e9817f70d9ae03959a4eb5131aef5a839e0d8fdf6263c276b856
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.47.self_attn.k_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.47.self_attn.k_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..69b26d51243567c923c9836a724e317dd52fea41
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.47.self_attn.k_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b0b4e7ed90ec7b7ec2c5e8b69a928e0807abce2953ca0d7ec41a4b24247be23
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.47.self_attn.k_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.47.self_attn.k_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..78ed2dda2549f849863c7fe11abb99a605abfd7f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.47.self_attn.k_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7bdf12208b4486394bae4ad9ddee6a50341dfd78749f90c5441857cfb985a05d
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.47.self_attn.k_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.47.self_attn.k_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.47.self_attn.k_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.47.self_attn.k_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.47.self_attn.k_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0002b3b876123e40d231ddb22e4915b552357e17
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.47.self_attn.k_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:efcc2d8f106f368b577f0a537db38565664b5ced3b17da60d342c5dbb2b4cabf
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.47.self_attn.k_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.47.self_attn.k_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b0e46903ce5f1364276aa06f68cc178d04d1a57a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.47.self_attn.k_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:12052d680623d34549288680f32bb07b1213993b4c19fa6dcfa3e6b049aea390
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.47.self_attn.k_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.47.self_attn.k_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..878e3e73a1722c9ec8cdd45595a2c2d72f798b95
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.47.self_attn.k_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61d0402d914b211b15d18892ef845a8c9827c9a59b3440a1ba641ebefba9214a
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.47.self_attn.k_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.47.self_attn.k_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.47.self_attn.k_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.47.self_attn.o_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.47.self_attn.o_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..174a330fa3bea88dca1bdeeafee63eec4cab13fe
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.47.self_attn.o_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5a86248a969c702ae913d2eea70fd4fcb963ad8f5cabde311247d7832492c908
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.47.self_attn.o_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.47.self_attn.o_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c372816deb8d4aa9e9a1b1319329842254c38cdc
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.47.self_attn.o_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3142ad72cf695cf7637aaad49ade298ca38b8507839d9f1241618e1cbed85fa9
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.47.self_attn.o_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.47.self_attn.o_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f9f6b9e29d5e672cc632c6470ffb57a063b24021
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.47.self_attn.o_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:14ddbc8e8d07913b2c0a63f21e313f751a2185c9742b09764deb17957c4d4a6f
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.47.self_attn.o_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.47.self_attn.o_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.47.self_attn.o_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.47.self_attn.o_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.47.self_attn.o_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..581e05bc1bbbdf0d41d5a55e159a7ce75f818e45
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.47.self_attn.o_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a6615aa2926fd5fcff8eb1d6376f604572a8c22758ba88a6bd51c1a2792add66
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.47.self_attn.o_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.47.self_attn.o_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9fa659680bc8865baeed2da1716aecf50966fc09
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.47.self_attn.o_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dbcb42cc439cb3251986ae76e02613686d2ce2542412ff8c37581e029e51af7f
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.47.self_attn.o_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.47.self_attn.o_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4a5192e9f4039871e75ecafb846ab65c725d133a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.47.self_attn.o_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2b3093ab2f8cb70cbf3cbef1a0788cb3092a1b2ff646b37f780cc90fd20d4b50
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.47.self_attn.o_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.47.self_attn.o_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.47.self_attn.o_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.47.self_attn.q_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.47.self_attn.q_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4e2192bd6e3a9f1345c74a360d00aeaef9a23550
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.47.self_attn.q_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d59c40b1bdcc4aeaf4f151fcef16dd6e1ecc50be8fe0791a0b2531b8ec71b68b
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.47.self_attn.q_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.47.self_attn.q_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e61d2fe1744200058b36c07ad010597eb5fd353b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.47.self_attn.q_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bde77c36b09c9a136ad47b4f58b5ad415418f5b5b42bea00574ff9c14cb086e7
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.47.self_attn.q_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.47.self_attn.q_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..35a75bc2726911dd2c80965220863ee4dd20cbb3
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.47.self_attn.q_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8340d079f167b5bdf881f18e1a0ce330fc88268083f13e27f7016bd7f892febe
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.47.self_attn.q_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.47.self_attn.q_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.47.self_attn.q_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.47.self_attn.q_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.47.self_attn.q_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..737af1382f7db63e0065351e3235c01791945f93
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.47.self_attn.q_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:12a5e8c3c149e79196868032a415971025b3e91cd578b04becb3fede5c167456
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.47.self_attn.q_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.47.self_attn.q_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..37206f89071f554f6794a7b9e538988115be8c0c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.47.self_attn.q_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1408de2352d7e14671353777237c7e8ad2543f3c81e23eeb317875429dfc053f
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.47.self_attn.q_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.47.self_attn.q_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..23de2b84682c17aa797129507fce256de27b08da
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.47.self_attn.q_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d8f96e185b382ea891583a33293eb61152003985943ac5e2ace94ce51f154bb7
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.47.self_attn.q_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.47.self_attn.q_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.47.self_attn.q_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.47.self_attn.q_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.47.self_attn.q_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fcec23ff9177707e6b0dd00aa2adad992c784bf5
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.47.self_attn.q_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1a43f486ff34a671d11bc15ddca57669a54896f86543f70b3b05baab52b53111
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.47.self_attn.q_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.47.self_attn.q_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..61e00d8b2ec2bee861abb433377fd781d1c3e6f7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.47.self_attn.q_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:27cc9e7bccbad97e3bf904f55435918766c1010f335ca8231aadf79a00081259
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.47.self_attn.q_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.47.self_attn.q_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..50e9abce707258fa89f198dabafb95df291ef310
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.47.self_attn.q_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4efc2dc0edf2650b1d7bcba2e349968ecf09bd25ae54bc3935636314ae7e1c2c
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.47.self_attn.q_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.47.self_attn.q_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.47.self_attn.q_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.47.self_attn.v_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.47.self_attn.v_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ddbbb6a587469497ac499bc42294e3823bc0b95b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.47.self_attn.v_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b0f7c76c9a6d3217a99d7a543e94ec319a7f9a15ec214a9cb0b74945cb214e50
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.47.self_attn.v_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.47.self_attn.v_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d21ea87f7894f5b247e823edcde9b44d6cfe0042
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.47.self_attn.v_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:21638d3d782273dd18dcce49ae7e7cf3dfb865850d65fe19d9776479a32f8d18
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.47.self_attn.v_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.47.self_attn.v_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c555aae65c999deb11fb4ac44aaa3d7f8adfa101
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.47.self_attn.v_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4b96d01e85e48ecb9732f039e44ddb08f508b7d107a62d765a295f0be3f3af99
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.47.self_attn.v_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.47.self_attn.v_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.47.self_attn.v_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.47.self_attn.v_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.47.self_attn.v_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0e72583e98d61a5bc86dc7574d85fce62c1f732f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.47.self_attn.v_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:974c24a1d26a3dfa4174fea6bf1046dbd931c8b00a591194cffe73f2029d4f72
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.47.self_attn.v_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.47.self_attn.v_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f7cb23ccf66b6e0690e82b6905b16cffcc6770f5
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.47.self_attn.v_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:908611d22d58eb53c2568612770b706ce7b9d3aff2858368fc11b0c9125dfa32
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.47.self_attn.v_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.47.self_attn.v_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..001deb31883d276dc914c308a608ab55aa760bd8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.47.self_attn.v_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:56fd731b73a2cec07b4df5c4b157a224e432e55bac0d44fbad9db09bf0f28b6f
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.47.self_attn.v_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.47.self_attn.v_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.47.self_attn.v_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.47.self_attn.v_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.47.self_attn.v_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3088b5a78d0c2b45aa8f6d743ce728608a6eb5f2
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.47.self_attn.v_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d289381e19a633060d0fddde68d4e00f56b160d0193469c8aa31004d02f78fad
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.47.self_attn.v_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.47.self_attn.v_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..803cee54b15e27175bc58c591b87f82cf06e9085
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.47.self_attn.v_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2d65543dc76f27d05255771733e787245f71e4bfd6abe9bdca4faf546439bf65
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.47.self_attn.v_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.47.self_attn.v_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..582da67ad309638718fd1a4ea1df5e4d6202baae
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.47.self_attn.v_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a36c9e02d270e7f77e87fa96d1140ca9ea5bfc9d7224462013ba97aade599fd
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.47.self_attn.v_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.47.self_attn.v_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.47.self_attn.v_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.48.down_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.48.down_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..02de999bda6489ad6f8ebd79c567fbc2616df1c1
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.48.down_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:90245a4cf00ce4fd2ab7e8fb689e43f7ca3f08cc89a7949628fff6902580a52a
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.48.down_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.48.down_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..148895d38b7447263a78198e4cc549e4c7d9e651
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.48.down_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:39aacbfce326ee457478bbe5ace5b12af921bf757412c71b20819441ae085ea3
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.48.down_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.48.down_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..10e55b6d5c641e0252f8c5b25e2a457d2cc1311a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.48.down_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:07cd6c4e702a7033e4710eee45d4679c2a6305fef6f0c86678c3ca286360dacb
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.48.down_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.48.down_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.48.down_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.48.gate_up_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.48.gate_up_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e4176b11594c156b2dfd330a11a240f4622df835
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.48.gate_up_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7f9318b066e3d928ed8e9eabaf02ce09c1ffc717aa14fc2099e1f5f1545efb4c
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.48.gate_up_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.48.gate_up_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a6fed9ca6f88fdf6e50ff4b82f9a6830a5055f23
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.48.gate_up_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b1082b9e439c466fa91b99884df7c3324bd0abc29ba7d629b3152d861671ee19
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.48.gate_up_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.48.gate_up_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..eb44286b14056e27ab4e0e621f38ec256ce89d0a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.48.gate_up_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d347646b78785f8cde490517b5a411ed5245bd852838f819e683f39507adf182
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.48.gate_up_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.48.gate_up_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.48.gate_up_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.48.input_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.48.input_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b0b70e4f88055d9922ef78302144057f34c00c9e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.48.input_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3461abbaefabee23f8abac4dee7d1f60a0245da2ccebc0b94f49c33efd53aaba
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.48.input_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.48.input_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..53f7063def19968afccce101c8dd3f9747b4175a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.48.input_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5ae8d000d0afd05397268b150055420b9a83ccad7f78e7a8468b6cfa480c5d1f
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.48.input_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.48.input_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..79e0314947ec359a4bfad0d1ce41c5d38e7a2ad6
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.48.input_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a53bafcba307ade562976aba6aafaa7356959d4eb2dedff6ea99c6fcba7d8401
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.48.input_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.48.input_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.48.input_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.48.input_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.48.input_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9815de77a234afb97ac51fbf9ea56237ee75e4ea
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.48.input_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bfe2f4d8bc2621a3e6fd1dee3ecbf91796fc0ab6564eec8ace381698c7cadd7b
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.48.input_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.48.input_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..97bfee270cf60ea752f187faa4f37e9712ab0c5d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.48.input_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e886e14cd47a25054fcc954b4e2b47613a74b4cab326460e879277a0f886d18e
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.48.input_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.48.input_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..22ad461dec877c3617df7703fae62ca0c6ed4d78
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.48.input_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7e51d5dadea51eb519d23331eed1a68b8f8031a2e8cbd3c700bd0c7b12ed7139
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.48.input_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.48.input_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.48.input_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.48.mlp.down_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.48.mlp.down_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f8fc43eab662ee72bb588ce107b3080b9b1d0f18
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.48.mlp.down_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1f169fc87447a3347cbaca3dcea2342f65c08ec3e7512a3e9c3259e8ea6e792e
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.48.mlp.down_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.48.mlp.down_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a80bcbf40f1ccd5df65e53a6851f57b3d6eb3ffb
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.48.mlp.down_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4d481568ac41688c9b3075f9a3d54b90addf4272a7f09d002d1e19b985c2a8ec
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.48.mlp.down_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.48.mlp.down_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0751dca73202434f034efdea06efd5cc59d60522
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.48.mlp.down_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:41e5da8dc56dc8b4e99d86581df9336270f4430b5ee50577704e490fedbf961d
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.48.mlp.down_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.48.mlp.down_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.48.mlp.down_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.48.mlp.gate_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.48.mlp.gate_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1ae507c7a78257386fd6fae8b21ec386cabd650f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.48.mlp.gate_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fdbe77f17586ba5005b590c6099f91d1f629a0834f611a18cce113f8e8d5097b
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.48.mlp.gate_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.48.mlp.gate_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..def8e114f980eb03c6e28c4d218bd7b84e9cd09a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.48.mlp.gate_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d65fdb2ec6bb5a1c91e787aa5c67c28565faea9014cf614951a6b64fc788b114
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.48.mlp.gate_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.48.mlp.gate_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4808290747aa6332b0466f65698fddf66c6bda9e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.48.mlp.gate_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b2e8bb4fc63a8b46f2cc04fd31e104ff8269ef49fcfc3a91acd1347642129d16
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.48.mlp.gate_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.48.mlp.gate_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.48.mlp.gate_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.48.mlp.up_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.48.mlp.up_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ac043665ded8253e3512110f89de3debb4db4c2e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.48.mlp.up_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9cc952a582ca1e82b596784243ec0874bf003a93de37f18559973ce1738c42c5
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.48.mlp.up_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.48.mlp.up_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..243dc7126d1743d695cff4446858da51fe7c368a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.48.mlp.up_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:881fe8dda577fabe644cef85a6072e3e2399c9dc24f2165275df15770c6a2e73
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.48.mlp.up_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.48.mlp.up_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fb64fd06d548edc47a996337020a7811ff827451
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.48.mlp.up_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6fb3fd032b4e2c371b798f839fd62ca2b69ee87f345fb3a328a11a3710b60310
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.48.mlp.up_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.48.mlp.up_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.48.mlp.up_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.48.post_attention_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.48.post_attention_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..acb5b3dfd23cda280c13582e4dad4a5adfa1681e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.48.post_attention_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:067a14496b238b92645d89a15923a99c273d25f0999345a268d30b61825fe33c
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.48.post_attention_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.48.post_attention_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9b605a810d9143b205e5228200167d5bc170acd7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.48.post_attention_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:af6b9814c5757a6a9247be74c38eacee201efd4dbf2c7ff68853484d0ef027ea
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.48.post_attention_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.48.post_attention_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f089fb4df06858bbec59a27e848487fe5be8f960
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.48.post_attention_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f12111e23799fd31eac1dc5d9a837199f386f44b3615d1cac14a41b595179137
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.48.post_attention_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.48.post_attention_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.48.post_attention_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.48.post_attention_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.48.post_attention_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..84515d35ec6813ec19cfec7b405397fe060aba96
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.48.post_attention_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:40a223c6cd7813bbd5f0e1246374136c174c838b60b02d48dbc75064e8bcbcfd
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.48.post_attention_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.48.post_attention_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c4e86254a629ad7f5b1bfd787ba3af589a7628d1
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.48.post_attention_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e27c5c1f4e0c2f1d6893f2388d246a1a018f99eeaed2d77e19ab5b0485df53e3
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.48.post_attention_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.48.post_attention_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..bb69b14311e847b316a631e27d6b0ff2a3181516
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.48.post_attention_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3a6e25dd025df1f7ab5f077c501fcf699fb02c0d3914b46449e7119e38fba9fc
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.48.post_attention_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.48.post_attention_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.48.post_attention_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.48.self_attn.k_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.48.self_attn.k_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..61e3b9103e3e939b89fa5f9c146192fa62fa4cf9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.48.self_attn.k_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cb9759984ed0587012fd0c2856d3535a30a07fcbf1e353d66e1da1c668252b7a
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.48.self_attn.k_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.48.self_attn.k_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9f17e02a748b8304ceca0a15bb7222360c4fdc0e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.48.self_attn.k_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3e51d0d8accff2b8ecf492a284f22cac01a1bfa48530aa39967716b6c1303029
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.48.self_attn.k_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.48.self_attn.k_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9347393fbb97e1b8e8ff658f82b6e4bed3500ef7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.48.self_attn.k_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c392796fd870713ec5a8f3f929626f4a6347baddf2fadde9585b67a04c748d86
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.48.self_attn.k_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.48.self_attn.k_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.48.self_attn.k_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.48.self_attn.k_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.48.self_attn.k_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..aa334e93f09c0c122759bc71b229a9ef48201bc7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.48.self_attn.k_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d052d1fdfea82c8338c97cc10d2132d0f7dbe8c23ea4fdcda11e8b5ed0538e8a
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.48.self_attn.k_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.48.self_attn.k_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..254623f12534fe90245176ca0eecd10b972fabed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.48.self_attn.k_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8656fc3795de784a31ebc1958ea5721881384997531a7bda799f6afd5ee5eb94
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.48.self_attn.k_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.48.self_attn.k_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e6438bc870d99b40c5f803365b1495ee1be42878
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.48.self_attn.k_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aa8ec7edcd4db449aabf6ee770a69c8bf33f510b100a19bba6eb22d1ee139316
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.48.self_attn.k_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.48.self_attn.k_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.48.self_attn.k_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.48.self_attn.k_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.48.self_attn.k_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1cb5e8d1c77fc07a5337f152a447d41c69d76b12
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.48.self_attn.k_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c186c6c127de7e97753f56bac3194e8f86096242a224b74ed1adc11f99f07be2
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.48.self_attn.k_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.48.self_attn.k_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8906156a9f99d984a826d9c7262796b18016886c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.48.self_attn.k_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a53d493c1cf057657d02fea4347a606a2ab2ecea096b688dcbcf1ed16fba1ee7
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.48.self_attn.k_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.48.self_attn.k_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a56ffa23e761e3b8c0a4508738c0d7837891de6b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.48.self_attn.k_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4d3a1e99d52fecb31de014f10b434444dd044073b1311f8166e6a311ef6808b6
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.48.self_attn.k_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.48.self_attn.k_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.48.self_attn.k_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.48.self_attn.o_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.48.self_attn.o_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..877e7164c2ad85d9aba7e2bfcd935df61a432dd0
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.48.self_attn.o_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6667180b09d37f9eb0e94fe9c0a18a0cea67b7814184c496e0d97c6bfda06d60
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.48.self_attn.o_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.48.self_attn.o_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..90a367a812222f48520aee0b1dca632373da40f2
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.48.self_attn.o_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d0213af264af47170bbb37f8d05ddee07f349d77659d79764e1bab696ae1ff24
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.48.self_attn.o_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.48.self_attn.o_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..847d8c1698e4bd31e5fadf71317a440bc51e1c22
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.48.self_attn.o_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6f078666d23f00a5cdc608fd4104f26411abc3b8f4ca3c6804ddcb20f8296150
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.48.self_attn.o_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.48.self_attn.o_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.48.self_attn.o_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.48.self_attn.o_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.48.self_attn.o_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..02ef9592dca766ded32e39ca1de8a64831b17885
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.48.self_attn.o_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1a60c7b695e4455c982435f751e56992f05fc5df04701d7d220a08f5021244ed
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.48.self_attn.o_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.48.self_attn.o_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..21704b2608ea3868488cc0b8f196f1b5d2df1830
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.48.self_attn.o_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f10b9abe3638d29ca0e4f7a462cd35ae6dbe27d842a1f058f343f314b3cd9127
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.48.self_attn.o_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.48.self_attn.o_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..94d361decacb8bb3946d212b64b17ea00edc5115
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.48.self_attn.o_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:800de1b7dcc0220dfb27f3013137281b6f0ba0e896345479d9f210671488d945
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.48.self_attn.o_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.48.self_attn.o_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.48.self_attn.o_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.48.self_attn.q_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.48.self_attn.q_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ebbf3d94e5af9f4f6a010a094de0a4471e708bf1
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.48.self_attn.q_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:103a5005badf05634cc6ad0609e27da847e04e29bd926ad8fc4c18e09e4ee8b4
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.48.self_attn.q_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.48.self_attn.q_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cfb6ef55dd2e3a4da5fa7c3c4e536a31214e94aa
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.48.self_attn.q_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:606483c18b820fe9ea49820bb0a936a2723791a8a873d8889c9df82f9c56d71a
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.48.self_attn.q_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.48.self_attn.q_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..61edfdc3522b7d9077307e5173ce377bb505a7e0
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.48.self_attn.q_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0d081a1d1361b4276282eabeed45fc6abd726069709d04604d22b8ffa4b8a0c3
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.48.self_attn.q_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.48.self_attn.q_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.48.self_attn.q_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.48.self_attn.q_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.48.self_attn.q_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8054c0a46c71cdd160b8e5a55f7d338ee9776570
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.48.self_attn.q_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a709195e43d7057fbd44f73a225ee12ea8c121970537c5e90c553682dc856fdc
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.48.self_attn.q_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.48.self_attn.q_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c7778733435f36a1d78895b808f9ec59df32ab3e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.48.self_attn.q_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:19864e14c80a9c4c08e019c3844509575b1cf1841d4405f1b847a44458bd777b
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.48.self_attn.q_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.48.self_attn.q_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..76fa2da95555b5c264946bc62947126b4d28193d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.48.self_attn.q_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fabc8c2c994baeef4ea5c6c93b8f0ca5f2f91bc8e5d12d553933168e274b6e62
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.48.self_attn.q_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.48.self_attn.q_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.48.self_attn.q_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.48.self_attn.q_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.48.self_attn.q_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d05cd76d40db99796b2cc3f237c75b9ca3e23b3b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.48.self_attn.q_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:39eb991b2d49ec907a0ae5b1c890b2e9bd049facaca471e3e9dfb08fbc4afd74
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.48.self_attn.q_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.48.self_attn.q_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7bf7549b17a2a7e1b4ed2eb8b787ab8ac33d08a0
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.48.self_attn.q_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b13fe4b9df19d74f2febc58407fcc7dbaf562e2894d832582c6781a1e829836
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.48.self_attn.q_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.48.self_attn.q_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4cab5f8e9fddfca7b8d4e1c97b62403254495232
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.48.self_attn.q_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b3f8f9fc0e629315626c7e79b6238c3d2ec4e6133e70dc5a5188d6f7a0b93163
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.48.self_attn.q_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.48.self_attn.q_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.48.self_attn.q_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.48.self_attn.v_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.48.self_attn.v_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..75a74e3b7a8e10c1e4f925e6ea14554d770c1bc3
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.48.self_attn.v_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e55e4a098f110e6d3bb41cbccbf73ede434105fba53012f0f87c7816e430b274
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.48.self_attn.v_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.48.self_attn.v_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..92d11388c41ff8b0403479f32fc5396365b02721
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.48.self_attn.v_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:caa18a20781ef1dbba9afc5b3861b935aa12b5a41e0ee58048df77edea39a950
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.48.self_attn.v_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.48.self_attn.v_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e80b554d3de0bf6cd06bdcc7c4e8482cde710ead
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.48.self_attn.v_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d15097ad80db34da30e002a1bd1e302818a678df2b241a0820752b81b946bd5
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.48.self_attn.v_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.48.self_attn.v_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.48.self_attn.v_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.48.self_attn.v_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.48.self_attn.v_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..56f8d216dfc2fae30ccb419fab4b020a4b0f114b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.48.self_attn.v_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e45c41d06adacf71ce8f60f1c7417b431a48f95b197f576b41c6ec9ac6c3e2ca
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.48.self_attn.v_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.48.self_attn.v_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..31606b80f3455e92313410bcf671c4da91911d0c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.48.self_attn.v_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0c91df6e52b55758461b589e779d75dfd560105f2ab9921a8993b828cd5a1ce9
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.48.self_attn.v_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.48.self_attn.v_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b9e03973471bb5c163dbd6df0c135a7a5cbd2d2f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.48.self_attn.v_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a8fa549a10ff7fd1b4a4728a545a1d51f765f174c0f77c3acbdb1c929abcb92c
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.48.self_attn.v_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.48.self_attn.v_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.48.self_attn.v_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.48.self_attn.v_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.48.self_attn.v_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..099fcda30614ce63ceeb84a9e069ec5258942d3b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.48.self_attn.v_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ecd93a71492e8d589fdd03bc023f013e3c0e3f5ea21bbaff9b18159667b22e36
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.48.self_attn.v_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.48.self_attn.v_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3beb9153ab6a71eb46ae5f35ad2a77c6efee71a2
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.48.self_attn.v_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:16219ad60efb128577ec49cc36c5bcd3ed031b755c7aa0c23451388c4863f179
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.48.self_attn.v_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.48.self_attn.v_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..60fbbd85eb231c5e3b3f844d63fed50fe0fa6b85
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.48.self_attn.v_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9897dfd8fea746cb8b2dc3082225fb5925475b84504ab8ba7d77703599e469f7
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.48.self_attn.v_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.48.self_attn.v_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.48.self_attn.v_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.49.down_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.49.down_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b5efb1c409e7dc57d50085a13c19e49643a486f9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.49.down_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:478eea5a94ec024af4071bb0db5e5ff88c8a24032796aef0489a8fe83d40fd42
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.49.down_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.49.down_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e40b31ce99d9acbca7a356f53822743ce3752729
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.49.down_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ad6ff45ae81b4cad3ccbbec4ff86c72ef1e4892be4f37cc2db196677738d1f99
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.49.down_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.49.down_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0de061166c50af86b56c22a100d9a4168215778a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.49.down_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9657614ed624bcd52418a10e6afde2fdc06673dd8385a341ece15e9c1d55164d
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.49.down_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.49.down_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.49.down_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.49.gate_up_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.49.gate_up_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8683fdaa6519ccd4accc2e6a48b69353ee343cca
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.49.gate_up_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5c51ce113b05baaef8671939c2ddcbc058bc3ca7b0da2b9c13084413bcfd26ce
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.49.gate_up_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.49.gate_up_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7afc67e03dcfedb7f631007b05fe82e5ac7b1d1b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.49.gate_up_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a69391d02c11f2b131551bb4e07003ed0650d811f6a7c8f67fd03a58128d29a3
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.49.gate_up_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.49.gate_up_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4300befd9634b7ae2202fb7624b9def4169a17aa
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.49.gate_up_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f2624422df8bfa584d06b7513247d982dd830e286b869bc9a8bfd148ffeb5354
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.49.gate_up_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.49.gate_up_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.49.gate_up_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.49.input_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.49.input_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b9b88172bd8ab2d906e8e10823bac1a0a5fa6982
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.49.input_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eb85ef9fdb0e3f76ef63e5314275e12484eda2b05c4b96fababb40f15f9b6269
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.49.input_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.49.input_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9353a4055cd1a3d1c63a07e8432eaf2275108b57
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.49.input_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:22ac5707a3e8302af83cb0a1ed4388c150ac58a97623490111c92ab1729da4d4
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.49.input_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.49.input_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a5e158cbe7c0ce41e194f48fe2fd0aad18faf0af
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.49.input_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:73ac7b25547e9c7fbd7b5fa5b64ebf1968a437282b710cffe727264e076dd46f
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.49.input_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.49.input_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.49.input_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.49.input_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.49.input_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3b1d2e2353547d08d3a3e63fce33cb9a71319612
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.49.input_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3a9a4d33b5cd0f5f7a144e2c299445a780ded0cf93c31e3b75dfda91e4598e63
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.49.input_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.49.input_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d5f08bf558531c6fa84512237e27f0f6bdc110ab
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.49.input_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a1d8f6ea72ca8785f1c005d642ca782b94f3102ab0a7c8d3b87a2c2365592619
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.49.input_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.49.input_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..60fcfd2bd75d0c90013d35aa7e6d1e0378543910
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.49.input_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe20fb158953f4846a50ddc9dfd9c5b1163a99f2339e293b82c1f523b4278d33
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.49.input_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.49.input_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.49.input_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.49.mlp.down_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.49.mlp.down_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b17843a0069919b745efdc80e7d7f0a32cc23b82
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.49.mlp.down_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f7e7d1f6b45281d5061c931cc81022b52799549c9c893248b23573a896e98467
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.49.mlp.down_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.49.mlp.down_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d7233b7d4d1bb72f476d3331ca4115fca3e44b41
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.49.mlp.down_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5be3922472e9cac13e19e3d931fbcd2a192531a81457f456e5b501120c7e3928
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.49.mlp.down_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.49.mlp.down_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3a0296ee95e276cb6b4388423019a922efe93ded
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.49.mlp.down_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ac671bd0d42a929fe934e11223b2eb4f94f38ec85deccc47f981e4de5dbabe5d
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.49.mlp.down_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.49.mlp.down_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.49.mlp.down_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.49.mlp.gate_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.49.mlp.gate_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4a25818410f4600a1785214ff4d1d63287fe9bac
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.49.mlp.gate_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:deab7111b07dcbc3c12acd0e6ede8db65ca951a6309cb538e95185013851fa0b
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.49.mlp.gate_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.49.mlp.gate_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ec3a1db77c796ee660caac26cb7852b788caaaf8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.49.mlp.gate_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fba91382db1b2d4c0942594b56b5b83baf7a8b559494e3e554d082a7576e8701
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.49.mlp.gate_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.49.mlp.gate_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..37a04b5f6b25c00b5d377f4f4e114ad75641009e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.49.mlp.gate_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:51802dfded03e2ba3e35b0d0163ffb555279e3280db9d8e4e281a6aad4eb59f6
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.49.mlp.gate_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.49.mlp.gate_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.49.mlp.gate_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.49.mlp.up_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.49.mlp.up_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7ca6e05bbe9400cddd7ca89b912ceeae7dd08143
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.49.mlp.up_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d9f0defd05995f7baf00234d2b2b1331409dcb5c4976e8654dc7eff572080fdf
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.49.mlp.up_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.49.mlp.up_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..96251a63c36124431677aee5d9c91f883e3fdbca
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.49.mlp.up_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a5888d7751e9bd270e6d5b1930bc4c29aaf7a5e0ce3cd5cc4be7e481937a8d6e
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.49.mlp.up_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.49.mlp.up_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6984110c4181fa55557f83a3dd9c9e6ae7651e40
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.49.mlp.up_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:44feca9fc7562f57decccbcb74b9a4865f8eacd33ac3721c90d15032fe8e9f7e
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.49.mlp.up_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.49.mlp.up_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.49.mlp.up_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.49.post_attention_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.49.post_attention_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8903f55475e688f8a0fcbddab8c659e9c1416ebf
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.49.post_attention_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:16e2a61d440fde5d27b2ee5fec24ee287ed708a96926ca8d6c584eee62fc3516
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.49.post_attention_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.49.post_attention_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5ea0d109f4cc8ae9f984ca170b022a6db6383629
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.49.post_attention_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bb796da917066468e5b0740c0f3e6c2c54046c871c690ba2b736d6db8ce04c00
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.49.post_attention_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.49.post_attention_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ecc23fd7601d8aa95c32caed7ccf71240fc441d7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.49.post_attention_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1c7d6e46950f48e14f4c3c50a58fb5874a7c6b4cc97378905cb16efb9228e0c7
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.49.post_attention_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.49.post_attention_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.49.post_attention_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.49.post_attention_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.49.post_attention_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2f839abe2293cb46e6a7f49dca45b2af6fffb1ab
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.49.post_attention_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a9d1f29340b1dcf406305c1fe9f8211bbf0099adbc759c73013cbc936e63ebd1
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.49.post_attention_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.49.post_attention_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1d0ff2ffa9a2127716015e6cd8073290abd6967e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.49.post_attention_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:377942643ed5f2aaca530881ae0bd0a4be6234f19aa1d91d2842cf5a480a78a0
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.49.post_attention_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.49.post_attention_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3040dd91330a3da2e73afe06f1f558efdbab32a6
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.49.post_attention_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f3fd77805d1928667294500437a7fd82c096827745b18af930a2f8dd7a64ec0e
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.49.post_attention_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.49.post_attention_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.49.post_attention_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.49.self_attn.k_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.49.self_attn.k_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4cfbaac02fbb72f0fc2e2d407bc2231e09f2ed0a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.49.self_attn.k_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e97c42ff4c713785ae986fc79f1dda065921ec60de957176178761feefea87a4
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.49.self_attn.k_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.49.self_attn.k_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9ee45a118d9f85b511156aa8d48a1a2bcb552a51
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.49.self_attn.k_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3820f7f994d0a33573657f9b5527b993906ac9ebd2ef5e5d1fda8d99a850649f
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.49.self_attn.k_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.49.self_attn.k_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7b7fe04bee7e8e4521e4fa885efccbaf9f1608ab
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.49.self_attn.k_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4ac04f1e95edf375308dbc54d107dbd05f2bdb24e4882de257429d3edd4d4164
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.49.self_attn.k_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.49.self_attn.k_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.49.self_attn.k_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.49.self_attn.k_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.49.self_attn.k_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3ff15f782b78ccc165c9e46c0b4d02d838c20bcf
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.49.self_attn.k_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7b868b2caa7c6ddfae1da178cf168015d1151dbf3bfd54c2c2dddfe1d1e7c96d
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.49.self_attn.k_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.49.self_attn.k_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7430e385842abef9e4f8da0452cf1c46bbabf8f3
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.49.self_attn.k_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6aaf714e3c0532f2bbc0104afe34d9037dea70aa16134a17915295ed4fa76b9a
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.49.self_attn.k_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.49.self_attn.k_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a5e9ae832342a32fd102b312447a32d74dc5d59e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.49.self_attn.k_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5fe7827d9442feed38cabb891bb2b8408efc5286b7699dc04d147c2dc07f5f18
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.49.self_attn.k_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.49.self_attn.k_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.49.self_attn.k_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.49.self_attn.k_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.49.self_attn.k_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cfb11a5b57b1ed278802e7d76c489b2a7b6fae3e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.49.self_attn.k_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d94d32036b8aee0cf8867205037016778aa4e045f15daac5ad07c8ae50864487
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.49.self_attn.k_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.49.self_attn.k_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1471b097a36802f71f99f2e3bc3c05bc964c4a56
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.49.self_attn.k_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:860e70dd578a2c40d523bc01c144ba623b3a3725dfdb90240e45781f498e0947
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.49.self_attn.k_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.49.self_attn.k_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65f438830268df257e7eb0f96395daf13e83901b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.49.self_attn.k_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3822bb7987db7fdea3f9d2c89f2d35ff4249850f92cdad557d6a53c5eabcd747
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.49.self_attn.k_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.49.self_attn.k_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.49.self_attn.k_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.49.self_attn.o_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.49.self_attn.o_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ec8932d53e94186a75dc2f93de64c6d4fbb4c149
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.49.self_attn.o_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e0811072a5c16caa0cebf0c5391d2d11b3c375a2d6034049a3adb3e4de0ac8d6
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.49.self_attn.o_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.49.self_attn.o_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..905d48f2a0a69fccdadb6cd829e74dca8a24d412
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.49.self_attn.o_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:523b91cbc06a35836cb8007d155cb3c1da2cc23de8ac9ba908341a7028a76ab5
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.49.self_attn.o_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.49.self_attn.o_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d4ba95016c3e6e334a4df6f138a25ac8d30d6cbb
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.49.self_attn.o_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2a7bed064db0ce2dc5161720b85fc1bcee4c22a3e53c81cd5e8a28f09ba7a800
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.49.self_attn.o_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.49.self_attn.o_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.49.self_attn.o_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.49.self_attn.o_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.49.self_attn.o_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1d2808bdb7f70d4220b55aeff579813f25544763
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.49.self_attn.o_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:47d2c2a8742c20420a8e0c588f0db742dcea98ee396ee1920e8c53c0c7a5ae2e
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.49.self_attn.o_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.49.self_attn.o_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cc5dfe1d6c4387b6f8238effd5d4320cf83adc83
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.49.self_attn.o_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:efe6d771c19d774fbae2587ad263a78f9cbae49e184ab08ec91af203bc0884b0
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.49.self_attn.o_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.49.self_attn.o_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cac852d83d2e1d498992ea5b5a2abcbb76f2997b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.49.self_attn.o_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e003eda440474670e059967b87a9ddb4513208b17a76a966c920b6ee993d5085
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.49.self_attn.o_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.49.self_attn.o_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.49.self_attn.o_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.49.self_attn.q_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.49.self_attn.q_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0e0adb1b00316f9f5cdd96dec597db5be53139c0
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.49.self_attn.q_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8e2fb67773069d557361626e663924b9deb1e84f1f85a8371a523db16d897e63
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.49.self_attn.q_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.49.self_attn.q_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4760dcf2ed754c29a07c4ceda3e5de957f911181
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.49.self_attn.q_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ceb567ca56c5a2ca70349cbbd6443dc97e6a9438de795293bbe5f61bb7db4730
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.49.self_attn.q_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.49.self_attn.q_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..aafc1c0f227d8001e1cd8aabb1cfc13097c679f9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.49.self_attn.q_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:43f6ace28a9b72eae820fa4cd9663b6615639723d411cf4bd7fbf9dd978c5bed
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.49.self_attn.q_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.49.self_attn.q_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.49.self_attn.q_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.49.self_attn.q_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.49.self_attn.q_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b747adbadaa1b0aabc5da96771fc3c8a59c30308
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.49.self_attn.q_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:42d9336a2e4fbf71e153447413810500d12386dbbbda209e941cdf441f8437b7
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.49.self_attn.q_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.49.self_attn.q_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9d23e103cbfc12ffbcff21b999d4bd982ea3cefa
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.49.self_attn.q_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5336e397f14dc2ac5293b1b5afd2dce63562306d6585e56c215a708569a235fe
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.49.self_attn.q_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.49.self_attn.q_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2631603a05d23af0c16880d0a31565ac417143c5
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.49.self_attn.q_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:add1000c631a9d8a6e1ad92610a92a11c83e77f24f167dfb44437affdee826f1
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.49.self_attn.q_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.49.self_attn.q_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.49.self_attn.q_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.49.self_attn.q_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.49.self_attn.q_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..48cad41bdbb920abe2ddb458e76453f583e6a0c6
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.49.self_attn.q_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:db00caa5ef0485c2fcb16c07a6337d1b422accf3dc59c56740bfb545ceed9ae0
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.49.self_attn.q_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.49.self_attn.q_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f75c22c2e96b75a036ebce6ef4ac1ec76b6972aa
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.49.self_attn.q_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c31752e1e86dd7e8c740279372898f0e34385dc627b44104b2cee764c482dd90
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.49.self_attn.q_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.49.self_attn.q_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d245f406a9480fce7f21bc8500ef83e2c5aa1eeb
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.49.self_attn.q_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4bfbd135c2c117071ba810fba6e763874d7efa9232c156d34a796fbaeddddaa6
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.49.self_attn.q_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.49.self_attn.q_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.49.self_attn.q_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.49.self_attn.v_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.49.self_attn.v_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d51be277449ef255204ca784ead0e35107deb4af
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.49.self_attn.v_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f94ca050008026ea0c3fa575ae9d1e8f5f2118f5984eb7152587201d52f73f9d
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.49.self_attn.v_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.49.self_attn.v_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9e3cc89f6b574f8edfc6942b5fc3b5359ceef964
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.49.self_attn.v_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:22c191f82825eb5f72a7a92e5100ceedddd86915e5b4d4730987a4bfd24d0170
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.49.self_attn.v_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.49.self_attn.v_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..837582e19b33821fc4ce2765de40ebc5d784ad76
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.49.self_attn.v_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5c0fb65196edef48c675cc4c016a2aec98841f330373b46722416264758a39e6
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.49.self_attn.v_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.49.self_attn.v_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.49.self_attn.v_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.49.self_attn.v_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.49.self_attn.v_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4dd9727ed6c9eede5ef7eb2b46335392607ea18b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.49.self_attn.v_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a7647da2506e822fbe898c82c65c74930aa2460421dd93e08417a2a71409bfe5
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.49.self_attn.v_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.49.self_attn.v_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..60744576e8302d36fc72beabd833837d278a11af
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.49.self_attn.v_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:98df4378914a9bfd181c50f31764b8deb10b5043c971aaf03cefd919ffa0fc4b
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.49.self_attn.v_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.49.self_attn.v_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..613ee42d0a655241a212e15d116667a3cc412b6f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.49.self_attn.v_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0ef7a3ac2083aa6d6bbcdcc8cd05010a72a8eb0297e56a351009d8d18796d138
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.49.self_attn.v_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.49.self_attn.v_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.49.self_attn.v_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.49.self_attn.v_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.49.self_attn.v_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8f9dbe316491e69ff3bcd93ad125877d84d65a0e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.49.self_attn.v_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:18393c25ad78211b6515a1ae0fca876990d69879f7ab6bc9d050a2e2fad9ed1e
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.49.self_attn.v_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.49.self_attn.v_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..409320eb2539ad40f314766cab33f416e4810ee4
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.49.self_attn.v_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e8e4ef51545a5ebaf0ba1a657f729ef91ec8aed24d82a390fbfebeee75ba551f
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.49.self_attn.v_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.49.self_attn.v_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4a20d6ab767268baaf3e34aeaaf0d50941edb6c3
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.49.self_attn.v_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3bc4a3efbde72ae38dadc29bcb0b3f55e69eee7c13e1c5a8b1c4b9cda992df2c
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.49.self_attn.v_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.49.self_attn.v_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.49.self_attn.v_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.5.down_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.5.down_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..300841623fca86f4e49e7d31f20bbbe22a107b94
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.5.down_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dc176f8b922fa912be8cbe6258cdcbfde46f9ee98925ddfec5553152582e1d42
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.5.down_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.5.down_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f82e20e46f9c15a357760fcccf073006867776ff
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.5.down_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:10890f4e85c6d75ca9a44e5cec60042ff6a5a38dc07441b4c33914c77de38b89
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.5.down_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.5.down_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..61323f5bb8fb28e647f0297a3bad3092cc3b24ec
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.5.down_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5278e6243e7a37a5d9d30d4c102152a3f64eb1477e430e2b6cdcca3f1bf39fef
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.5.down_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.5.down_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.5.down_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.5.gate_up_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.5.gate_up_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c18e32ee61367f77789ce8681d770cb342943c2b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.5.gate_up_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5c8b65950cd56fbdcbb1961f102f5157380e05a20213ac02e79bdc23982088c7
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.5.gate_up_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.5.gate_up_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7d40779c15ea5cda46a721d92c5b3e12183d9616
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.5.gate_up_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:21bb3c98780cd62606e6407fc806a144be1daa96ca42d9f73c1b9577a51529fa
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.5.gate_up_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.5.gate_up_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fb4ff82ed5638037177a8967e2a07c388c97e328
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.5.gate_up_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fa83b1ab24106165076f54eb6ac9e71544e10c8bbcefee4980027c1971765c98
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.5.gate_up_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.5.gate_up_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.5.gate_up_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.5.input_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.5.input_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..12e567df5a6ffa9e8a05c1a40952020bf55e2b33
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.5.input_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1573d2b0f985e103284a9e6663d9edf163f05c3a43545f25c2c6a41009e2054b
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.5.input_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.5.input_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2be897f645517deb7ef5f45bf1cddfd86ba763b3
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.5.input_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b032b8d5172d5afb7b92cf8ab076e9298773586c118e3e86a2ce7cf83008e442
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.5.input_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.5.input_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..35aba2a4e2907a1d3955d935dbbef7e714fe90a2
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.5.input_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b03163fea017bba36da777169a25e6272118fe51f7cd6618ee489e249d6de9fe
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.5.input_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.5.input_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.5.input_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.5.input_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.5.input_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..23309625152781ae72f470668b40f7e64be162b1
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.5.input_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:463ef3db6a6384ca674a92c072b95350611467389bbd36a55c3aa7696f775fbc
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.5.input_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.5.input_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a4526c407a3e5fbd7e1e8c7dc68f3bdca27594e3
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.5.input_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:894f7a9c8a4a66f2a7d26217c6f6a6a51e4887500a4fcf1190a0721eec990755
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.5.input_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.5.input_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b2c195433d29a99f27bafe257fb7a4ad5e1f2152
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.5.input_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a1ff2fc3dfb28c58f56f08ad1ebb86f689788812d58370bd47da7e85c375763a
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.5.input_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.5.input_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.5.input_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.5.mlp.down_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.5.mlp.down_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a5dd4f4f763f7b8ee0261fc26e0def3da41f72e7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.5.mlp.down_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:38f94c987a40d58029051b8979504575774f4b8ac71e0032d4a30179f616a8b6
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.5.mlp.down_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.5.mlp.down_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ca16de999ef94bad8103eadef068c966ef508b84
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.5.mlp.down_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dc65ca3a8c28c451fb8c85dababba7c9ace2267761f6b61c69bd2055a616b013
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.5.mlp.down_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.5.mlp.down_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f7a05cd271f186c427a4ac127b30f7fb97662229
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.5.mlp.down_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:93de437d919b54c4b54b80a270a69a127bc9d7b5bd8cb96b2211a1e4bf0d1f36
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.5.mlp.down_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.5.mlp.down_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.5.mlp.down_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.5.mlp.gate_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.5.mlp.gate_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..004dcd7c4a517d627459fca8ddb274a969abcefb
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.5.mlp.gate_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:915a4ff24831ae2231640627bd9bbb551cf966064520f014ada1e576c4e9e293
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.5.mlp.gate_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.5.mlp.gate_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e5448bd5fea3317e2d4b11d3581c30e38587ab36
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.5.mlp.gate_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:006b9c819583494c8f9f732468c3e9cb68b0ae7160ea89faa7b446a07966e9b5
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.5.mlp.gate_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.5.mlp.gate_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..186f9ca6752263ba9e4b81a792d8d6a5b52a962e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.5.mlp.gate_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f3fe0cc0e5f04fd1564588a86c5fc46b1ebe100eafc2d884c04b88cb2c4e9fa6
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.5.mlp.gate_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.5.mlp.gate_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.5.mlp.gate_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.5.mlp.up_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.5.mlp.up_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2167477766131fd3fc75f98256ca6a5b46811b6d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.5.mlp.up_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6224aa3983c49b3d590f28f1330b9e54d9a474512afbd1f4d28af811581ad920
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.5.mlp.up_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.5.mlp.up_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f436d7ffa8a2a6b3207ddf1104fbff478e9f25c0
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.5.mlp.up_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ce6d0e47bb6587deec4bfbe52e52f73f3e2c7ba2bd157c7f7eb6d0262ebc8133
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.5.mlp.up_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.5.mlp.up_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e33047506c3e274fb9175be07fa01f5f1ded3b3e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.5.mlp.up_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4ff666479145610032eb205c81c809eb7ebe71a27d1d8c8f4a2d9bfd2f96ed60
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.5.mlp.up_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.5.mlp.up_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.5.mlp.up_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.5.post_attention_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.5.post_attention_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3220171248eb61616aed75c0ed847ee958d4026b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.5.post_attention_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6ce33b5bb6e075f654424ce1cfe5020ddd61e6b1fb8da01f011b778c057bc918
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.5.post_attention_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.5.post_attention_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fb42e4d6e42d3ccef4f2c75732b2fcf1d5fc98ff
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.5.post_attention_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:454266c8345b68eda79ef9ed273e208d62104bb8fdcb3f5b0437bdc48e595cb7
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.5.post_attention_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.5.post_attention_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fefb9bb195ed6e97888317cb97c9cbb1a02ea7b8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.5.post_attention_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1b1cff04580838fd4cd7f2edf7ec8be459ff441c363d1d37a125e1d4a69dcd61
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.5.post_attention_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.5.post_attention_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.5.post_attention_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.5.post_attention_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.5.post_attention_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f961be8b1d18cbc7a907ed8c0c38c46e4d33a0bc
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.5.post_attention_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:178e717a0074204bcf8dd1e1ee102228d3363da243d433b91a58a4e75d52fd2b
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.5.post_attention_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.5.post_attention_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cbc8adf8a55924fbcbbdcf74f55256a8cfac9fa3
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.5.post_attention_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f62ca5541cb3a5cf7aee0027350582d0c11749674e1aef14efb5ca98b39e330f
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.5.post_attention_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.5.post_attention_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c364e6edfcce953fb63cb1b171675970f593e84a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.5.post_attention_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:09df9e8422cfa11012c768c22fb022c152811c8213ab774fdfe0ddd3c200e47a
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.5.post_attention_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.5.post_attention_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.5.post_attention_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.5.self_attn.k_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.5.self_attn.k_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fb08f4f0e75e1c5c6af22dfc8318d2d3239887d1
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.5.self_attn.k_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:362f015771f330ac0d1c1cae5a35061dda76a0fb607cc2d9d849c8186628486b
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.5.self_attn.k_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.5.self_attn.k_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cb34069500d6aebaead31bbd00d7048359628b81
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.5.self_attn.k_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4bc05655e996d4f2b4ef24e9c9cde780633ea4be714e0fdc25070885c1627682
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.5.self_attn.k_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.5.self_attn.k_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5aa6dba4490344b0247140a0f255b4fac0a62791
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.5.self_attn.k_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1b98faa7b6af78d3889d8b88435bc6ce549be0fba5b679319db7e7978b92f3c4
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.5.self_attn.k_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.5.self_attn.k_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.5.self_attn.k_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.5.self_attn.k_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.5.self_attn.k_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..961f134c3df4598b20df24fada8dfb88115e5bbb
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.5.self_attn.k_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9875c5e7fa756a702f0db8754b049ec71925137d331c59ed0b380c0c09557cec
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.5.self_attn.k_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.5.self_attn.k_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..04d6e9cf98b4b90c8accebb54451b6375b425b0f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.5.self_attn.k_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:90dc401d8bb62ec8188a8933f639a7c609d3b7e45d2457b634e9d9ad1354b991
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.5.self_attn.k_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.5.self_attn.k_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1de2c28bfa5a9872098c9663d6202ba33e36095f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.5.self_attn.k_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:168292a1e692603eaa6ac4a0ff11fe47aaabac2b9e7d5f761edeeb5a905acbde
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.5.self_attn.k_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.5.self_attn.k_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.5.self_attn.k_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.5.self_attn.k_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.5.self_attn.k_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..02a65c0bc957eef0d58c0b888e71abd459a37686
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.5.self_attn.k_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4c3d7f970779b8a885ec6ef37374707a33b28da17a3854adc762f48750f91e33
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.5.self_attn.k_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.5.self_attn.k_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e88275a18696aa242c5290d90a66f6d087eac135
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.5.self_attn.k_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:723d6b5ec0b97159fa3ae6d2a1b54e424b997c5f96d5a8835b81bf5e72753fdb
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.5.self_attn.k_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.5.self_attn.k_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..bb92a3756c8b4dcfaecb9d7da89191bec4b93fc6
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.5.self_attn.k_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b317c4c2e41db117e26c41070e5d0bf267c0f4d260e6153fecf9251493edebb7
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.5.self_attn.k_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.5.self_attn.k_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.5.self_attn.k_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.5.self_attn.o_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.5.self_attn.o_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..22a626465fc04a3e45f1c1593ebc7802ecb99411
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.5.self_attn.o_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:628f4f3de092bfe3150372c0b703c3143403ec5b9838f3e12da85ea4bd685eb2
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.5.self_attn.o_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.5.self_attn.o_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e045e0c92d0af010c027df2970395333bfde1fd4
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.5.self_attn.o_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cf9ea8fc101af27090c40bf3a6c15d9025e7e09002f77db94dc990ceb743f1a7
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.5.self_attn.o_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.5.self_attn.o_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ee4653c1ecaf2cfcef023a2aa68b91f3ce1c0ed3
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.5.self_attn.o_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2d96a374140260bea36b03c0d7b4bf3b42ea1df1804213f8224cee413dd69785
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.5.self_attn.o_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.5.self_attn.o_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.5.self_attn.o_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.5.self_attn.o_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.5.self_attn.o_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..57ceaf9fa9441181e06027da0add223d49018195
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.5.self_attn.o_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:10f1fc2e489f69aaf73aa8e77cd7df3cee2eb3784db6b839fe583f1130e67133
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.5.self_attn.o_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.5.self_attn.o_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..63cf5867dfd6358b8fc5754b011f9b7583d442a0
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.5.self_attn.o_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c419ab5af8c790860f005675ea81cfbbbad79ae9c71f3f92c94cfe2135c9a228
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.5.self_attn.o_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.5.self_attn.o_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cccb552fb9e1d4dbf3c3516529e51328c5684e4f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.5.self_attn.o_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8961d99b0fae4f20d35149488aee891ce695be0445da75c3095923cca8b87a64
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.5.self_attn.o_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.5.self_attn.o_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.5.self_attn.o_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.5.self_attn.q_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.5.self_attn.q_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ac0d8083211aad0261803c572da5821215ce351f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.5.self_attn.q_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3d992302fdb5c52ecd20ef154e373031cbcbb25a227e79054f83423305df82a0
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.5.self_attn.q_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.5.self_attn.q_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5790054948ab9158176a0991cbd1a08e74022664
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.5.self_attn.q_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c89233056f15586d8cd4169abf82d776f0cfc9f8e72c4655fbe1f4750b747556
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.5.self_attn.q_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.5.self_attn.q_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8c905a9bd148ab607db0461c4f313626216313d3
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.5.self_attn.q_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc2ea4cdc1461a0c8d7b6741b4b2f533b941558ff892c1a2821c5121ae1c3345
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.5.self_attn.q_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.5.self_attn.q_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.5.self_attn.q_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.5.self_attn.q_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.5.self_attn.q_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f4a0701bc3e33f4d097bd4e0ce5757819ccf0e73
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.5.self_attn.q_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d609a876fb85ba67313d32b358c10baf7c461ac506e8c0a8cc932aeb010135a9
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.5.self_attn.q_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.5.self_attn.q_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ceaec99c501db23941db2d30e97ceca8a5e329ea
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.5.self_attn.q_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:defc8191d63992c4e516ac86a6e4a5039e715181b7d5d34ea982ce1d1bcf9fa5
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.5.self_attn.q_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.5.self_attn.q_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8300575d45ddf574e9dd256361ca4d46843238d1
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.5.self_attn.q_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ba9579bc10d181194afefc061feccba18faa83f9d084b950f5320caaaa422db0
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.5.self_attn.q_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.5.self_attn.q_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.5.self_attn.q_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.5.self_attn.q_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.5.self_attn.q_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ee0394dcb2e935ff0ca75770082895f400b438bb
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.5.self_attn.q_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2fcb2c994652b07864ed5be190494e6b7fdcf00a40f6c690d199174ebaeff3a1
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.5.self_attn.q_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.5.self_attn.q_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e26f9fbf46a13beba8911afeccfa975a556c90b7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.5.self_attn.q_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:40400e99ae5c9e8341aeca1b29eef5196dbf6a11025e92269512cb26b9992637
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.5.self_attn.q_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.5.self_attn.q_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..57126b98ab1ae40aacc968cccd18ca2cb25e7c71
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.5.self_attn.q_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:45de152e9fd847788280b24087c70d2254fb7243134027b2595974c913e409b2
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.5.self_attn.q_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.5.self_attn.q_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.5.self_attn.q_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.5.self_attn.v_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.5.self_attn.v_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..81957694c6c013e1ff2c3d4585510fd178878c1f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.5.self_attn.v_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a05573eb8f31fff763254334f78e1eefd97e675559ca7b93affa84d619cb212e
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.5.self_attn.v_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.5.self_attn.v_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..bd0abb79ec2de4c4b0368d7ae4004204cc957eb3
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.5.self_attn.v_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:127f215d27e2b6d6412a11b3ad551b94da48de48c8c8b9887bfe3fb74557c3a6
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.5.self_attn.v_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.5.self_attn.v_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..15d503e05273bc6f996c7f4f944cc97210bd8810
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.5.self_attn.v_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:75ed15bacb846e388d4ba40efcbdf68fb1dafc371ecbd21161788a88a256ff0f
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.5.self_attn.v_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.5.self_attn.v_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.5.self_attn.v_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.5.self_attn.v_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.5.self_attn.v_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c5e8832720277a89d58bfcaf16f211a927d2f149
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.5.self_attn.v_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:872cf6b7c05cbc4c0419557ffc6974148d2561d429a5fc42d0a4d2b23057b190
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.5.self_attn.v_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.5.self_attn.v_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..563a7e3b9d2d0f36cc47730dc2dddb5b241d9f4b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.5.self_attn.v_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3a0dee0ae26e46cc1926883853e3a5353b2d926795b65f43c618e91dc079d995
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.5.self_attn.v_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.5.self_attn.v_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6e5231bd1ee336cec68b072855753ad4d5839595
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.5.self_attn.v_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:665aed4283753605ad4a2ab407ab572bbe5b73ad196b42a1061a0ae94b5f611c
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.5.self_attn.v_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.5.self_attn.v_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.5.self_attn.v_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.5.self_attn.v_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.5.self_attn.v_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d12dacbfe583ae4d0753a0703e04f7fd6328f89b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.5.self_attn.v_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2071dbcc01ac8954d9bfa83fd863a80f4dbe127ac6eca58bcb89cdd545069b72
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.5.self_attn.v_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.5.self_attn.v_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..43135219f460636e7998fa3294a4aedf175d2f89
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.5.self_attn.v_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a308e9e9531a1576e34921a4247f959c8f32aa96c4ba11cdd091203696168ff9
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.5.self_attn.v_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.5.self_attn.v_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3671a1da5fe9a3fe3bb9045bdfc099df29d2f1b9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.5.self_attn.v_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9ec371623cb2b52ccfd69fea0c3410bd2aa2d34f50e78e67947cbcec2a891623
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.5.self_attn.v_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.5.self_attn.v_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.5.self_attn.v_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.50.down_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.50.down_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..84e15bcea0104c96f0a00a0f8eee94b21451bdf0
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.50.down_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1e20ed520434351eb4b0f66477fcf6e23d2827b1bcfb3ff361ac36dbe1db2283
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.50.down_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.50.down_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..da9b1f94b0c2499dbbef4bcdd5b7a9a0268c5252
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.50.down_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:efb0d7b83244e0e1ac332d3ece6b873941446c058f2d6d1bca1f6c524debe5fa
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.50.down_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.50.down_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..699a239195de8124c0b447f1ff07bd38426fbf6f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.50.down_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:99cb53db112aefc1e87cdb26145bc013329ce7144a24242af995b28ced84871f
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.50.down_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.50.down_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.50.down_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.50.gate_up_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.50.gate_up_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..71fc2160431746d35bd91903f9f0c51f150c1947
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.50.gate_up_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:97518935824945bb55111876b0e57873941c1b4e89286c9c95ff327e40c6624b
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.50.gate_up_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.50.gate_up_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2ea46b17ea5375ca8f5864d5015d12fc45aabea8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.50.gate_up_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:196eeecd483b50480b0806558f09feaff6ffa4e2bc88b032d64edbd224343b95
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.50.gate_up_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.50.gate_up_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e5e09f96f3ff30c1921da459845e88a518b547e7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.50.gate_up_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c06a475be1ac969d88c24843035f7709f93ae45a7e903b066c1a6e28bc7609e4
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.50.gate_up_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.50.gate_up_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.50.gate_up_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.50.input_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.50.input_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a05feca84aa8b885b6ed1fb171587cbce4b7898d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.50.input_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:87e4a898208beebc7dce89e50cad377b9e4b8eef3f480ba06be271901a8a1afc
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.50.input_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.50.input_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1f2b4bf2a02d73fcf8801107d5febb9f0f423317
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.50.input_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8f29cfbe4073fdf42a33cc650eb8f68c018630bee1481a5e08c0afb5bd86ad5d
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.50.input_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.50.input_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cc91fa6d384ea4dff0306d65b3e19a3d7d194fd2
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.50.input_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3d1d61a850c3facd272708b7a7b0ea7b47dc67e2938db1528e4b5b701dd31b10
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.50.input_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.50.input_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.50.input_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.50.input_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.50.input_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..095a45abae0ff9edd02aeba0268bb907a89ab2d2
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.50.input_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:03611df4f78658d97ca2b88ce1aa364f5f62941554fe8dbc942c8b37ebf4dfc9
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.50.input_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.50.input_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ecec0a0ed57d2244c03c235f82cb7711777e94af
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.50.input_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c0ed08d20914d6429cb222d3ea419b5231b746a948ed42d5fbb16ed33af3bc27
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.50.input_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.50.input_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f46e98183a0ffd2cbb02d1d8174a04b418a79e40
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.50.input_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6598fb73b2d90a8c32301c849e489b6e01b9c7996c9c965acf54ee0df70ec782
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.50.input_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.50.input_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.50.input_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.50.mlp.down_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.50.mlp.down_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fcb57ec197363400a3693a0b3c6804e8eb16eef0
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.50.mlp.down_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:53b2771fc211f3f4c6646c32cf189660de7298758851d257dcaf3d70e2b17a83
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.50.mlp.down_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.50.mlp.down_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5989c332adbed8c419d0940e28aaff077325f5e7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.50.mlp.down_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f3e1af16901ebc8a96f8b8513349e555eeb68ecacc2bb15a6fbd7d931fbdf314
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.50.mlp.down_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.50.mlp.down_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2914dc2457aa90bf3d81a5a940aca1890e05ddab
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.50.mlp.down_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f9f8e4d9689221958af09dcd23c3c624ce5aed1e9916f7d30582938e67abd867
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.50.mlp.down_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.50.mlp.down_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.50.mlp.down_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.50.mlp.gate_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.50.mlp.gate_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c0b3cb4bc778769649bd3009dcb6b2a7c418f026
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.50.mlp.gate_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d79150d5062764e352dd0669ad341a87742835ac9f3b96d1b31d629d2657a059
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.50.mlp.gate_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.50.mlp.gate_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..92de9c24e15ffbeffc24aadc87f359b9ead325d8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.50.mlp.gate_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0dd72225333990df8d0143e91ba0c4926b4ed6b10a750d68576a47d088d30bd6
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.50.mlp.gate_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.50.mlp.gate_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b82dfaa8651709558f5529a46dec8c4a39cbe24e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.50.mlp.gate_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b531cd715f580827816a998ba2090a76b781668b190c4d66c5a9f5483a17d875
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.50.mlp.gate_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.50.mlp.gate_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.50.mlp.gate_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.50.mlp.up_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.50.mlp.up_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0b1e4d71211e865f814a8691b2d9f5867b0cebdf
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.50.mlp.up_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:daeec8e17efce1747227da10bc58fc5afcd26feb331cbdbf925202ab6acb2e7a
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.50.mlp.up_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.50.mlp.up_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..226d580fa7f7c88c8da725c932b243f00a57666d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.50.mlp.up_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e663cee052c9ccf091daf413f975f95e5cb42fd2785604b4b192d2172dcd57db
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.50.mlp.up_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.50.mlp.up_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..16ff45e26bb5cb7e8b8739e4461030e9c7c9153f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.50.mlp.up_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0595ef9332018b14c17e8d4178506ef7b1d33e727acb139abab699f42ea35e65
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.50.mlp.up_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.50.mlp.up_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.50.mlp.up_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.50.post_attention_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.50.post_attention_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..06832a28243d0a4900c4a15a0e0feb6765325474
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.50.post_attention_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2e758006bcde4813655c3fc309db00c8835ecba37d96c022f9f4c1ff88ca1325
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.50.post_attention_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.50.post_attention_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..879a980c0640ad19df8c919d3e2f23b77e7361d7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.50.post_attention_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0f25682a476662a0e3edbd2bc2fc370109235d81e9289660e81a5eef7e3e55d8
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.50.post_attention_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.50.post_attention_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8d99845b0caefb40d559bee14ea66610a1a1dbeb
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.50.post_attention_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7ee208e4ccf9bc09dbdb8a49b6ba8f3747d4054de2be0eae3750e66045d6d6b9
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.50.post_attention_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.50.post_attention_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.50.post_attention_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.50.post_attention_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.50.post_attention_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..704cf542d6d60f46b9814a5225c8b33b02ed39e1
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.50.post_attention_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0714d4b769e3aea672b34b5b8ac32f6613baa17cfe01126a5d160643f31db5cf
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.50.post_attention_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.50.post_attention_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2e7c82bc448e52ca83a5c9f1864df94814d734d4
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.50.post_attention_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:782cf51d3dbe7f0317d950329dc4977fc9743b9dca017f7210bc58582464ff0e
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.50.post_attention_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.50.post_attention_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1ff2c904bad80ee4e35d9c3c409d0be53e26ebbe
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.50.post_attention_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d4dee71af990a22109d3d27049d43905e58d57e0991378abd391621de7899b04
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.50.post_attention_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.50.post_attention_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.50.post_attention_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.50.self_attn.k_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.50.self_attn.k_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fdefd2a2a8610352fd4de3b6a2604a7ab0cb1b1e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.50.self_attn.k_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:634fd3f2d412b27d8534a517829630750de43d6d57fcaee171d6221ff9884f06
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.50.self_attn.k_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.50.self_attn.k_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f64a0a1223865f89699d4e724692a1a3cb390167
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.50.self_attn.k_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a24ce3118aa294440b955e1be5cf00f5199a60809d03cad8b17ee17e4b215147
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.50.self_attn.k_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.50.self_attn.k_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cc6ad64c7238cc77a3313732d27d00e6324223a6
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.50.self_attn.k_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9165734212167a63fa5cf870bdb51c24a4dfdb0df5c062833810ec031f287463
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.50.self_attn.k_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.50.self_attn.k_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.50.self_attn.k_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.50.self_attn.k_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.50.self_attn.k_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b97abb6e79ce3da89c93c468437c601537c7e48b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.50.self_attn.k_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a9ec0d6cb96a7f376d9d93027cb797c0b5aac2a9f33e65eefe738595fc9ce2bb
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.50.self_attn.k_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.50.self_attn.k_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f71e02f9a641866ffbd6e19ff4ed80661c2a556d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.50.self_attn.k_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4e404a1a3e9308ebe6b2c5dbd8709997117218bae71ae179b09afb40b4277138
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.50.self_attn.k_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.50.self_attn.k_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..444b931271925f17f9923c8f5690b393a60e31cc
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.50.self_attn.k_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ec80e22505cb4ac7b676516b4c8f668176e2d46dd5ab2ecf6d4a6b46e7aba803
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.50.self_attn.k_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.50.self_attn.k_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.50.self_attn.k_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.50.self_attn.k_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.50.self_attn.k_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8f894d902c101a09162145c3ba8623bafbb9ca8d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.50.self_attn.k_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:21a5cbf7679efb1f0a58d1a225da153c3cfc16eac115ff5c30e267a9a11485c9
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.50.self_attn.k_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.50.self_attn.k_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6c18890eb1f330ec2e3e713c1a28a9f2b0e543a8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.50.self_attn.k_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6e996ebd27ae504ef7d8d47bb87412b93f7766376d29ee4a89b5f51e9dcfa1f1
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.50.self_attn.k_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.50.self_attn.k_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..696c6bcfc7abb75d391c6c9e66f6e4678d52064d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.50.self_attn.k_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6059e06f4d90f54ed4cd6bac56cd1a4e3deee335441b83273ee772285fb093ec
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.50.self_attn.k_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.50.self_attn.k_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.50.self_attn.k_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.50.self_attn.o_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.50.self_attn.o_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..04ce6718b194950f4dbe094753e19923bf8cf28f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.50.self_attn.o_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:317a2ff699999cc838d9064b9017f13618398fc39262925fecfde22f5dae8ba6
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.50.self_attn.o_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.50.self_attn.o_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3b03b29e2957f7461925d0e4b4b2ee7f0ad3ff9d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.50.self_attn.o_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:541aac57ae15364c0f1428f622b7d526753e6a1e6de558afceaf2305f1656f4c
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.50.self_attn.o_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.50.self_attn.o_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1601630da2ceb9a3928bafc982984f2006f08d3a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.50.self_attn.o_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:00fd26ca99040f3a124d978b1cbd121210c941d79c75f22fabecec62e9b98de3
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.50.self_attn.o_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.50.self_attn.o_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.50.self_attn.o_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.50.self_attn.o_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.50.self_attn.o_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3408c0adc3d354135ca5f4915065979c589c81e2
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.50.self_attn.o_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:49c80413100b87607bc6b6986365b6d54c52f5437476850e9eeab8087f802f95
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.50.self_attn.o_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.50.self_attn.o_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9780d2cea4ada41ec16e6ca753e80d23e057f4de
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.50.self_attn.o_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:916f6fbfd586e3b7f30fd9fee9e98b3670751ad58e9eda05ee573488170939b1
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.50.self_attn.o_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.50.self_attn.o_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..13427f297b81593a8c7fb1ac33307e48eb1ec444
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.50.self_attn.o_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dbb28b132428879d3695e976612a282e1987b9626c8f6fb4bc04e69c367a6d8c
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.50.self_attn.o_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.50.self_attn.o_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.50.self_attn.o_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.50.self_attn.q_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.50.self_attn.q_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a893be48471ab6df17ec25bff38791e9827e108f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.50.self_attn.q_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0f4f46b1458a984437dc41c10957f97d8c3cdf326390c37e3db5556860d167a1
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.50.self_attn.q_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.50.self_attn.q_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7b6215e2d797eddfc0487b06aaff91850531fb62
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.50.self_attn.q_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:adac5507b9fa126cae4bc2e0cbee9247ae7d99507a827a9aca41cf5b747c1608
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.50.self_attn.q_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.50.self_attn.q_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6beb1cb25a7410f508d85a340a52fcbc94376cea
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.50.self_attn.q_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7fc06723db0db7eb49f81e60cea4b78282c5556193563c81d05ca556b6f1393b
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.50.self_attn.q_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.50.self_attn.q_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.50.self_attn.q_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.50.self_attn.q_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.50.self_attn.q_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..acfa29a064ca31aa144f4d97561c543f742446be
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.50.self_attn.q_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:45fb12b0e49b9f6f4bb72d0a2f11fffe7ea8b23fe793241a3d2753a61402ddde
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.50.self_attn.q_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.50.self_attn.q_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..da15c2863dc3a8bded05aa7dfccacd728f9c5537
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.50.self_attn.q_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1c9efb7f0a152fdb02a06fdc5f7471fc68c0a72cc305737df1672874383ed42d
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.50.self_attn.q_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.50.self_attn.q_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..94c5b4eec33022cc17285494b7531e4401526564
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.50.self_attn.q_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:709546667b157bbb15119f6d3da8f3a13fb3a55ee0666898ffa6e27c362ac807
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.50.self_attn.q_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.50.self_attn.q_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.50.self_attn.q_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.50.self_attn.q_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.50.self_attn.q_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..15ad842139ca0ffceee60e8dbecb1b05439e2a70
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.50.self_attn.q_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dad507537f2296b7c87dc0a1d17db13f4019bee3480ebac6f0666aa1fcf99e63
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.50.self_attn.q_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.50.self_attn.q_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4b4da653f0197b3ac3264826eab9e2148c783ddf
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.50.self_attn.q_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8647ab3933c50116e1d89103ac211b2d21d47998fe01cb85ea02c5d8316302b6
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.50.self_attn.q_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.50.self_attn.q_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..517b3b1c9c6f6e564eab37d0099308ccac417b05
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.50.self_attn.q_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:76eada7fed050d0be335d2af5169d3bcd4162266d989454c1b7de19e0083fd28
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.50.self_attn.q_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.50.self_attn.q_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.50.self_attn.q_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.50.self_attn.v_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.50.self_attn.v_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c4c51d3e06f9646928c8eabbf666518671b31c1f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.50.self_attn.v_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:992fb14cd3581ed3715ac3f1a97fba18654ba4c2779431252e540f92b05b8d11
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.50.self_attn.v_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.50.self_attn.v_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..754bad71412158ae87a8acb4e52398fe2d4a548b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.50.self_attn.v_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:23df767c649f50be8ea5290a501991985fe5a2a90df1445e4a7048e481842f2e
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.50.self_attn.v_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.50.self_attn.v_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7d81fe55018ecc574214e8e7f374e9bd2e52727d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.50.self_attn.v_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f1add7675cb98485f5e334c34612bfa2264206895bd59b07b4c2765a0ab7a004
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.50.self_attn.v_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.50.self_attn.v_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.50.self_attn.v_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.50.self_attn.v_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.50.self_attn.v_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..645042e4a325e8e1b387f2ebd7d391e41d45abce
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.50.self_attn.v_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:52261146b56b3a99b73697e0a2890fd8e85939e1625cc8a09b4e526c4b693a89
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.50.self_attn.v_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.50.self_attn.v_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..82b9460e4cd269b205d954a084f9205c7d077b87
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.50.self_attn.v_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b74c0a785e8ec90204c22a14930057b70f8d0f622285a9dd0a5cde257191e53d
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.50.self_attn.v_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.50.self_attn.v_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..76984e24b410d47233893da096e18ecf5a4715cd
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.50.self_attn.v_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8253e5bda159ab47523d3d3cbf1f75ced050397d5e59c973a13c2cca9be13b6d
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.50.self_attn.v_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.50.self_attn.v_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.50.self_attn.v_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.50.self_attn.v_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.50.self_attn.v_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e857b559944614b58a5caafa4f7d975bb663624a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.50.self_attn.v_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f9ca5a73bbfa52bb33d75a52afd56d24a052c20c759a4652918ab6da94db76a0
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.50.self_attn.v_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.50.self_attn.v_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e47cfa8c61b89396ab9d28490b132dc7eebbfd70
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.50.self_attn.v_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f46396e075feb1f478faae5e75b2094ea02cf2afda047f2d4517fcc77cf5d720
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.50.self_attn.v_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.50.self_attn.v_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..136d6a40ba21a8fe4e081123c140140bbcf77e80
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.50.self_attn.v_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2e48e97eef37685791e24e5258e66b5b7626073c1f676425eecb6b1d4a8fff5c
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.50.self_attn.v_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.50.self_attn.v_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.50.self_attn.v_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.51.down_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.51.down_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9db05b6fb64b6f4633e98cff7829a71fcab65a24
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.51.down_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a6961e034242f7366a04a0bef01ba25f1706e8f74a7dd716daa171f45c311919
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.51.down_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.51.down_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3f548a60be48689b19eaddef50af12671d1bb00d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.51.down_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ebc66d16819528730de3c99b32d8f03813a8bc6c6fa0e2492440f74ae9967426
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.51.down_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.51.down_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..96b2f1c746ccaa694514273cb7e9cd0d8ebc9602
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.51.down_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:017f4992bf35eb39ea39fc4c5e749ac349c79da8339ec941d012e89f5c3b98b5
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.51.down_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.51.down_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.51.down_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.51.gate_up_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.51.gate_up_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..89913880b74c08a0bdbc312096045b23d91ebdf1
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.51.gate_up_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0533c58297d75344325af55c7628fc68f6dde7df56e08d4ed04fa907457565ba
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.51.gate_up_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.51.gate_up_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..99cf5f0eaaf01fb708f9b21bc1bd4a4787b97034
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.51.gate_up_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:56b222598763be5ed27d9eeb369513613f886c70b14684bd71c92720b1afc474
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.51.gate_up_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.51.gate_up_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b8ad531c75ade04ec45a352008f029fc38d9ab18
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.51.gate_up_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:11e1ab330e299d5c7a7f7c4def333b0d52aa0e86f088d5004aa6a3d63205605c
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.51.gate_up_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.51.gate_up_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.51.gate_up_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.51.input_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.51.input_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..76bcaac0f2586b4a6a01f289d7e54c44fd7e0f5c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.51.input_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6590bb0c6d51f04152c2d625bfd0cbcb212cae0bc266898aa5c6e53b45437d97
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.51.input_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.51.input_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0a1e53e1463c3796b30f63939a5e321167f58610
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.51.input_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6bd4f0286d00e67a5c90896737efbbe948bf681fd633ff05ae96f9fb9b250b4b
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.51.input_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.51.input_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..520eb4c05431683ac5da885fa6f0993a39126d3c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.51.input_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9eb2b3271bc75bb08bdc8109b6a663275fdc6068982f350c59c7d3fad2841cc8
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.51.input_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.51.input_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.51.input_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.51.input_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.51.input_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1acbab35ffdf61b914e55828d81eb2ce87550118
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.51.input_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8d70b04d97e0b64bba4374ad16d77c243cf9491a3c94aacb70e0356cc5cf875a
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.51.input_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.51.input_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3c1340289288eb446c0a289c8c1752ffb103d6aa
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.51.input_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:214a4565f211f4489fa1b1007f1117ab61b0d818f11cd938b335c6398da7ae6c
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.51.input_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.51.input_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3850ac02b8a2ba86f98467e42286595249148fd6
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.51.input_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5e1d4485c18a4ad59d51051d8194d816b17c07c8002a3ff2016adb7302c85a61
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.51.input_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.51.input_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.51.input_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.51.mlp.down_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.51.mlp.down_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b9b56c80dae3f6f63177da9b919868a871c54bc2
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.51.mlp.down_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ada1442d3926d0f57fdc1304faaf3259aba4c8d5ac543202feb82a400fc632ad
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.51.mlp.down_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.51.mlp.down_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a697b36bc626a4e8497380188b87cff54696e3a9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.51.mlp.down_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a399c4b538616d63886dfa1fb4b046780e706ca1497da80afacb3b66ca8520d6
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.51.mlp.down_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.51.mlp.down_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3bc7ea2ad539b7e1aef0f54e4497cf898d376f3b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.51.mlp.down_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:338e70796fe32189561fe2861f96816e61b4b332a6c47015374cd8dfb7ce4259
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.51.mlp.down_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.51.mlp.down_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.51.mlp.down_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.51.mlp.gate_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.51.mlp.gate_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9c4d89253c8e7b40537d65d4bd13ac1856c31d1e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.51.mlp.gate_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a4a551e3ed14b0ae5ca300773e7607c80e1ff3fd20245a6fd7cc37b1e28a031f
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.51.mlp.gate_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.51.mlp.gate_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4cfa7912b978502d79022757aea9d5a51633981a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.51.mlp.gate_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:028e8f2924980142c4d0a0b3682853dbb219531b6b0a2466f39e752ce5dc2005
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.51.mlp.gate_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.51.mlp.gate_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ee331b164797dadfcf036c087aeef7bf0bdfe276
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.51.mlp.gate_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:23bb05ab601878410c7437b4bf1f3ddf18ed830a98f537eec4396a95e4386ad7
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.51.mlp.gate_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.51.mlp.gate_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.51.mlp.gate_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.51.mlp.up_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.51.mlp.up_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d1628786386669b4b5261dc2d35595eabc572ac5
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.51.mlp.up_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:354be1b139d291003afca736381d895ffa03f46fa765170042215f8962b3bc77
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.51.mlp.up_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.51.mlp.up_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5a5771b63d405ea9cd788c0b91d0a7481e24386a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.51.mlp.up_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fc46f033124c2875d775e6afd71ec7405ddc962b0322867f347d71e83612048a
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.51.mlp.up_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.51.mlp.up_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..adc68f9a62aec40626c808029d7a64f38b28ab1e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.51.mlp.up_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c8f5550ea0dabc961614e6f7fbb21925af2a7c4181ecacf57f6eb9025539a0b6
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.51.mlp.up_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.51.mlp.up_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.51.mlp.up_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.51.post_attention_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.51.post_attention_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e78ff053161f7000a879b28213c4374d4594b4be
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.51.post_attention_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e70b257943e07bfa908ca8cfdc5d4d15e434e0bccb78844bbdf24a96ec7b9b4a
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.51.post_attention_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.51.post_attention_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b6cddf12e444ef250744d9570a3382bcd048eecc
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.51.post_attention_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:69a2ddb87446354cd6131cd73305801306a5d518cab1e01ae2e9e1754cb8bfff
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.51.post_attention_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.51.post_attention_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5461bf7b2e9851f9d1862ed8232f70a942278848
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.51.post_attention_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:16b909db8ebb876a5e93b88d0bf96e8803c1325ce66dcd8ecb59caccd12f14ad
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.51.post_attention_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.51.post_attention_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.51.post_attention_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.51.post_attention_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.51.post_attention_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..08e45e71e41255d54289228d1b2c6c274976e755
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.51.post_attention_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a6ffa78dba369290ad2e8267f80a0d58e3e684199e5088b73926ed94821daecf
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.51.post_attention_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.51.post_attention_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ebd25e6febfe3b91c54440148337cc9af806534f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.51.post_attention_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:78d2155b7a6041908dcb181946903b8c61852ada9f10cc9d8c2a91a8e2e9a9b4
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.51.post_attention_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.51.post_attention_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c474082aa2150efdc4de9ebac885ef290ecac21c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.51.post_attention_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61f9bb32cdb1cfd01b24e5aa1e1a55e4906e9d9a529fd5ca2a7252800fcc9255
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.51.post_attention_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.51.post_attention_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.51.post_attention_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.51.self_attn.k_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.51.self_attn.k_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c3f2048c6cd284974b1d3bf23bd6443343d3a893
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.51.self_attn.k_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a52a45de6fc4b1e1b555eefa8ee25cc52f73a186baa959e9ca03e55a2682100a
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.51.self_attn.k_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.51.self_attn.k_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..488a63e173c4a45e4ea45c4cae26b5d3ca84cea6
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.51.self_attn.k_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:006e7a12776625db1fed17293f0c6057a1a85136d507aff567f0491cda7d44bd
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.51.self_attn.k_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.51.self_attn.k_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4cbe10d355944a62fa0a9c29db26f764a07cf90b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.51.self_attn.k_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cfe4f390f40d5d272312dd091471a09be109b9458a41fbb4be4ffad2a2f688a0
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.51.self_attn.k_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.51.self_attn.k_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.51.self_attn.k_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.51.self_attn.k_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.51.self_attn.k_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2877ae82a410fe1684eaf5d755a58f4c93475aae
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.51.self_attn.k_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c92a5b6e8379b41812af9de6ffb6be9b4db970d010557e23fbda315644ce0b1e
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.51.self_attn.k_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.51.self_attn.k_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..48610dbd42b99a35b168b331c0bb8535f7c520ec
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.51.self_attn.k_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eb7b6ce4f9cb4d014bae80159efd0dd151f92680c07ef00e68370f36c077cc32
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.51.self_attn.k_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.51.self_attn.k_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5b006b353f91f1b01e5fabb4701f58f92d0c6b60
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.51.self_attn.k_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:80ced9d504fdd4cdcbca21549dddda16ddf35ae15eea8fd7033ddb7f64027543
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.51.self_attn.k_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.51.self_attn.k_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.51.self_attn.k_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.51.self_attn.k_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.51.self_attn.k_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..46e0456c68c9a863d0436c47b10597d62a8830a8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.51.self_attn.k_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9a8a469c15f0509c1a9d6fe048415ac4d019ab362f723c721611163c1b02329c
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.51.self_attn.k_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.51.self_attn.k_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..33b21d0de25c34cc8bbc1f7495c163d541a5c4b3
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.51.self_attn.k_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4e12f38fc373c6b1fbc3a5bdf273083f549ea8b15c2c674d7a91f7528ede5b38
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.51.self_attn.k_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.51.self_attn.k_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..823a7ee87bb0b4504fd82b6f44ea6d66f841c824
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.51.self_attn.k_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6173cb9b92691c91540b7e6295b2872f42da300b6ec1845bd04691be77872d5a
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.51.self_attn.k_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.51.self_attn.k_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.51.self_attn.k_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.51.self_attn.o_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.51.self_attn.o_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f26ba87ee6572798beb1448325866022db0bd582
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.51.self_attn.o_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6cb02a228c89551bfc5ec6efafc165bdc523aab9dfb48d775ac3e52c17ea33cf
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.51.self_attn.o_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.51.self_attn.o_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..da1bf0120c66ca6eb630b1e1d5cc2d5b4623d7ba
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.51.self_attn.o_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:33de5c591d58ca3d1a85344af812cb74a287f1a6099fa5624cb48095d8905292
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.51.self_attn.o_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.51.self_attn.o_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e995b2e5861d655820684019164de9bd766a1203
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.51.self_attn.o_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b72c64117556a7f966044b29fead79fff44c51c645030109cd2c91dee82b36bc
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.51.self_attn.o_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.51.self_attn.o_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.51.self_attn.o_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.51.self_attn.o_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.51.self_attn.o_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..40b20b6fa604bf538c9f2d58ac75947a1a7561d3
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.51.self_attn.o_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ee563ae278f20e05aa6bab29734d04ff5d5d4e25374cab5e1d7524f5799040e2
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.51.self_attn.o_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.51.self_attn.o_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6313ecaf681cd823262264552cb3d5966525e145
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.51.self_attn.o_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:86ebf91619517a5f77274c3aebcd0c0e6e3406a0c1749acc72919faeb618ef9c
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.51.self_attn.o_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.51.self_attn.o_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..80ab3b7e443e191c0f05480da5294fbaf1a85f5b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.51.self_attn.o_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a524970e4144b2b1e7e188d9715604e875ee5a5f9dfc07e4f1ee22d672e0df7e
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.51.self_attn.o_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.51.self_attn.o_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.51.self_attn.o_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.51.self_attn.q_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.51.self_attn.q_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2139f7cd1fcb7c66d00bed7c59d15de3cdf08d76
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.51.self_attn.q_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f5e339ea8ba08ac67a0fdef6a32ddec0fbc9c7b6142c717e4ecea2c0adfdca22
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.51.self_attn.q_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.51.self_attn.q_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8bfee9aa11df869adb5ad8572cec9985ad79c611
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.51.self_attn.q_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c78006388be70d5d4b89be7c7954aed276645a7178c490fea45f67dbad841ae2
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.51.self_attn.q_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.51.self_attn.q_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..514763898314f7b858b8d38e866ac92a308463e6
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.51.self_attn.q_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cfd45c8c04419a169f9c96399c948956e8b9a110df234f420b98dba1fbcfca95
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.51.self_attn.q_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.51.self_attn.q_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.51.self_attn.q_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.51.self_attn.q_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.51.self_attn.q_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5c2b7ad5febddce2ab4e7ed2632a927bc0bc1693
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.51.self_attn.q_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9f3b2b7728644520857e99d3e6fed2a85382cf8c2fae22094a88286712ce0df5
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.51.self_attn.q_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.51.self_attn.q_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6a71f9d8f6acd01d78d0433f75e1531aba0e2437
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.51.self_attn.q_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ddadb7e89ddcdaa9695c9aeee3b4984b54cf317029b74815314169ef0b7cafd7
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.51.self_attn.q_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.51.self_attn.q_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6fcf25938a1d104a4726492683a0389c86626c2e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.51.self_attn.q_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:140ffcc4de6c5e9c4924d21ddb2d18d3b3120727b6b35d23e9b84a5a33472b6a
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.51.self_attn.q_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.51.self_attn.q_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.51.self_attn.q_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.51.self_attn.q_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.51.self_attn.q_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ae0fb03933ae3fbfab6c98d195a064c9091c2815
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.51.self_attn.q_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef647b354432283c59d79c45286f1b1567754ff5003ef7bb68b4981de422bb0f
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.51.self_attn.q_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.51.self_attn.q_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..66e4fa8e110d44792f45d9c17e945a5f2c46a13e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.51.self_attn.q_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:64d13522a7247958b107c84df19f5520c8d8b705d9ac6e312392fb95f0b1ede7
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.51.self_attn.q_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.51.self_attn.q_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fb01be5086482e33b7563f56fa3181f552e2377e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.51.self_attn.q_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0007686442cbd7224ba4ca8d4bd665d33d452678864301a65de9599deada694f
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.51.self_attn.q_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.51.self_attn.q_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.51.self_attn.q_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.51.self_attn.v_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.51.self_attn.v_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..76af488749cb941ab09a85e32893d9153ba0ad10
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.51.self_attn.v_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3e18dee46fb101d101bb735de8e28af7ae70d488ae855ece1e168bd51abcbec7
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.51.self_attn.v_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.51.self_attn.v_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f05706972d82853fda087580bd827a07a063e57c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.51.self_attn.v_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d0b8df1f0472bec35fd159419a1a036de63cee38d38e6d5594f88a74f93f4904
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.51.self_attn.v_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.51.self_attn.v_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e6e2fe6b89bd01ac8742fec8471a2c24920816e6
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.51.self_attn.v_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3c4dfed04f527adb52ae7ebac1ef8f9be91a84893a5fb8f01192ae88c73fe736
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.51.self_attn.v_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.51.self_attn.v_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.51.self_attn.v_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.51.self_attn.v_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.51.self_attn.v_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..518cf7c7ffac7467d3381ccec83e691f83735254
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.51.self_attn.v_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:25a51ab39e8f03c07f579c9f9c51d87b3b91eb24eb5b733a83286627a67453c6
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.51.self_attn.v_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.51.self_attn.v_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3e74e230a2ae001032fbe12a1b3e0b7ee042deec
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.51.self_attn.v_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:131270e4362c769f0c865d5fd3937431c50dc85dc38b748c34ef2c8ea8474b21
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.51.self_attn.v_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.51.self_attn.v_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d4192048f4caeb168e672db6730399cb480fd93f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.51.self_attn.v_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fc59e8614e969e74f696974d40aaa35b0b98d84e60764cf738a532fbc37e6e07
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.51.self_attn.v_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.51.self_attn.v_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.51.self_attn.v_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.51.self_attn.v_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.51.self_attn.v_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..456c9c9559af83fe55833fd63f0442426939aa98
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.51.self_attn.v_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4d16b26d4b28b36754c8c67b07106e2189656622775a82532ee6a9f7fe031167
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.51.self_attn.v_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.51.self_attn.v_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65da25f516d720db3599472355f6510d61a97b94
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.51.self_attn.v_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2b035e5e2f4e39b3980854c22c6e6ece50ce5b20e533a71bb78debeff2c70c06
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.51.self_attn.v_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.51.self_attn.v_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..eae1d0bca0a2011302b4de9d0125db08190e61f4
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.51.self_attn.v_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:45a8e675c75390dd98c91e8f54577a639cea8cff8bc8fe1ae87e4ee8a51f8629
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.51.self_attn.v_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.51.self_attn.v_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.51.self_attn.v_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.52.down_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.52.down_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..513ae253cda46a639266a69368d2546069328418
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.52.down_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d9bf692c6ef656a07f2b433424c73444cf0a470451d461f7167f7bee3477dd26
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.52.down_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.52.down_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..92086304c401cdd58264d73fac214f7390388a04
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.52.down_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61b53e3681d54c81c5007a2d3cad41f90f7f7415af621709db3ebae64610f7aa
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.52.down_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.52.down_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8d4dc3296db04094a571e106f2cb50c640303e2e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.52.down_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:810dff006d2e83cb68b13851c4b228ea3b917a8058b212c13b8bf63fc1e282c0
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.52.down_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.52.down_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.52.down_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.52.gate_up_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.52.gate_up_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..271d667f3f022c650aab2c983c2e2ca19b44470a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.52.gate_up_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4e6ff63663ac84aede160d2bc4001852172aeca64ee8e101bb0671d1d399d821
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.52.gate_up_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.52.gate_up_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d385d634e3751e6f205f24f977570021632ea742
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.52.gate_up_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9530994dee5c564e4c5c9365d0ea0b2434aa4626c3ef540f220a76f08143b066
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.52.gate_up_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.52.gate_up_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..01af4203720023432230a98d11812320d5b156ec
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.52.gate_up_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4dc574fef37d33f2b6fdb9dd7621246d7a7d1bd03db7877d45c6ad04473bb3a6
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.52.gate_up_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.52.gate_up_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.52.gate_up_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.52.input_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.52.input_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1e77657c17da32879b8cd2aefde76bfc0fbff070
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.52.input_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:66a265054f2be27628a5070565230f81b8707caa491bf4e48c60969f0de3bde4
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.52.input_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.52.input_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9a87d31610c8e23c5c2e4b61d932bc554bf6b8e0
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.52.input_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3c34e1f31d096c8dae4e7db8d6fadbb726ed0412dabdd4c65e6dd38ebceec951
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.52.input_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.52.input_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c466bfdda676f979286052348a04ee0bc11dfcd0
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.52.input_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:16f657b647ff97ea217303bb31829697219677bb4d1ca1d2b118d7730921d4e5
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.52.input_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.52.input_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.52.input_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.52.input_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.52.input_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a408891a542e2c791a7aac81985e28917df9352d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.52.input_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e46b38522f8b5482d61571e3b2a5bd4e89161718685ad8e92a67580ffb395b40
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.52.input_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.52.input_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0e7217d29937f5a7fb8c60515c8235a90a8c4f2f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.52.input_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f927581fe1a9463ff3372d028c450de851592ee7ce67a1d90d0ee5b1d51d7565
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.52.input_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.52.input_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..15c45a909ed8edc83a30755dbebb090341d4cfbc
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.52.input_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d59d37682425769668e1615ae63767a8278dde16ce65aa7ae77a779cd579b978
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.52.input_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.52.input_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.52.input_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.52.mlp.down_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.52.mlp.down_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9cab561798d1431115ba48130045ef0617b56403
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.52.mlp.down_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:992ea1a9e70b5a475a23f4f2c68fa9726342fb08427f5616f03815069e7db103
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.52.mlp.down_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.52.mlp.down_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e5c8e84d70490790f2c88b2a0c4251eacc563875
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.52.mlp.down_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:075a6f6edbb28943a95abaaa19e4a84034d5d5db77d350360734a1ee949dfbac
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.52.mlp.down_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.52.mlp.down_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7aff1cb613c6e86ace4738dc67a4ab6602d62ed8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.52.mlp.down_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9b014b0e510acce1b7e628098d418a3979362903c507ee88d845b32790c0e930
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.52.mlp.down_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.52.mlp.down_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.52.mlp.down_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.52.mlp.gate_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.52.mlp.gate_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..70d0a6293181101914f6e9f359f5806ca5a01a4c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.52.mlp.gate_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f9d570e649f179bf76b5bd2ec5fa884589254c865f3f06d63447fbbf428acfc5
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.52.mlp.gate_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.52.mlp.gate_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a2d2ae5ebb4020a25608dab5324be7652dbf448a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.52.mlp.gate_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fda54dc8f000eca3e1a7ba44aed7501f5391205b917988a9c0f8033043222d2c
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.52.mlp.gate_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.52.mlp.gate_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5d2cf8b37301e45e615cf2d4e16cf62eacb8c03d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.52.mlp.gate_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e82db266215d39d1aea948b55e207c7f3e3cbd881b53c4aeaaec11535549da40
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.52.mlp.gate_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.52.mlp.gate_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.52.mlp.gate_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.52.mlp.up_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.52.mlp.up_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a605e51b93a8b0b2092bf14a415f7a3f1a1bff05
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.52.mlp.up_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5eca2213883e4e59c6d2306790208a87ac837dd206169101c92632dfcb3765f4
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.52.mlp.up_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.52.mlp.up_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b79b0069f37a0c96a5ccf45fbf3f611381a68cc7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.52.mlp.up_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9ebbbe65300f9f73ea1b9c8645a6613c9f4fa12c0fae52defc35ffd7f9f114c2
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.52.mlp.up_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.52.mlp.up_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ed42a9aac5da95b528e0589dadb638d91708ba07
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.52.mlp.up_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a8962b65923763816003d581e6d2644e23aa08b3c9fe63590f99e3e8b6ef7bb4
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.52.mlp.up_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.52.mlp.up_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.52.mlp.up_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.52.post_attention_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.52.post_attention_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..604b3313f3c4ebea80da55f512418be0e6b041b1
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.52.post_attention_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:edb69c648e3e6e9dedf17d2eae15bdb85f5e78e17ad8dfc5a8befc4e64e40383
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.52.post_attention_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.52.post_attention_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b38f0a2a6718920508de0407193927f86b0b9393
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.52.post_attention_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2553fa95022c9f3f88f13e646abcaea2519e7ff0f00835d29591b33cad4306b5
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.52.post_attention_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.52.post_attention_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2030cc702b09e3ce220a5ad2172c2f19e1b4eace
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.52.post_attention_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:607543b1e78edb92ef3f5afd66172c29be80325b82db32e44125c8ea2c241cbe
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.52.post_attention_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.52.post_attention_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.52.post_attention_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.52.post_attention_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.52.post_attention_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9182cc230164faf80619ba0dc0d2fa66b284178f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.52.post_attention_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:76c2ef93d010b47122378c732236910a4f204c20aedbadae0f6ab9dd85f72df8
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.52.post_attention_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.52.post_attention_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3c37938440ac53644476f8d8480accb6f52957a9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.52.post_attention_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f6a104d38eb448544ebd58dd0a92477e398e47a8c7925f6dd6b8d933be522308
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.52.post_attention_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.52.post_attention_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..62aab55ce65e52ce32ae0af272e8b7916c2402f7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.52.post_attention_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:02d2ce6779b09529a819385cb00c6defee931b7f19022d86dabda9e908172387
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.52.post_attention_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.52.post_attention_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.52.post_attention_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.52.self_attn.k_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.52.self_attn.k_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e442c87f2d6d60829f3fe2d293b94226f0dcbe0f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.52.self_attn.k_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:607f789c76758ac37c50146a758ead407bc18b051214fbcde961ba6979afeb9b
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.52.self_attn.k_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.52.self_attn.k_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d17c20b658962d269beb2e10bdb907ad1a41b9de
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.52.self_attn.k_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d5194723aa75aeae1cf4b1c5642d263272b5b7d13d80010bf2d18a7d8490dc87
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.52.self_attn.k_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.52.self_attn.k_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..af58b68236a58ce484075e19805b242b7f8c905e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.52.self_attn.k_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a90e7ea2aebe7ca30c711ea7f0a6bcffe9779af805aeb94f53349b0a87fc3d99
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.52.self_attn.k_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.52.self_attn.k_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.52.self_attn.k_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.52.self_attn.k_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.52.self_attn.k_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..76b86bf82106e058e4c670008abe5d5407b0276d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.52.self_attn.k_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ace508d44cb7606ecff17a9a785c44695c07a15917e54d17b6d897477c54d500
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.52.self_attn.k_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.52.self_attn.k_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..18ed007975988ea86406b0adb8eb68bb0b57bbc6
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.52.self_attn.k_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:971edeebb6c58de2baf670a3c1614cf48018ab77c3afa64745e10aacac7052d9
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.52.self_attn.k_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.52.self_attn.k_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2d667456f2bbab61ae599c3a0dc87888fe19e81f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.52.self_attn.k_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:93414c1cf50b4cb0ffb19a75e23de771c6ae66b53d379ff8b1ef9da01ade23cf
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.52.self_attn.k_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.52.self_attn.k_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.52.self_attn.k_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.52.self_attn.k_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.52.self_attn.k_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a03fd141d87d7a64c749751fd9a0f80af1b00f58
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.52.self_attn.k_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4860c7d597e34f3361cca8f1a65e45e81364b06bb0baf978b393c2cfa9da3712
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.52.self_attn.k_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.52.self_attn.k_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6f70cd77a0e4d8217817e9f1be4c7cb88cf5b6e8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.52.self_attn.k_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:26261bfa107235e7390784058b93c4ebb35f3e4e54617b2397f237e1f8789e62
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.52.self_attn.k_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.52.self_attn.k_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cb5eb291dd50a5955d51a1c2b01f8590914a5ae3
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.52.self_attn.k_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4d933d87058e23312ccb7b93019d67dc6317cb3265f4206799ee6d787ce6a7a0
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.52.self_attn.k_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.52.self_attn.k_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.52.self_attn.k_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.52.self_attn.o_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.52.self_attn.o_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8451f7c58993b9df4476e1a7ebe11846f8f2879b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.52.self_attn.o_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ee02ab9761b4fa17c22c99247da0c3304508f83b640914dead9426b57b9c7ac4
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.52.self_attn.o_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.52.self_attn.o_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5b1914cc9bdb1b75b476f660969eca680267416a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.52.self_attn.o_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f438c20d95168d51aafa61e778450e8b12eac0e7f5aeac01b03fef3befd1ac98
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.52.self_attn.o_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.52.self_attn.o_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b746258cb00ed43c774f5085faee2f212341e6bf
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.52.self_attn.o_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ab3e01a15e81864c7427e6227c87e2d405e6e0327c785944b90fc77611668dfa
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.52.self_attn.o_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.52.self_attn.o_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.52.self_attn.o_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.52.self_attn.o_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.52.self_attn.o_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..32cce667bbddbc6813950a064fc89f84f9365525
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.52.self_attn.o_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3a7086e3ce24bf132b619dfca3ea72ed57a95d66eeca462c61bbcc8a1bf168fd
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.52.self_attn.o_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.52.self_attn.o_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..385de07057991009f31db31b8b8bda27bafa945f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.52.self_attn.o_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5c0ec6eb1dd9a890d4e584263654d49057b40eaef28dd45059c3e1903be53e1e
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.52.self_attn.o_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.52.self_attn.o_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..38074c7ceebe1f672643c65c7205804fe5c25059
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.52.self_attn.o_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5a63577b9b814dccb3e08c9993f1c37b322cfcec7e39b85994e4db7c11e64a0d
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.52.self_attn.o_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.52.self_attn.o_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.52.self_attn.o_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.52.self_attn.q_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.52.self_attn.q_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c02edb1fd008aa4110b8555ee437dfc32cb81ef5
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.52.self_attn.q_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4d761c839726470af280857b9644065d7c99349998a80f45baffb9d497da9e2e
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.52.self_attn.q_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.52.self_attn.q_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b98059f7192d1bcef5f9184bba9c77370098e321
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.52.self_attn.q_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cdf297c5d9e6916704ed4ba112de64fe0d91a8a3226dd09213bb88eb73da8151
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.52.self_attn.q_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.52.self_attn.q_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..df6335487c04e7528ef6b18c67da989a675176cf
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.52.self_attn.q_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d85a1f47be29c74b5211f1e7a285fe6463be1c6e6929be8cc5ee32370e6956c6
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.52.self_attn.q_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.52.self_attn.q_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.52.self_attn.q_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.52.self_attn.q_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.52.self_attn.q_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b0722cd60d273065ff3f6cbf12e1b6576c869282
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.52.self_attn.q_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1d85762d63b5aa1719d0f56245a1288c65e740dfec9e3690c100896185ca999e
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.52.self_attn.q_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.52.self_attn.q_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2ac21998abdec027659eed45f6840e221456e3ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.52.self_attn.q_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:98ddda0605c462a9f195b26cbbe09d11ec9dc4ceabdd9cad878172c2232bec0e
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.52.self_attn.q_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.52.self_attn.q_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..82862a1476fbbbef7abf0b88c8c5778ecfa9f355
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.52.self_attn.q_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0c201df522ecbd854dfcc9c17ff27f9fb2908b684dc9b7da3b51bc6f9fa369c9
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.52.self_attn.q_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.52.self_attn.q_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.52.self_attn.q_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.52.self_attn.q_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.52.self_attn.q_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..bbf46756e9ee73deeca6e2e21136560ac718d842
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.52.self_attn.q_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:de956d4dbce692f4f2d43cbb4e71ff9f58e6b7b1012cc3faa53269576a107edf
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.52.self_attn.q_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.52.self_attn.q_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c20e6b490a816698d1967e9ba19cfe5b36fe9236
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.52.self_attn.q_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2cef3dd123cda7f817b4915bbced3842e8e3687bd0ed28f6673e6591815008ea
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.52.self_attn.q_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.52.self_attn.q_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4a6dc67c55ce25a77e35edbf4b43fa2a1c292f0f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.52.self_attn.q_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:96d320a71d06333c6d7fc2b495d5a9db83f21d2771520ce9dfb3eee5163bc4fb
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.52.self_attn.q_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.52.self_attn.q_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.52.self_attn.q_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.52.self_attn.v_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.52.self_attn.v_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ef40f2760a86f3f35686dc9fc7fc5a1bfafb177e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.52.self_attn.v_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c2a35fc6495e18568cabe693bd3f070249541348d51c932c4827e740e905be25
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.52.self_attn.v_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.52.self_attn.v_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ea41e1458e05877e10760d80bdf4ab562f7a5871
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.52.self_attn.v_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f89d4e8d6c58bcaa40a0e56e8e68b1becc745bf7db0a909e11f4800a51a7d302
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.52.self_attn.v_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.52.self_attn.v_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ecb610afe6985dade2738f200f096da03c4ecbf5
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.52.self_attn.v_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f1f5c29f384151999b2d3c6463be4f74d0ea57a68ea150a9c3984a66e13d6541
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.52.self_attn.v_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.52.self_attn.v_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.52.self_attn.v_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.52.self_attn.v_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.52.self_attn.v_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b4ec91dae56d9bbb33ff6b0664fbc9c031fc9ca7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.52.self_attn.v_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e3bfcb4fee69929c8b5db4fa1569e7a0fa902906a18bde6ac270158a1e4314ad
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.52.self_attn.v_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.52.self_attn.v_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..edec10e371a507c9e1525f84841757f5ecc5eeb6
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.52.self_attn.v_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:538c7bbbe4f8983b121a62b42e3cec49f334df33c12ab504c3cdc85ceecb3b93
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.52.self_attn.v_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.52.self_attn.v_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a989058458845bb17ad0cc05e62621c9f9c9c586
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.52.self_attn.v_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:12464f870aa947a5a2de8338b4cceb635ec6ae0367e435686b3a0178467bb4ff
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.52.self_attn.v_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.52.self_attn.v_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.52.self_attn.v_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.52.self_attn.v_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.52.self_attn.v_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cd8694a660d342cb245556d36decf0d2b38e5707
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.52.self_attn.v_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b00fa67a8294682e2978072ba42e2bd228b195ca6f83b11ef817e0d81c60be09
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.52.self_attn.v_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.52.self_attn.v_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d7f3692a9b73b11508e5905492b55bc1f07a362a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.52.self_attn.v_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dcf7f8f8c901519b5b7a2b368ec4fc3f4dd97659b8cbd5abd23deb652ae96ef7
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.52.self_attn.v_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.52.self_attn.v_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8f98c83be44366a441347db6a20210d427b5c1d9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.52.self_attn.v_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:25558810d650c7222d95b46fe1c576d5cdb559818cc59a838102ba9cbef7f738
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.52.self_attn.v_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.52.self_attn.v_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.52.self_attn.v_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.53.down_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.53.down_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8114be5cdc52113b2b8ca60ac0b622471c234713
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.53.down_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bbda4df5babac4eb147fae73981178daabc9ee40e5638e4d7cc701f531c703ac
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.53.down_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.53.down_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e6e4d39daf199da928d0d121d902e1788ef61a52
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.53.down_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ad146c20adbe1e80d3cd03f929ff181138c09ab64b0e3c668bde9b5234cc90e4
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.53.down_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.53.down_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e29a333febf428f6e14bf392b56325806ed1b1a7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.53.down_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5103b0b381fc4d1833361e82c3a87be1d397ae13e4ee292738073beed6ad2685
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.53.down_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.53.down_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.53.down_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.53.gate_up_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.53.gate_up_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6b312bb16f4610c772ecc38bfdf758e70f8e4f4a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.53.gate_up_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:08531710c0eda43de7d4fa0ad59f68e75a87827d6c24104175072ac38ed7213c
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.53.gate_up_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.53.gate_up_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9031ed9662dcc576a740062aae0af02d4abc251e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.53.gate_up_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cf1efe3ffd7652a895d28ea6457be6200fdc587b68229468041695dd10d825c8
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.53.gate_up_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.53.gate_up_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9134b090faf1eaf32b13a3f54a02f23088273882
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.53.gate_up_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:11d767b72ceb3d9ebd8c527d96fbd2db5c09572ecf425cc3c864d5bbf9e4b942
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.53.gate_up_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.53.gate_up_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.53.gate_up_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.53.input_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.53.input_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9cdc90435c5e7743ce4a16aef7bb5e8efb78377a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.53.input_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5fed166451c36c5675a46b1505446ff16af574047308bf02b7b3dbf982f66dc6
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.53.input_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.53.input_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1612df1b4f4df82f7f52fbd4cd109ca353471734
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.53.input_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d82e921223c80f9c0206bddcd9fbd7fc5e704f06797efbc71333b5bb14dfcd36
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.53.input_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.53.input_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e3df045c99a3c1a0049ffcc4edee78b520fa8c5b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.53.input_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:df6b7ebd578508deea06142d4bdcc5b4f533d4b026a14e0b7ba333e9abd407d4
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.53.input_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.53.input_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.53.input_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.53.input_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.53.input_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..378650fbb963d7ee02fe59179fe467788c48c189
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.53.input_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:05d3fe68af2bb228aa76ac8ac2e305d7d4a58732f8d51e095b796521ca28773a
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.53.input_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.53.input_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a9c232bc02fdb325d8f15704630b8cf1f9474dc9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.53.input_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7c961dde1a02fbcf4c46be33c78463fe8624f9f14662b4a3964a9558c76e1aad
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.53.input_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.53.input_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..930e273556e8a21c1d933ba08440f3f2c755ff03
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.53.input_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:403703e1e8e4b0445068f02e58673d734cf72f76efd1ae45b872bc7eb941e706
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.53.input_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.53.input_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.53.input_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.53.mlp.down_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.53.mlp.down_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..024d245484df153537ac981a6945a4c28ec2986b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.53.mlp.down_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a917674239ff63a3a0448fefc4557cf3efcca7f30a3385e914c8b305c406a0fc
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.53.mlp.down_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.53.mlp.down_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b273a578fbed2c967f23c757aedc5fe2d0619d87
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.53.mlp.down_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c76e7ee2ccb3b2f5db4a41c51aef5768f6a5474e3cce6cc35111d9a23845bb45
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.53.mlp.down_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.53.mlp.down_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5c2a6bb91c2d2c0131b8d622e7ef746bc30800ab
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.53.mlp.down_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d5466d0e460cec985777fbb14b58b771c19b534cabec92532cfe857956e9268d
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.53.mlp.down_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.53.mlp.down_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.53.mlp.down_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.53.mlp.gate_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.53.mlp.gate_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b81107a71b247ce75729deec82498b22db3ef851
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.53.mlp.gate_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:054953bb116e002478d1b42f4687001d8df2c49830aabefa25d0ba29f5d5af01
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.53.mlp.gate_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.53.mlp.gate_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0442209b062b8c18391cd95dd968048875b12298
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.53.mlp.gate_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f58500927db1fafd200644463e517e326bd252a8f88d0488d6e14d454c158a84
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.53.mlp.gate_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.53.mlp.gate_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..913dd7cd249c8d2244f310e25d9b6afdc67e6cbb
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.53.mlp.gate_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c6108db95605a34e58ce821b421adad127b80895489d3a1837c79a3786e988d3
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.53.mlp.gate_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.53.mlp.gate_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.53.mlp.gate_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.53.mlp.up_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.53.mlp.up_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c1690529e51e19d3cce40909a807b6ed7d686bab
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.53.mlp.up_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8129e3a60fe82209c999602c77d9ae050fecc72728485dd0aa89396134973f00
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.53.mlp.up_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.53.mlp.up_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e0052858db98ec582765ee04ab7939cc08f6d5a3
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.53.mlp.up_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:421ba666b938a0a087d93d2dd4b79d7b4cd71c6d840afaa8b192e6e429d7d81e
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.53.mlp.up_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.53.mlp.up_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..23fb473edca2377c5fa03224d731eb91b46abb01
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.53.mlp.up_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b2023e10099471857afe9a73e27528d639b2b7924220ced651e8d38d76b3d00d
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.53.mlp.up_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.53.mlp.up_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.53.mlp.up_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.53.post_attention_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.53.post_attention_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..289ab53431e21d954dc29073a298f1377fdfe1b0
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.53.post_attention_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:77690f8fb70a44579d3266f45ceabdd5ccec9e22f47c11a32f8fbdee4f03c756
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.53.post_attention_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.53.post_attention_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..42b2b1171bb587753e9607ac201c96ca3486eb8b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.53.post_attention_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fdd67633b0050e7d456fbffca36f74acce2dd847ea3fa8a12fc52b8de0c111de
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.53.post_attention_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.53.post_attention_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ea6bb7537192f4fb9eaee3e9772b01eeb97e80eb
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.53.post_attention_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f1ab87aff695cdb8bcaeb409bd2c3bcc2103e1bd6c84f5098579de994192e8c0
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.53.post_attention_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.53.post_attention_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.53.post_attention_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.53.post_attention_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.53.post_attention_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e85e5d0d2c26e4ac4007f2fc6da05883c2136d7a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.53.post_attention_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fb75bd3e58321fb63149a42ea575bf036a77ca4c1746d84b2b7ff8317d0c817d
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.53.post_attention_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.53.post_attention_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ada1110c1ea5574c46ea178ed09756477753cdc0
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.53.post_attention_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8b1d436ee6e1dec3dcec599f1b4cf23ec96150c976ecfabfd08ef0fc04777e63
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.53.post_attention_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.53.post_attention_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8bd677c078f280f0fa87985fe50f041454cfea9f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.53.post_attention_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3c12b190573630191100429c7514cf70dd207507de3e90e1e10ceb3dbeafb6a3
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.53.post_attention_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.53.post_attention_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.53.post_attention_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.53.self_attn.k_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.53.self_attn.k_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1f0e079a98834c2520feedfde5a0bf157c1e0cf1
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.53.self_attn.k_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a8f1587f9ee7b5f3cb5bfcf81deaa4dcc0f6b66c091079e7ec32c4a0121f2065
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.53.self_attn.k_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.53.self_attn.k_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..11a8452c5daba99f9d032ed5ec6499adf56b84b8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.53.self_attn.k_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:023b286ede27bfa2fda48d9a44d2238c5c4d9ece4f55df33b9e2cb6c7fa07623
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.53.self_attn.k_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.53.self_attn.k_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f8bdd12f2373e41d7585ddcb75e72239176e23c4
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.53.self_attn.k_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:745a571e8edbfa297a9a616ac9dfa916e568a3965d2965c44835810831142cd3
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.53.self_attn.k_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.53.self_attn.k_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.53.self_attn.k_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.53.self_attn.k_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.53.self_attn.k_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..91f9f0bc04ed191f90e7616c7c45c6f1a9e7899c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.53.self_attn.k_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:69899f75151abe1ab27ab8609637862e37e0b2ecfd0320dae7e13c4164165b0f
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.53.self_attn.k_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.53.self_attn.k_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..bfe357768cd8c6fe729edfa9e9cb47f656da5d92
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.53.self_attn.k_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d6e94720475d53c1304c3ce5793ef12182039cd74f2661e2248cd1a703f8eb0e
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.53.self_attn.k_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.53.self_attn.k_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c35a1eb395f9c9950bc6b976f2a48afa4447080a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.53.self_attn.k_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a024335834e6c79dbd7eda7c3da80602ad18c24d175c9694fd74fc5547a21b7
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.53.self_attn.k_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.53.self_attn.k_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.53.self_attn.k_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.53.self_attn.k_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.53.self_attn.k_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ccdafc4ca75c8e19e05e50b3210a5ba590a0dda3
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.53.self_attn.k_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4dd390d801b93fed8cefc94bb26f24d04b98aeebab3738128d7e6ffc0d2fa1e4
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.53.self_attn.k_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.53.self_attn.k_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e4ca08a8b1091ae29ca639d6f24ae81d7dc2a156
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.53.self_attn.k_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d955be443fc2a0c4ae6aa987d92ebcb853bf8e213d0ded70b180c62984d6c058
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.53.self_attn.k_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.53.self_attn.k_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cd7d087cf71fe29dce5256f3ef1f5fb4f53bf702
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.53.self_attn.k_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6eec0cbf1e57a31ff19fd6bcfd89701adf9e21b091dcd088d343cd07004e4a5f
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.53.self_attn.k_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.53.self_attn.k_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.53.self_attn.k_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.53.self_attn.o_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.53.self_attn.o_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ff4d6802b221fba8ed1bd7c76b85a884a284f69f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.53.self_attn.o_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d88261f80b612cbf700816d05783cc6a7568b972bb92521db63457dce6910df0
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.53.self_attn.o_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.53.self_attn.o_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4c8382b41c73a4edf386511b4d22177521bd292e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.53.self_attn.o_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0f5ca03e3616c17b654ca6456761c17aea31e38f4977f5729afe703270d11801
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.53.self_attn.o_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.53.self_attn.o_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..11ae5dfe57ddc0af95f57cbc8a2653052205d0d5
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.53.self_attn.o_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:afe671c0847c846a463c7aee9650c15475911f850cd07bfe8ad67460cef88094
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.53.self_attn.o_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.53.self_attn.o_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.53.self_attn.o_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.53.self_attn.o_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.53.self_attn.o_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..687d8211cba47ef4af83b6063b914e2834953976
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.53.self_attn.o_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8c5eff2fd04d819a0bb1fcfe5ed8da255204a88a22fc08b59938ad35062b99d4
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.53.self_attn.o_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.53.self_attn.o_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e38027dcb74168fa04eb2f6fa915edcc61158a59
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.53.self_attn.o_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:95344e74e94f505079d6f63504030a4bf23a07521dd09bba75e72b5c2e2a4682
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.53.self_attn.o_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.53.self_attn.o_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6185fc15626e36a2a966026cea7c8cc9d5e3e6e5
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.53.self_attn.o_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:04b9f327d57fc6ec75a3a3947fdc605845922cf45c03d244d9c60b915b0df4a6
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.53.self_attn.o_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.53.self_attn.o_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.53.self_attn.o_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.53.self_attn.q_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.53.self_attn.q_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7bc6b37b07f6d5287cfd9e89f77c9eeed1fc7a50
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.53.self_attn.q_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:507f802ab0fecda797e39083c811ca02af06f9b331a5ab80805b3285ca49afa8
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.53.self_attn.q_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.53.self_attn.q_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..40ad2b0463466e16605c0352ef6643f29dfe2de9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.53.self_attn.q_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:94f8bf040753ac19145c7f42171ae59b825b0275258bf8a2f4a97a59acdbbc7d
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.53.self_attn.q_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.53.self_attn.q_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cbfb68fff69a63b3bf4e785140b2ec9a37f9e226
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.53.self_attn.q_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a4302a6a0321d6ecb60263bdaa99d6fa0ede6e6c70f56c01d2c0d5e9011e49d2
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.53.self_attn.q_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.53.self_attn.q_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.53.self_attn.q_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.53.self_attn.q_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.53.self_attn.q_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..996a88ad7ec915dd2968fea81e9da0cbd5b6f547
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.53.self_attn.q_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a99144fa56315654cf9f0e4a0ee543ae5b61f1d7fe638afcaa78718606cc849a
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.53.self_attn.q_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.53.self_attn.q_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7863dd2be4e1e1d731799ad205e275e616dadd5b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.53.self_attn.q_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4e45f0ebcb5b51933a017c08e50d90dbe616e693d1f91833470388dfbdee1d30
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.53.self_attn.q_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.53.self_attn.q_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7c23e91f0c7fc2146179082677f673bb80ba475f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.53.self_attn.q_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1f3b8090355c66162018b70d196e4205ae96f2862072a8a2313c6ef5483d88ff
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.53.self_attn.q_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.53.self_attn.q_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.53.self_attn.q_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.53.self_attn.q_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.53.self_attn.q_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4f0db39a0137135d4262ed8ecba5187f4cafc5f9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.53.self_attn.q_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aea48678df921f19dae853e30fe78cd06d7164ca6e6a71f81a0d97f38a5e53ab
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.53.self_attn.q_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.53.self_attn.q_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1e4a4abf3d0203f2c9476fbb27c04805f5d65e30
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.53.self_attn.q_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:34f46a21a1e3d8a2a8b4ed905f7bf15f2daf2d2dda719d8b45488d30aa8bfa7e
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.53.self_attn.q_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.53.self_attn.q_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..24bfd4dfd2cdfe1c9bf0fef5043097ba8e2dcd69
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.53.self_attn.q_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a8b56ab59cb31980032faa9c5af16478d7b68c0dd9b13d3a97483009cc1c78ee
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.53.self_attn.q_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.53.self_attn.q_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.53.self_attn.q_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.53.self_attn.v_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.53.self_attn.v_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e24eaf9cbd087bea58a239c9bd62190e2c0571cb
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.53.self_attn.v_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fd5cbdf7b2699b2762fde626e946a6a2497f5bd43cc1950e0e2e78d9a595d767
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.53.self_attn.v_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.53.self_attn.v_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..58f994340282964393e77824c97503bd82864144
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.53.self_attn.v_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c78ff84e633ed0ffb041ed4937b3639af64f1cac2ade5a512e4582b972153e7f
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.53.self_attn.v_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.53.self_attn.v_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b11a9dc7e66248c4bc0dd1b001614d9542305769
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.53.self_attn.v_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a0d1e20068a9a75bb819ec4734ae1388ce1f43841aebe2e266af7d00cd0fc8b
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.53.self_attn.v_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.53.self_attn.v_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.53.self_attn.v_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.53.self_attn.v_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.53.self_attn.v_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2720881704e3d872a82e09a90692e577f94c9359
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.53.self_attn.v_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5210d8f6e5f6a6ad38d21bf43be15902373482744d9fd60f7ba666243fa06dd1
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.53.self_attn.v_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.53.self_attn.v_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ead8ad4fd8ffcdec0c3d4234a8dc94cf15b16d8d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.53.self_attn.v_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:14d6a5ea06fe9281b216b075a249447ef12d00a449b77ad89435ecc37c35020f
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.53.self_attn.v_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.53.self_attn.v_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e3306c8a52984959a0407052ba56bf968015e648
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.53.self_attn.v_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:58d38761e72f83ca339f036b54c8ad7b77c94155b1c929b0db0b576a28b538b1
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.53.self_attn.v_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.53.self_attn.v_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.53.self_attn.v_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.53.self_attn.v_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.53.self_attn.v_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..88874a3b9415d42452ceb33fe0423c0fc8114105
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.53.self_attn.v_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d01d1f4afccd1425df40c74f262e96e81625adefbb07edd54c1d9c7476574e37
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.53.self_attn.v_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.53.self_attn.v_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f52021d0985518efc43849a042ffea020b3b9151
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.53.self_attn.v_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5c0c897f0402d23bced30c5dbc3d977b0d96d985f95f59b2409a41547ae99a71
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.53.self_attn.v_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.53.self_attn.v_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..64bd3882b5fbf93c0341533ca475571197e42d75
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.53.self_attn.v_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:374983c6ab5f5b33bf8474d9dc7edc80122d13abd2456cbc66720e6583930ad6
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.53.self_attn.v_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.53.self_attn.v_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.53.self_attn.v_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.54.down_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.54.down_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fee090bb0cd5ffe33463f3490fa69f9a217ec518
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.54.down_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a0f9d7e67f377d2197d9e37245abd2571e603d3a7292052065c7b46a4fbfbd18
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.54.down_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.54.down_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3110b82345235f259a286fd88c05ae9f18a19547
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.54.down_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cdbe97f8ce164a3a9bf17e030c2fb2abcaca080940a215c973c54233d3d0b3a8
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.54.down_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.54.down_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e094be7b3c515955af479d396e69dba572029c76
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.54.down_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e81b863c07807ef0fd1bb9b7c7af91607da3ea2a23db582632b4eb9fa1889a00
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.54.down_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.54.down_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.54.down_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.54.gate_up_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.54.gate_up_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7b43b2a18453dd43fd64bf5b151008550c496013
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.54.gate_up_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be73416ec3a78e9efba52dbe83c4f5891f784068362041a71218d46c17d9aa7f
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.54.gate_up_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.54.gate_up_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..14324061240d78d7759414e806632da57eaad7cf
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.54.gate_up_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f99e4602a402bef7caa7b315860e449e7e8f2da27c2e22cc7fcc1076a3f9d130
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.54.gate_up_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.54.gate_up_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e66a8cc81363ee5c5801f52c1271065920e63c83
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.54.gate_up_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a95d761aabbd91a08cf2a68fc0802e631f4b2294145c5785469d612141614a5e
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.54.gate_up_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.54.gate_up_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.54.gate_up_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.54.input_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.54.input_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..970b24817141e20a2a830a9faf9e85d8d01e7f29
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.54.input_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dc66f979b84da64a3c0a19096a15f17842f344f24c60b6cff822979a506d2944
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.54.input_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.54.input_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..43dd228e0af68fb970d85cb1316d1f99bc0d3964
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.54.input_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1f56a546642f27aca498998cca7ef3b095b81842fda64cf2aaf5539ba2dd5c4a
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.54.input_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.54.input_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..09a17494f607c8af311d1ae8adb18be5cca44d98
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.54.input_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eddde014a355e237b18e40cc45cec0b560c0abe616ed8608d358eb0d9befc7e3
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.54.input_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.54.input_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.54.input_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.54.input_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.54.input_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ae9933d002b5fa875c3bf62dd7ed0e378450d171
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.54.input_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8ff82c327261bdeb18e3ca504df76df3ee9c9cc5574e92139c096e597765f468
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.54.input_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.54.input_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9a17bcf48bcc7e1e062a37377a71288b19093f18
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.54.input_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d0f2668f3adfe305b7255d24fd4fff2c5d5305c2e682b541e88acd6a0ae3086d
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.54.input_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.54.input_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2bce75244a5d83ced26b629fc35448725396f33d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.54.input_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8bb57d41402057e1cd6ed98641a459ddad7d413a539de3ea0a0d6700ac4cbc45
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.54.input_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.54.input_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.54.input_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.54.mlp.down_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.54.mlp.down_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c30c6129044dc3a22550751dfe64d778204b37ae
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.54.mlp.down_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f89166d2ff150ff04b08abe418a8d99e0852d68203fef61e0d88c6630ef14574
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.54.mlp.down_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.54.mlp.down_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..bded8026da4d60319ea6431ac0de1ca479773f17
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.54.mlp.down_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4ecc848ab1a0ced4c36cb8ed01b4f66521fae3c2c5f6a0d90fab1a3a1fff0cbe
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.54.mlp.down_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.54.mlp.down_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..349eef390a56a62c9bb8fd3faf57daaff797e320
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.54.mlp.down_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:606329c015901e7d37a2b638a8d04e9db706897b0ccb927c8dde3fed17ab1161
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.54.mlp.down_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.54.mlp.down_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.54.mlp.down_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.54.mlp.gate_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.54.mlp.gate_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7371ed2b62743eb17759a888464179141f19cb6c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.54.mlp.gate_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ab7647bf2c7093dd1e76d893e155efb1763122af175d9f5f656749c3024d5a5c
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.54.mlp.gate_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.54.mlp.gate_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..892284dde493b5c2ef738f7745402fd1c8fd9b89
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.54.mlp.gate_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6e12b044a1b73bf34dbec75bce369927db2aefd7c28e9e756bcc7b19b3767904
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.54.mlp.gate_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.54.mlp.gate_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d67c84c3690874d3aa480912099dd434af699fec
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.54.mlp.gate_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:32a5262abbce22f69d8ab4b7b61cc0a055f9c60a2b079e051cddc1444d9bb956
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.54.mlp.gate_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.54.mlp.gate_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.54.mlp.gate_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.54.mlp.up_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.54.mlp.up_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..aaa83eee1225d423230a1cdd553e681b7430d392
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.54.mlp.up_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1234ac4c6ab404a3c52142fc441104ad9826dfe519b737b0088599c61ca2d701
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.54.mlp.up_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.54.mlp.up_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c4a7ac80403994104f280b42a78b57113f5c27f9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.54.mlp.up_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9ba9b42a66d90c3da33cdcdce9f834cce3158cc6d65127c8120a597f656ce9ca
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.54.mlp.up_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.54.mlp.up_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..bc4b5802ec5972f2ccb070a6ef24e68455ab2dfa
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.54.mlp.up_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:43cc80ac551a3b0f127f86eac4c49dfe774aabad6ddeb525bdf2df1c0090af89
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.54.mlp.up_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.54.mlp.up_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.54.mlp.up_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.54.post_attention_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.54.post_attention_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0782ac81e3e8c0cb5e6d35540e830abee020b74d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.54.post_attention_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:299c2b7c39db52ff3e060359b4982e356ebc594bac4e955a9b05495cb14ab296
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.54.post_attention_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.54.post_attention_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f34e96b84200483ea7f6a76487a43a6a0b6072ca
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.54.post_attention_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:62b604ad535792e01e66007620a35942cbaf889c3b011e630748d972e13f020e
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.54.post_attention_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.54.post_attention_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..14cd5da8659b1e1d46bb6d4bc8b9f910f9191a6b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.54.post_attention_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a587418abd4502801beb8343cc7c497df37953d9ffcc00a3a8c4932c230a370d
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.54.post_attention_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.54.post_attention_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.54.post_attention_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.54.post_attention_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.54.post_attention_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..627ca40c8ce428b51b5c9245cd92c99a8ec8aa0e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.54.post_attention_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aa3ca4370722278740db697dfd41a3dc71573536e834e5f9a1d336c6db1aa685
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.54.post_attention_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.54.post_attention_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..05d39ce58ec073e00c279370978184bbd7b56adf
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.54.post_attention_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e75e802c55584e2bb9c34f1e6e0fe9e9a805a4a22886f6decf54610af4cbdf0a
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.54.post_attention_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.54.post_attention_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..480097557ac42d17b6689a8e2e5cd2a0a8b39397
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.54.post_attention_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d0a027f9e05807236b7b6e11cb3a9ff1fcbab2cd4cfd7c7d9085614ebec5c15b
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.54.post_attention_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.54.post_attention_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.54.post_attention_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.54.self_attn.k_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.54.self_attn.k_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..453e9b7b4d2d01704b05dd7668b7975c3c29f9fe
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.54.self_attn.k_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:608140873a753fd5144dbbf40bb9ec4b6c1d72167b62fc0fe450b8123a2fb4c9
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.54.self_attn.k_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.54.self_attn.k_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..23bf2fad3a46c0c3bc342b2fe46fb2aa782a1691
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.54.self_attn.k_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fb7659cd45a7913758e323ac4fd8d928babf451639de9bebc3c2bbeb95bcbb7f
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.54.self_attn.k_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.54.self_attn.k_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4f85c323e9d6752e0b643ac98b4454676c7bdcc8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.54.self_attn.k_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cbe048c31e1a6e4182ae88a21b6e234ad8fec4e1fd709853b2077960ac0d453d
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.54.self_attn.k_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.54.self_attn.k_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.54.self_attn.k_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.54.self_attn.k_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.54.self_attn.k_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..bae23eb88e38474cfbf79ecbba75a9e50c2f8a7e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.54.self_attn.k_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b46f29e1b4dc10d8dda68f0a8da5192418a7168586f4ca0784f1dd445d70510e
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.54.self_attn.k_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.54.self_attn.k_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..aa0b05e40230f685d0ddbd60700705ab81138b49
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.54.self_attn.k_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b19076bc1ef4fb379bb77908aa39a2e277cac4b631532af652a237d110a0334e
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.54.self_attn.k_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.54.self_attn.k_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2a8f5355819663aa45e234ace73de4a6bc4787fe
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.54.self_attn.k_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0ace6c0785b78529344ddce5eca4c84e3d9f5f511b217927dd729992285c423c
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.54.self_attn.k_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.54.self_attn.k_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.54.self_attn.k_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.54.self_attn.k_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.54.self_attn.k_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ed83d51d90619d50da80ccd8ca0483f0536f6259
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.54.self_attn.k_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7a0c91ae50c8a1cc6e47f2d55d166d067943482f599b9314422ed7d7faad0ffd
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.54.self_attn.k_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.54.self_attn.k_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7a6d77a792c720d0c1ea9c197f23d5d28f723101
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.54.self_attn.k_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:47fe10fa2b85782813aa96fd14f71e20502e7923559ddb095bbf0d0799ddae69
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.54.self_attn.k_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.54.self_attn.k_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3bdc6f0bd078c5f140b312783397598dfd35f9fa
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.54.self_attn.k_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:18f53a78474b557f8b3af9cd4856cf0c776997ddf516f03ff61b7ca30b5e4502
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.54.self_attn.k_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.54.self_attn.k_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.54.self_attn.k_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.54.self_attn.o_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.54.self_attn.o_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8717a3b3f09738203a8b9742c76efd82f9ccef62
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.54.self_attn.o_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b6bad14b6030af0017c2e53003a32563814c9fd0515f2b035494bf659aa2aa90
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.54.self_attn.o_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.54.self_attn.o_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1a4162668524c261d59cecfb1851b3110ee7fc11
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.54.self_attn.o_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef64d6695be7368c45c83eddd5ad1da25ac7c70dbac046ba0d17cd8a1bdd9e04
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.54.self_attn.o_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.54.self_attn.o_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..76ff0ef8f1a35d556e033abcbb06a3c7e109cf45
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.54.self_attn.o_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d683daac02ee9f168a4936e7f521e274bbdd9847593778a0ea6c2f8dff2de983
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.54.self_attn.o_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.54.self_attn.o_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.54.self_attn.o_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.54.self_attn.o_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.54.self_attn.o_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..94617d6c41bdd00040cc30e1f862f7360dcc8673
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.54.self_attn.o_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f8d5c24c5fa41266a0fc84623826c56c292d255b9b8064df71f665371873253e
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.54.self_attn.o_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.54.self_attn.o_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..80696a15e870da422fb948707ebce8f1b6b40584
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.54.self_attn.o_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9b42026f27838acecda04f9b3852e9826fa979e0a4cfb95df545cc08d6c25568
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.54.self_attn.o_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.54.self_attn.o_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f5426a22937716f6af3a7e9bf0a79410bae301df
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.54.self_attn.o_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd16c6ee82f151318de4c14e772937f780befabb5d83b8ab2938b5031bb6172d
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.54.self_attn.o_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.54.self_attn.o_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.54.self_attn.o_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.54.self_attn.q_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.54.self_attn.q_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..17df44a53882f54463e97c75b63b15378c7d586d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.54.self_attn.q_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d7c45299caeed19394b6e4a158a226391256f431ca834098a892d4953821ad47
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.54.self_attn.q_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.54.self_attn.q_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8539ee02447520d609260d9b2dacc5283e6b0362
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.54.self_attn.q_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:21bd17751054fd5fb2f333fbde8b4393551a899521e09635fc0e003f5b38e485
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.54.self_attn.q_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.54.self_attn.q_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2d5e77137b94709455fd710ec31d80098f142a40
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.54.self_attn.q_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b668acbc93806012bdac46168b135906139b8a0dfbf1024caa4434abf1945820
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.54.self_attn.q_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.54.self_attn.q_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.54.self_attn.q_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.54.self_attn.q_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.54.self_attn.q_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c7fe43f09ca40acc686674ee3ae3e409eda8d3e0
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.54.self_attn.q_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:80329b73dffa9889f5282b54870ae3fc148fb701795fdae62b264c4aceabe992
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.54.self_attn.q_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.54.self_attn.q_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b683237dc0ac3e3af2c69be57afb4f6595ff10e4
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.54.self_attn.q_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7adfbd8f9e53f4d8277eb102e6bd1502bcc0d1efaad5ce61fc7edca800d97130
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.54.self_attn.q_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.54.self_attn.q_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2ac6504b07c68fa376784ca1524483b62fcf9cee
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.54.self_attn.q_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:21d665b7715a13bf1a502b71cd4f79e820672d882c176c0d2f82352422842849
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.54.self_attn.q_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.54.self_attn.q_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.54.self_attn.q_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.54.self_attn.q_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.54.self_attn.q_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0bf75795797bc728d6f2fbd3c80f149e9382f377
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.54.self_attn.q_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b655c00979871285695d432721177176470b107a0e242f4a4fea3f75b1bd6045
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.54.self_attn.q_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.54.self_attn.q_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d37222f57a21e3e18555e665dbce7016ec023886
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.54.self_attn.q_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c3ab7c922130a3791a759a88c3cec55f3f873911ca03d4aa11b0c3d38b4c1342
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.54.self_attn.q_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.54.self_attn.q_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9b7ce829c9528fccf3558c8ba39a01ecaa99e4ee
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.54.self_attn.q_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2d4d3d12e466679fa42ebfc9a73111d13610f35bac8c686623992c777d943576
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.54.self_attn.q_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.54.self_attn.q_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.54.self_attn.q_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.54.self_attn.v_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.54.self_attn.v_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e78cf777cdcbf230c047e26e45b6d746091db84f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.54.self_attn.v_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a8fa1dceb463af0b54a7c9fac85d139a0217855560b711ad4b7f1e2a73a1c027
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.54.self_attn.v_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.54.self_attn.v_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..84fe66f8e2b0397975727ad7959fe69e6991c78e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.54.self_attn.v_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:937a2304b614e012dcfc8e1a4594b27274cd3200d21061f0999e819bcda7904b
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.54.self_attn.v_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.54.self_attn.v_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cbb76b2028511dbc03208caf1be249a5d6e6523c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.54.self_attn.v_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:edaf61ba26e356256bec81d4d7647645903e3dba2c1982ab6b34f0b1d8541501
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.54.self_attn.v_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.54.self_attn.v_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.54.self_attn.v_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.54.self_attn.v_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.54.self_attn.v_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3e8d522d14b553b96989618fe9fab465d69add4b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.54.self_attn.v_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2aec5d06796d240adb34c0c271283feb5d5824d5323de3cc09c4ce3e512983c6
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.54.self_attn.v_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.54.self_attn.v_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c86f16f23dfc42180c0c514f54093d064b8109d7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.54.self_attn.v_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c2bbec5c58ace4fbc4c3c0905b6ece035eda9f3d3c368d6727d9551ae2a84fc5
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.54.self_attn.v_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.54.self_attn.v_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a29a79d7488a0f5b73696cd58f48385100b50ce4
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.54.self_attn.v_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:11d3a901a56bbb20aa4ac8d528301f88be7c53101f4fb64ef7bb1ebedfd17dfc
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.54.self_attn.v_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.54.self_attn.v_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.54.self_attn.v_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.54.self_attn.v_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.54.self_attn.v_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e1cb5c6b6c5cc9d73deb96682a0120e7b5988f49
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.54.self_attn.v_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9cb2c8a4a22abcddf65d9eb01fd92a49c4c4765238b094554f7a530de38237f7
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.54.self_attn.v_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.54.self_attn.v_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6679b18b99f570b999c4f683fbea5be5d3a1c84c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.54.self_attn.v_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cee7e47375ac55ced2e3b000c241cadd5d8da44211525d03cbfa3c625cceb2ab
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.54.self_attn.v_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.54.self_attn.v_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6071fab563b79f979e62411484d3cf2571c42080
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.54.self_attn.v_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e678b0adc07c7adeaf4e52749b17b5d039af1d7804e7d0070911c5a445a3a14
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.54.self_attn.v_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.54.self_attn.v_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.54.self_attn.v_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.55.down_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.55.down_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7d32cc6398d8ae03f5bee7ed979fe64e8d652152
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.55.down_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7307961841558a30814eb8012145e30a3b88ae16ffef351074e9db6fe7ff6401
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.55.down_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.55.down_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..280d7c0c3db71c9bb72e8b1e95124aa060742031
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.55.down_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e7fbfb85d0284e5e8117168c6fb26e24ddb0c8ad871131b5871fba982377563
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.55.down_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.55.down_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..214fd359c1f8a428e508bbb1057fe5dd725cf50e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.55.down_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dc98a4dbd03a4cee22e3ae48d485f020844ea5f26b9294d1c5b05cec91958c8a
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.55.down_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.55.down_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.55.down_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.55.gate_up_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.55.gate_up_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c17f932a6ed3f4f175580ad7a0f58f04a9ec1081
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.55.gate_up_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4c4481fb6252ece7c44b771aec8ad0ba95076466f5f22823fabcfe5a7724846f
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.55.gate_up_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.55.gate_up_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f1259b5d0934831c837ab8075731e76eeda932c1
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.55.gate_up_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:99dbe7e86027aef484a7b37bf72700248d3d2927c1c8a3603859d73f9d52e1d5
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.55.gate_up_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.55.gate_up_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a81ecbe41e50995c758f013b82c5388f8b1d6f0f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.55.gate_up_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cf3c35fb3aafee9dfbe6ad42479aa916fc34d65820c60471e9e234662fb28f07
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.55.gate_up_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.55.gate_up_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.55.gate_up_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.55.input_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.55.input_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..dfc0c0817497c33009af048140275f8c21912745
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.55.input_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ae707c32788d6298a32cdcd2879edd34c60570baf12d9baf69ed1f79128d34d3
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.55.input_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.55.input_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4f858d21fd0ec1e2a68c515c81011de1cd07bacb
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.55.input_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b5cb4f4ad2f55503756d87538093e47bfe3dd3c5d4ec0fdb5b365b472c22d5da
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.55.input_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.55.input_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..84c9226e8bec434440462254b70e0cb227b0c7f4
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.55.input_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:28f490621a8de304a511219631446f4d0e06c61233aef2cc61b52fc79e524d1f
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.55.input_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.55.input_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.55.input_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.55.input_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.55.input_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ff4c1b4243c6f58f845a4e8b632c3f62a68382a7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.55.input_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ed9a171bfbbb2c611ebedba08db37427245ece29266964c96aee6cd9e485d75a
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.55.input_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.55.input_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b2be2414e3d7064e26c4d35b8122c61156f9dd5c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.55.input_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:77d8d07db1feca533d21fadeba8087b5991885576ab2a3fc3289c0ec43230059
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.55.input_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.55.input_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b01cdbe1c2e1c2c53c871e03cf3b6494eaa16057
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.55.input_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:db9bc5f21f0be3e4083ad79e54ba16065cc3abcaab36aedf69091006553517b1
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.55.input_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.55.input_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.55.input_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.55.mlp.down_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.55.mlp.down_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0818ac53d5d8ad13867da978af343eef723c7a7c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.55.mlp.down_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:584198dd5869693d0d3816a66ad2bd872454382577a54ff320560717e3c02fcd
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.55.mlp.down_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.55.mlp.down_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a1a1507e268a1862edff676dad8f8e6256f28236
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.55.mlp.down_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a694c195dffd2bb40b02164f880054a87b4b409ffb85d7d33b007581c4953823
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.55.mlp.down_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.55.mlp.down_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..76503fa29800448a33aa97e52d2e1cb3e0301d63
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.55.mlp.down_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:30e0cfabf3f36ea237890a6a35007be10618d4148cdfcd4b439754a96728f7d9
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.55.mlp.down_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.55.mlp.down_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.55.mlp.down_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.55.mlp.gate_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.55.mlp.gate_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c01399099c384778f5a7b09fd341ebfc3fbd68ca
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.55.mlp.gate_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:990c41ddc6bd8d54e817b846b86ccc3b4c9bcc1ad568ea74d9b0566b2ce37d24
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.55.mlp.gate_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.55.mlp.gate_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..767632d388c81a9332d860b9e1deec57b9e9f6ad
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.55.mlp.gate_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e4020202ce68a211eac7fa0d6c038ce1f13a660dd48a69adf4d30da0bdeaadba
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.55.mlp.gate_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.55.mlp.gate_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..73668a8c7bcd552b36e41e175f89d9a8a64ecab7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.55.mlp.gate_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b310f4753c2377a4305d5ff080a71f0347e1a58756e57774cbea558f2ce9f6f
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.55.mlp.gate_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.55.mlp.gate_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.55.mlp.gate_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.55.mlp.up_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.55.mlp.up_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..00a23938432e09662b7fde61892a24e28659b946
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.55.mlp.up_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b9215a6bd8eada62b2ac67730cf76c685cace76bc8845fc2353101c8e98d4f66
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.55.mlp.up_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.55.mlp.up_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6cafba3c2a90567e47853b4863a03040e190973b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.55.mlp.up_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:686f59027cb14e6baf0998139996de72d6a2638f2aebe8ecb7f71ade18009cee
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.55.mlp.up_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.55.mlp.up_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..660aa7aa1fee2b0d92aea1b09eef35e5e47ca1d6
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.55.mlp.up_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:687cf43c448194cd25abf4ddb456133205ea14fedd8450405e5bbb12baee93f4
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.55.mlp.up_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.55.mlp.up_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.55.mlp.up_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.55.post_attention_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.55.post_attention_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fcb9b0538396e8f994f71a871e4aa8f4c7605fd5
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.55.post_attention_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3901799a83abed3184ac3f3abf2b1d4f5454bbc9999d97fa52d4f03427e4fbc2
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.55.post_attention_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.55.post_attention_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b3812c0fdff65f3505c210c02322b051f4d4473d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.55.post_attention_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5294ed040100f1c19534c702257305856d4f45540d5c69e5c47278c2b24c2dd9
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.55.post_attention_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.55.post_attention_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..39cb77c217592802b15e2d06d1aca06ee2b50507
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.55.post_attention_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5b17debc215d1ec464a12adcff7c341df576d1f83980d757c8e9175c8ee80854
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.55.post_attention_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.55.post_attention_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.55.post_attention_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.55.post_attention_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.55.post_attention_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..170199951a36a5cd104b1c5b00fed3dba0d8ec07
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.55.post_attention_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:91ae869d5b4923605eee7d452313815376e530f668ac548778f9bc4c11b643a7
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.55.post_attention_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.55.post_attention_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8d055e27437c34107dcc0294c8cdd65d587003f2
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.55.post_attention_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:40e29b9e73fb68f997d9d00ae24ce617dfa8184a83857fdb4d11225358b2d579
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.55.post_attention_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.55.post_attention_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7c3fdc36a2bfc0614fcd1d6c5734f151b3b01211
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.55.post_attention_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b163bb487f9132349e261326c58807d33da909e135aa73d911c494cc0b470dcd
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.55.post_attention_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.55.post_attention_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.55.post_attention_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.55.self_attn.k_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.55.self_attn.k_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3936f2381ffa828716dea429070feeff3079bfdf
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.55.self_attn.k_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1cd016c7542905b54d32db091ad34cab9f651751394f57ed643777328800bbf2
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.55.self_attn.k_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.55.self_attn.k_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3eb1887f61907ebcc6c0ec3f8fbe30db0ac74a68
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.55.self_attn.k_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1f01e73d2338e4aca1e461a4e1315a56b8e316e8fa6a9dd76e185e1606333d9f
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.55.self_attn.k_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.55.self_attn.k_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6b2a3e4dddefccac0874c6cc824e0fd71884fd43
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.55.self_attn.k_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cbd31c9abfc147854e0f34e6621f4a3f2550520684d11b003173349f9826bc97
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.55.self_attn.k_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.55.self_attn.k_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.55.self_attn.k_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.55.self_attn.k_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.55.self_attn.k_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..39c44b9fac6aa2a53ec6ef73cec863c941c04ad4
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.55.self_attn.k_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7cfec59a48fa2841f3925518a9744712a62be58110ba010edf59864922043c14
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.55.self_attn.k_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.55.self_attn.k_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d190470daadced885e0a4fd54823b9b6bad57ad1
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.55.self_attn.k_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f23b6a64fc003c0e29df4468b4b7514028bec2b7859f4845ae6a4f067bb2671e
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.55.self_attn.k_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.55.self_attn.k_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..63f971fc2d98091be640b910ac4259e249079bc2
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.55.self_attn.k_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ae0f09fdb13e4de5da6d261b98d491e4fc2cdca9c1e448ad112cc5ae85ad28a7
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.55.self_attn.k_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.55.self_attn.k_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.55.self_attn.k_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.55.self_attn.k_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.55.self_attn.k_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a03c542b03d0247996c7097a7c52d44d40ba08b4
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.55.self_attn.k_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ae3f3387ff0cbc87ae52023044f5c68cb48af1b5e8898a09a0778c392d63dd7d
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.55.self_attn.k_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.55.self_attn.k_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ca59e8393b8f2714ddc7d04519a28a6e2baf182a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.55.self_attn.k_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:da82b7417c233b2ef2b5bd82c950148a169869dcdc57209fc09b83cc2c2aa78b
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.55.self_attn.k_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.55.self_attn.k_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7363881e6659e5371c4ec13283b4e71f831fa215
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.55.self_attn.k_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:44cfc6391365c346bdef93f9c784196a6997ce3b3f4f5a9dc0590c9faedccded
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.55.self_attn.k_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.55.self_attn.k_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.55.self_attn.k_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.55.self_attn.o_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.55.self_attn.o_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0e40d152501b62e506bd844d3e51f57f4f1abb58
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.55.self_attn.o_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b35a63956a95dd46460c9f16de9b5ae9c0f045d2058f1e4566a9a6222e338953
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.55.self_attn.o_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.55.self_attn.o_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ebbaacad442488e04b77a3e64a22cad2c6ff6607
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.55.self_attn.o_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:098fb58987d89da58d62f5572f7fa6f1933d5f949af14d8df440491e2a47994b
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.55.self_attn.o_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.55.self_attn.o_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..62c9ab7b6c3f4f4357398e1313caaa5a9ec4ef44
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.55.self_attn.o_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e0ebb61f07153eb8d457a3edcde03b8cb7587ebece3717550dee9a6fd656be1b
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.55.self_attn.o_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.55.self_attn.o_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.55.self_attn.o_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.55.self_attn.o_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.55.self_attn.o_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3b4aa611530bde42ef8e21826c08409a041476c7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.55.self_attn.o_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e682de511d4720b54ca161dcbbeb475c1d7d65d34e8c4ebae9e0e438bc137ce5
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.55.self_attn.o_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.55.self_attn.o_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..052a08de7124bcc299bb4bf095cf7c1f89ca14cb
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.55.self_attn.o_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:acae4f9fbf154dedd8c0ad8aee5c7865bb53125a2e5968765c90ea28c000afde
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.55.self_attn.o_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.55.self_attn.o_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..18577bd494a22dcc270de74c9b566f93dbdb3625
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.55.self_attn.o_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:82c7ae7a3aeba5330baa0d49912e4714e6c5433053cd2fd13832cb295fd602b6
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.55.self_attn.o_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.55.self_attn.o_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.55.self_attn.o_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.55.self_attn.q_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.55.self_attn.q_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..659e66471f87db9ad752755f286140d278d48ce7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.55.self_attn.q_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d07637bb309b36c0e5eec4760aac75bfe8fdee3f4f47636c169e03e4e8577f04
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.55.self_attn.q_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.55.self_attn.q_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f800bea1c0a3592082f5a0e73fee2a0cb7e3ce8e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.55.self_attn.q_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bb3b1551e1ba84bbb94211c5b6dec98bae88510acea578332baef2dc3dcf3791
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.55.self_attn.q_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.55.self_attn.q_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..921d745d141a4d25a51949a77afae3cd1fc81614
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.55.self_attn.q_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:711ff20592b03504441c4a8bdeb8f5d45ab6d37929b490c88311aa61c487b375
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.55.self_attn.q_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.55.self_attn.q_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.55.self_attn.q_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.55.self_attn.q_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.55.self_attn.q_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2baf57e73b947b31eb41d64b76f8d65323d2f740
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.55.self_attn.q_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5f4fb777edaecee900e459b4d59fe6a037493059bf09671f3cc820bae39c29f7
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.55.self_attn.q_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.55.self_attn.q_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d69648038879ce84413c0c5c28f3a29bddaf1ab9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.55.self_attn.q_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7e5f292ea60929f8058c15683a363b6acbf01305e8042372ada3b52e0b341b3c
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.55.self_attn.q_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.55.self_attn.q_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..71d03fac859800560bee3c48118de0c7980b9936
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.55.self_attn.q_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:354d5be40144209322588c8e9b1df5ff66a78fa437665ac3a0497a44d3555bde
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.55.self_attn.q_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.55.self_attn.q_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.55.self_attn.q_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.55.self_attn.q_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.55.self_attn.q_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..292cd83b463d572968fedaea817d2323a697db00
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.55.self_attn.q_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:67a2055c4ce2ab5813b886cd81953d92daea7c37e1fc90c71b411deb9e5b8934
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.55.self_attn.q_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.55.self_attn.q_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d5ea9766322b6cae0ad1950ba0e302042ba14920
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.55.self_attn.q_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7cd21cba2e2efa218f77b26c0f5067cbf2f9b507c45afa6682b2a99882ed8889
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.55.self_attn.q_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.55.self_attn.q_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..42ab5c1f7f0b7271efd6b509d78c9c0c0fedf19c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.55.self_attn.q_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:34b94ea9aa7fe6581f7416349a668aa58c52c40ba1760a9562a11a8a26a060c9
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.55.self_attn.q_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.55.self_attn.q_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.55.self_attn.q_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.55.self_attn.v_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.55.self_attn.v_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c6c80841bd8f902cb853290af425f0050198f19b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.55.self_attn.v_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3938362361a34326e864b115b3dbc1dd2fe6a64c64d7c197bf2cfcd9119067ca
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.55.self_attn.v_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.55.self_attn.v_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9ed75dd415c28cd871ede17b31df98d8fcfbfba5
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.55.self_attn.v_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b56765f252cc51d3e5ec57d071783ba2b63d8d14d1bf50aa9f7c1acc149d523a
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.55.self_attn.v_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.55.self_attn.v_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0c2fb2db1c1a36a58e5aa72f70fac813684f0ec4
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.55.self_attn.v_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:585972ba427a19ef5b32e116231e2c810c5bb2f19079613b45dbaa5faeb7c9da
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.55.self_attn.v_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.55.self_attn.v_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.55.self_attn.v_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.55.self_attn.v_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.55.self_attn.v_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f22cf95303ac7d68e0cf09721e17d3d26296c21e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.55.self_attn.v_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:104dcdbf3ad03cd555a7c6dd5f63fdf5b8e4ffb9bcccabc197e3445ffd8a2675
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.55.self_attn.v_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.55.self_attn.v_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2a3e26be8330210bc790add1ecf4821cd0f0db4e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.55.self_attn.v_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9cf372c57f80a097250ac5196a55ff4f7fe92c2cd6a1a5cd1395aa86d2862890
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.55.self_attn.v_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.55.self_attn.v_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3b46b722cc81da1124c9897f2c6f4750ad70244b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.55.self_attn.v_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9340a26f7236615ee8a1b9a4ae419fb2987a1e7041dfb9966c09c2f54121bf8c
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.55.self_attn.v_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.55.self_attn.v_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.55.self_attn.v_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.55.self_attn.v_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.55.self_attn.v_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8e8ca6905daa45ed852e92dfe3d042ff0a8b3dad
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.55.self_attn.v_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:397b38f2df4efb377edd97f52ff7d33fa0fce77f3d6b7e1e7de1df1ad57781ad
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.55.self_attn.v_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.55.self_attn.v_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4401c3f7fd3ed89f61a59213468b3474784793fe
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.55.self_attn.v_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:06e38b94b8d542318bd26942ecee58d55535e16b8a46514d9bbbe3a9f5b3c1bf
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.55.self_attn.v_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.55.self_attn.v_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ada82e34b1a53bd4909e724a14f343e10e7f48b4
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.55.self_attn.v_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5d0240eadd3d23338c352c54ff356b5efbc7974e6c004b2646bc9991970621d4
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.55.self_attn.v_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.55.self_attn.v_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.55.self_attn.v_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.6.down_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.6.down_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c31e56920e97b958c25c4526102333bb44711fa8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.6.down_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d35f00dec0f86a2757551133659e9a4b02ffb633bd93fb882790433a078ebed4
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.6.down_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.6.down_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9c6451476c8719ea63b7198ff2b7125556973d4e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.6.down_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:73f13789f09afd001157bafd08da495d67960931f7c803590e7fbd846269a2db
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.6.down_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.6.down_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..df52f28d4aa2d878cc6b4fba16c74e983e615347
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.6.down_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6720b2ba753eea1a1de5aad58a548deea1a25db333362bc8091700c4dd7bd132
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.6.down_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.6.down_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.6.down_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.6.gate_up_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.6.gate_up_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..55bf813311f3655f5b0ba429bb6357cd57b182f0
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.6.gate_up_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c37eff1b868a3dd83031e13575d5b0c2c42320359fdb70ff4bca72e6c7ed749d
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.6.gate_up_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.6.gate_up_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..46846121cfcc5c1c34c58c839b3557dfa9255d55
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.6.gate_up_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cd26af43eb7812ab40472218ce19e616ba9a790e23c2468f89f3dd421c9a3f8c
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.6.gate_up_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.6.gate_up_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..33cfb9e4a109e6a67996cb2db727f41dbdb8847c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.6.gate_up_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1f22ff69b8c3d67d9a411051ea3baca5bfcb923e294a7d2e023544084ef97571
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.6.gate_up_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.6.gate_up_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.6.gate_up_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.6.input_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.6.input_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c5d31cf37e7dc71be4935aed75e6f36f4d748273
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.6.input_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1e2fc9f311154eeb09007857b7a572518f3a66ce49ffd07a0a2647a35a2ffb2d
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.6.input_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.6.input_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7a6d81f5cae01c24426ff504e78920336272c3e9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.6.input_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6a6ad006a18584432843d941f3a05ce7af7856ab7a9e0dab586833f819f55eda
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.6.input_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.6.input_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..13709d2da7616473fbf3c9f34c9236bbd0e9a665
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.6.input_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1458d9051c513dd37a9ad4370740a1db8d141836d4c3ef8ab5808d9f15f0d445
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.6.input_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.6.input_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.6.input_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.6.input_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.6.input_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2902d22c68197191b0ada160e763f9687bece6c7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.6.input_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61c492b232da4db94eb9c98cae0952674f5186db88ff73902a210bfd01f82378
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.6.input_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.6.input_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ff1301e257a37335eeb84cba9f9f8ea64f93507e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.6.input_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5bee98551d3403cd65b4331c53befd82219fe11a960513c3c0c422953eaebfa8
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.6.input_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.6.input_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8355eab237d32a9450423d8ec9731331e88732f6
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.6.input_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a866c4283cfe59f3ff078f0a7c1103cd9d03116dd4b4230fa793bc059ea17483
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.6.input_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.6.input_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.6.input_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.6.mlp.down_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.6.mlp.down_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9866bfcbab91157a0021fd3d7a32463af5835cb7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.6.mlp.down_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4d8fb759c3dc36adb818fea4cd4a9c7ebca37b9256b9938a28767ace9621e795
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.6.mlp.down_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.6.mlp.down_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c2b7a83af35e0af245117dbacb975eaa9705fad3
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.6.mlp.down_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:70ec9eccd9acb281c40efe9eb09561ec6ffe8ed884ae183e345f04bc4c301269
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.6.mlp.down_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.6.mlp.down_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4b384be0db5d06926447ce2de9e95776a713ecfd
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.6.mlp.down_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:177e2ae116c864fc279593afdc5e176543b79762f93e8c4cf0c478d3acc7f834
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.6.mlp.down_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.6.mlp.down_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.6.mlp.down_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.6.mlp.gate_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.6.mlp.gate_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2b5e5ab9d834841fe842ade6341e9f3b1355742d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.6.mlp.gate_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:38a5f97e717484cc532aafed87fbac262ab15de6750e73522a01760897a45799
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.6.mlp.gate_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.6.mlp.gate_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cc8e97adef485e0a7e4dc13d1de7bf96f875c27d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.6.mlp.gate_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:904867b7723293e6610bb5393db14ad1e2df0705ca8757fa629394bb978824d7
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.6.mlp.gate_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.6.mlp.gate_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..56e569c7b8881f566ae34f44a91aae4899ac89c2
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.6.mlp.gate_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cd930c7d5b8486efd93215923e548a10f2a46521ecfafb9c132c040da81a13c7
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.6.mlp.gate_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.6.mlp.gate_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.6.mlp.gate_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.6.mlp.up_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.6.mlp.up_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..777b4915944601d0dc6ab7889db2c3ad62272726
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.6.mlp.up_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d5631bbcbe87bf22833ea0b019c71f95befa448e612e021eaffa75d8f35c6501
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.6.mlp.up_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.6.mlp.up_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7529618d7ebcc1dfea0bcc82b288b221fab84747
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.6.mlp.up_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0b1ec0cac268b1702cc9a043aee9fb3b8f1176f8dc6e0663870bbbf8dddd468f
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.6.mlp.up_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.6.mlp.up_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..816e14d98332d83ea48c5e1b72ec4a2c35b7309c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.6.mlp.up_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c389678ca042782865901e865bc5ff5d1327a3988b6c5d4505bb190ed458da83
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.6.mlp.up_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.6.mlp.up_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.6.mlp.up_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.6.post_attention_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.6.post_attention_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..479f4f6bfb6046c5dcd1b8da56c95d13da9e35f0
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.6.post_attention_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2c6f3eb4e842becfa79243a27f42feea364fbe6fb932435853d2edff5aeafc86
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.6.post_attention_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.6.post_attention_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e9bc955606f0a822a7805d14718f69064debb10c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.6.post_attention_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c65805b1b7984792b8a2888e7e7378a920b3673dc9eed4ae9ed156451bb86cbf
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.6.post_attention_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.6.post_attention_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d2f7f8a672edb4b0d7bdb3cbfd49551125c132f1
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.6.post_attention_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d75221106f0149e5cbca2307f9a05b9ef93800bde6b7ec9eb2cc980c5a399013
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.6.post_attention_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.6.post_attention_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.6.post_attention_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.6.post_attention_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.6.post_attention_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1219a9b76fe2c259c785e83f4a39273abbd90080
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.6.post_attention_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bf5ac1b31bad2f473e6b34c7fd62a3c4f0ee1130e11f7a0b10d1c648e8d1060b
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.6.post_attention_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.6.post_attention_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..77e458eb760983dd1fc36e09af9ade8d2066fb06
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.6.post_attention_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cd39339467cc93919315840b88a7b8c1ba65d1207e9f326e1a44eba514aa1f63
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.6.post_attention_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.6.post_attention_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1aa5ed40511acad2858ee364b72f96525b0731df
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.6.post_attention_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5d3e70995d38da4718bab30f8696a62421f52bcaa06a6bdccf4bf4156b6254d1
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.6.post_attention_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.6.post_attention_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.6.post_attention_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.6.self_attn.k_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.6.self_attn.k_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..56ed58d268a0e288468de834a65cb4265062e5a2
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.6.self_attn.k_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f3695df43bd8784bcb8798e24c422bef62b6c941134779cc8e9313b537402bac
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.6.self_attn.k_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.6.self_attn.k_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..311ad0a117059aa4ce7f3a22205c75b298edf023
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.6.self_attn.k_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3e0a3996cfae5f394dbf799edaa4e96076500d8b15253cbdf4d6ee06124a87cc
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.6.self_attn.k_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.6.self_attn.k_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..04e154806f6e26eb518e176872263fe249192bca
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.6.self_attn.k_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7c5bea4ad996f075657c1b9e188c624d7408455a4bf66fcef06d68accdc8ecd7
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.6.self_attn.k_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.6.self_attn.k_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.6.self_attn.k_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.6.self_attn.k_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.6.self_attn.k_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9a142d7434e8cd943ee1054adea18ac0b2dda82b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.6.self_attn.k_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9ae4f2957a25997dbfcaeedf3bf4989cd85e608a96ff0f6c87bce8f0c116bd7a
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.6.self_attn.k_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.6.self_attn.k_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1b59a118046cb8d99b669af57173299cb41eaab2
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.6.self_attn.k_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1735f23b0a24df4f6cda15e29df15f08e56d5095c1daa9565718b341add78796
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.6.self_attn.k_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.6.self_attn.k_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e6fe0998e0e852774815ee2a1b4289d2ed839a48
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.6.self_attn.k_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:31dff61831441b5fe6edb5658fdb1853ce7c76d43ef788eea6147091de255917
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.6.self_attn.k_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.6.self_attn.k_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.6.self_attn.k_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.6.self_attn.k_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.6.self_attn.k_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b926c9fb9f9b986448c2dd33f740df836334907b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.6.self_attn.k_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3445bc3b66443a32798b33ba43d9c0aad884f7990b597589f10c710e96891c40
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.6.self_attn.k_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.6.self_attn.k_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..05a6a44873c371a7054684e1b423b498e07f290f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.6.self_attn.k_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3726e55315aba2ddda49f1e0a11c0df2053fc25bc6e432b833cf4e2e58cc00cf
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.6.self_attn.k_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.6.self_attn.k_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..01d574c308b53d126df0ba126ed28cba2212cd80
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.6.self_attn.k_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:babc9deab6ca26f74733cc7ae7b5ff66eb0d8df8d6c14d70f4aaa470fadc6848
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.6.self_attn.k_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.6.self_attn.k_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.6.self_attn.k_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.6.self_attn.o_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.6.self_attn.o_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..807ebf1d3c52d0a59b0cfa92029b76d750fa0bac
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.6.self_attn.o_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cf04490e57078bb4028f36912ed91465b8f898cbcfe77f6e597153552f73a688
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.6.self_attn.o_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.6.self_attn.o_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..62bf4cc3fe401d7170620f11235693a7f71eb6e1
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.6.self_attn.o_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:647ad501b5fd2ce265bd4633fbc4fdca6341e5bd7edb6760286b2247bb9b371a
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.6.self_attn.o_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.6.self_attn.o_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..73cc60baf7a55fb43898ff0de1ffa6e03fb3a650
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.6.self_attn.o_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:00c5a94b737eb91fbead830b1d4cda923153909fdb473f4b0989ff6a67603f4f
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.6.self_attn.o_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.6.self_attn.o_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.6.self_attn.o_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.6.self_attn.o_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.6.self_attn.o_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65f4256f6b9a25ac6dd92a980133936ed40ba46b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.6.self_attn.o_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:03d4a4f7a559451035195c1e017eb564666f1958cbcbaf17ee504203a58ec7c0
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.6.self_attn.o_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.6.self_attn.o_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3f7cf71c0de084cf47821943e97168334bfbce50
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.6.self_attn.o_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c6f407201c88e4b65b6a4ae72992b661e4a007093cea746d37deab2eab7b2fd4
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.6.self_attn.o_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.6.self_attn.o_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..687599527b2fb1395ebf27448d471322c000c3a5
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.6.self_attn.o_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:39e6764e03d8bbc8eb418cfc5c35497d470e861bf2c96082376576f346b6688a
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.6.self_attn.o_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.6.self_attn.o_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.6.self_attn.o_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.6.self_attn.q_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.6.self_attn.q_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3f41e10946f8a7c12063ac6920267768ef08a187
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.6.self_attn.q_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6f4ed8b51736a68081673aab2401a642e629ecbc80b0b3d60c3334e14ae515b1
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.6.self_attn.q_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.6.self_attn.q_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1858893f6fab4159714c725beca375ad96522d5c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.6.self_attn.q_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7be9f821783f082d6536b8da5a9a9eede68dd7cc4b83ddf1b73598f40b45f947
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.6.self_attn.q_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.6.self_attn.q_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1a744797577a8132661691c1550d176c1bb3fbc3
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.6.self_attn.q_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:851db7b61fb6fde7a6fa9df41440f760976c7d9b13b746be5af1afa93a64e97b
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.6.self_attn.q_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.6.self_attn.q_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.6.self_attn.q_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.6.self_attn.q_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.6.self_attn.q_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2208a54afc32616558d428ffc3d253a1d30a1f86
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.6.self_attn.q_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a5bfb4f63ade357113c57c43da21b482b9786cbbd38dd81dc7c915b9c459d4e7
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.6.self_attn.q_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.6.self_attn.q_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cca9834e68ecbfcada11978a60b8886b1d116656
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.6.self_attn.q_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2cfbec15bded61edc90e35ed2502e34e57e8e8e7890cc335d2127057274db6f9
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.6.self_attn.q_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.6.self_attn.q_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..20d8ba7d64837cbd967f5ae49a8b17cfb8db7419
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.6.self_attn.q_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2deec3b7442dd650d2a5b945a1712de0a0dc25bf4745d5e3cf34e5f783366d62
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.6.self_attn.q_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.6.self_attn.q_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.6.self_attn.q_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.6.self_attn.q_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.6.self_attn.q_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..151c3bb2db173559eb19cfda237fba0c96ab227a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.6.self_attn.q_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:79df7e14e4577e543e2f46affa1d78d52d4295d20a9de917ca926ce67ed4f487
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.6.self_attn.q_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.6.self_attn.q_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b8024f51b328b3b5a340463daeb57ae857202055
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.6.self_attn.q_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:11c9f17b3d2d11b210d12a7f6750ed5eaafad8cad8a126bdaf0d45a3e8fd7c5a
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.6.self_attn.q_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.6.self_attn.q_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8e31ac9aebd110d53e1033942819a4181a1a710f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.6.self_attn.q_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7a5f1583929dd1187def28cddaf320677bb442e05b416131da78a89380998f6c
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.6.self_attn.q_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.6.self_attn.q_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.6.self_attn.q_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.6.self_attn.v_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.6.self_attn.v_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3149ed983ca5bdf31273c224127672b44047024b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.6.self_attn.v_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5a08d8d7b3b672d013ebf37a4cf9c01b81f3b251bfa5e46e7db6775ae38009a2
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.6.self_attn.v_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.6.self_attn.v_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4f3610be7ab76e5dccdd485c021fbe9961ce94e1
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.6.self_attn.v_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a0233110420b980970609f9b973ecc8f7a2309891c4546edf6ab35fa06790efa
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.6.self_attn.v_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.6.self_attn.v_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..174fa06cd3ee1706e3617edec390f7edc049c344
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.6.self_attn.v_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:36fea3c5df403da300c06010a922b4284922c171fc2abe8ea822b41eaec1e760
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.6.self_attn.v_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.6.self_attn.v_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.6.self_attn.v_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.6.self_attn.v_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.6.self_attn.v_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..19c6ea811a2b018936911bb65d15ce8db271f9f9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.6.self_attn.v_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:82ec01ab6acdc6b208c1d3edf2fbaecf0c354171238a32403d031c55600fafde
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.6.self_attn.v_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.6.self_attn.v_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..14cfb00c7c6bdb269ee81374333983cf63e0861a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.6.self_attn.v_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:16d4ac7e7c184cbaf61d34ab1de537dc3f616c64e6969c586430b0f2f260eaad
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.6.self_attn.v_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.6.self_attn.v_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ae51dbdc7196aa6ff484c40f15be61b271aef2fd
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.6.self_attn.v_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2b0ff96099c34d62c86f75acda517c82dc9585340da9c1249f22c07795b6d761
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.6.self_attn.v_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.6.self_attn.v_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.6.self_attn.v_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.6.self_attn.v_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.6.self_attn.v_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2f83784e601115cdb49da37a9c8841a27c04cecc
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.6.self_attn.v_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fcee5bdc2abcde1de11891dd9f33076233916726cc668eda1f871fc92b456f45
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.6.self_attn.v_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.6.self_attn.v_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ac8514d70ca570272d60eeebe1f9de1a27d7e241
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.6.self_attn.v_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8aac0df1b9578affd38ff5c7686a1b3416a5771226ef14546771b0355113aeba
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.6.self_attn.v_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.6.self_attn.v_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3a21457248f2ceff7cd774c32697dbece0edd593
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.6.self_attn.v_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe7e8f209cb06ba270ecb79f0a84f57f352b3a2341c1cf0a242aad3fcdbdf243
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.6.self_attn.v_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.6.self_attn.v_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.6.self_attn.v_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.7.down_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.7.down_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..30565e798c1dee7984cdfbdd3becfe7a28f2781a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.7.down_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:afd26561bb9fdb3fbd39fd804405e288bed51c41d150ba65d90e9268369f70a0
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.7.down_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.7.down_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8a16f66900973b2bd2a915593ff58b00d34aa0b7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.7.down_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:98c3507f02ae8f55ebdfa768fa88e4be1a85ab8a000c61cac646d6a1c449112d
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.7.down_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.7.down_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..04903e3d11a37417288b8413d48ae56bfb8ce6c1
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.7.down_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:952d6fa496d752af3a3bfea7b47c63f01f584c78bc2d48195418d4d7a765e754
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.7.down_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.7.down_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.7.down_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.7.gate_up_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.7.gate_up_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d21d789991d2110d5a7397ed571a6d6d5ed3f6fb
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.7.gate_up_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cf9ba3c496a63ec09bacb1dcbeaf48104d006f40455ebece98cf0c7704b2771e
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.7.gate_up_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.7.gate_up_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1f5fdace57da317701ed560600febdb118ba5b14
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.7.gate_up_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f767fb5cacbc99b956d352b11a5236dfafb4000e8067098003c63229cb487208
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.7.gate_up_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.7.gate_up_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2cf0c32c5b950218bd337898f004dd2e62e65dd4
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.7.gate_up_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e77b862c007bea99378abc645754ec52e7b14d9d7a3d5dd4399333e548dff43e
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.7.gate_up_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.7.gate_up_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.7.gate_up_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.7.input_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.7.input_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e81dcce8fdd5cd50a3e5258b791c1ef0bf0710fb
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.7.input_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4bddc0e270b2cdb2f04210d01c8f2728a9117d93937aefeb52fb4eb1a65467a3
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.7.input_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.7.input_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..26a5921b276c7de615ff3d4e6a8c4053b476d388
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.7.input_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aa4c2862f603f4e025105d80660211949c2811518f8a05c83461b918093dd5cf
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.7.input_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.7.input_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d585f7e9e95985cf854041fcccb5d5a662caff83
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.7.input_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8bb8b36f1fa8add0cfc4155b8539877faab52954f1af13070b3545d70afadc32
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.7.input_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.7.input_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.7.input_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.7.input_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.7.input_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6d0486f97bf293d05b1bb4eae6f785137d628652
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.7.input_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:56a728d7380df78aa0cb3b5a3d34963f4dd8320bcdb586c1413ed87532dbf090
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.7.input_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.7.input_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8ce7cfb79159d01390a83197005455fc6bb334ce
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.7.input_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2b16e50c30ccf8fc240e9863f13b5486f2a16fde8142b3e3dca8568dad978518
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.7.input_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.7.input_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b4577b16a0ffc3a972928311af423993b29e842f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.7.input_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:79d9a2cf86d6d4b1522c2de7c1bbce0bf28889e65584f3183dd988387845f42c
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.7.input_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.7.input_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.7.input_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.7.mlp.down_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.7.mlp.down_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..45ff4a4c9eab080e2e9d6310ca9e893cfaa325c6
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.7.mlp.down_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b8e843fbcbc9d5bbc6ceb97143e7a812e612b2d36e9c909d6040be35bc5202aa
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.7.mlp.down_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.7.mlp.down_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..73f5ecd5f13d5d7a66d9ba4391ce6c0076609241
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.7.mlp.down_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cb4855eb46e1f89c0bad378c4dcf161e519e72089b6ce4441ad7338ae0080a08
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.7.mlp.down_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.7.mlp.down_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..748bb933e50aab4c2bdb2b18840c79d251710da4
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.7.mlp.down_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f76a3c36a20a6d2b739db416ad664cba5b0444602e3d08624a93be571c78d790
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.7.mlp.down_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.7.mlp.down_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.7.mlp.down_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.7.mlp.gate_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.7.mlp.gate_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6bb31c20347fd2d1cfebcbf6db81079de678b98f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.7.mlp.gate_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f5d49763340465efa3c862597dab3a8d733ce665e6dcfca7bf35830d42f00e79
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.7.mlp.gate_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.7.mlp.gate_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5fd03ab58651ecf3d1c0d8da5513effc6a7a35f6
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.7.mlp.gate_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:01dcf5aa8dd6932e6980de425902c3a26f069aba1ff9e45b96f89bcc14c1db48
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.7.mlp.gate_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.7.mlp.gate_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fa486a25c196f58ee19694896b34d41dc74b20f9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.7.mlp.gate_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9b7fcb0ff4bd186892cf3e1ee7a85c2a3aa1bba54aa39be087816e7f084984e3
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.7.mlp.gate_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.7.mlp.gate_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.7.mlp.gate_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.7.mlp.up_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.7.mlp.up_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9a6b69d39dceb28b19b2dc6282810eeb05e892d5
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.7.mlp.up_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:481e6d941a874444fa01bd21f8cea1439bd096bcd5c9b55d7142d8fa862bcc50
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.7.mlp.up_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.7.mlp.up_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..bb5beaed6772e420fcf7ebe3474271e32b4971b1
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.7.mlp.up_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:871e921a7f91021fa296de7d3da00e7676c487ffd98e560e7e3568be23002ea8
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.7.mlp.up_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.7.mlp.up_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7783902fb0f42a05b6f369a2bccdf1a2ddaf0adf
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.7.mlp.up_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4e30c83536d4b7871df6d0c9be0536b661109edc2b8b4fbcd2f37563e05291fb
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.7.mlp.up_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.7.mlp.up_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.7.mlp.up_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.7.post_attention_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.7.post_attention_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..91990af425419f1e2a58942ee775895c7b364942
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.7.post_attention_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7ddd01d837bc3deb6e004e3941ef48cd9066746a6bce16f5cbe9d00186333f40
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.7.post_attention_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.7.post_attention_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..78522cd1129103fc472ffcc47877781484d38f75
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.7.post_attention_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c99b46ab97d54f592cece0c6c748e17c3aeb9673d6a33a160445b821115a7d1
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.7.post_attention_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.7.post_attention_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fd2258b96c0a4a17c9a4704e55d4a8c72b003dbc
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.7.post_attention_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d700f2ef213a10919f73f4363401adbc22f4e9d6ea5826bb90c194eb3afcaf49
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.7.post_attention_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.7.post_attention_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.7.post_attention_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.7.post_attention_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.7.post_attention_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..bc9929d3b845832ace58b3f6225984d0886e1e47
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.7.post_attention_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a0e247b94d98ab8e5f6e386bacd26bf0f8a87f66a6db43fff53f4d0e8b108e2b
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.7.post_attention_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.7.post_attention_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c0b7729e33fcd1c98889943423860512ed7ed3d6
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.7.post_attention_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cb8c190a9250d37798348060ca279d4582e635f2d5d887fb72ac656b30ef49e1
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.7.post_attention_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.7.post_attention_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..31fb7750cf4bc7fb74437930712d60f5d50f6371
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.7.post_attention_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c661a5f5c9e72e26c8a0f6cecc005a275fa91127fd4173be147f8b137c4bbd61
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.7.post_attention_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.7.post_attention_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.7.post_attention_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.7.self_attn.k_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.7.self_attn.k_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..423eb6fee70169e2cb560848e3007bedca670dfb
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.7.self_attn.k_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a51ae7ad0e8b9d173a4b6b00b4fe7018420b351f25b9cf9d109880c9b43d94d0
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.7.self_attn.k_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.7.self_attn.k_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..510361bb3cc0d634d751a07e4fe93bbda3073059
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.7.self_attn.k_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:38d5c1675c347dbdd7ddf7709665f78177a86d8b45d42afc6a4c0e78ebf28ef7
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.7.self_attn.k_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.7.self_attn.k_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0a71b9b01f0cb14936e6919f9a97404976737f0a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.7.self_attn.k_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bf3c03e7c3648324214698be31be987f8b3308be2f414b9accff6d35e5082724
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.7.self_attn.k_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.7.self_attn.k_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.7.self_attn.k_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.7.self_attn.k_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.7.self_attn.k_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..245d61391757c27d25ebf0c3ab4c9a91262f8ab1
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.7.self_attn.k_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a26a7162c9fb0ae1eebe4fc687ce8f77b1b42c4ad5ef3447aafd3d6d73148a8a
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.7.self_attn.k_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.7.self_attn.k_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f2850b5c56c80259d82fe5f5a1249408ba10d8b8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.7.self_attn.k_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9f772912877177fc578d6494cd9d8fa3f6cd1b2259c6053109dfcc71ac0464dd
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.7.self_attn.k_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.7.self_attn.k_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..090f3512cad580c36a79b578aa17cac0f48d60fd
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.7.self_attn.k_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5e0035ddec41afc4eae02ed9ba2835c538c545ea2519b2424d7ef4d7499c4a87
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.7.self_attn.k_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.7.self_attn.k_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.7.self_attn.k_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.7.self_attn.k_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.7.self_attn.k_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d9c1b875770189e5899f71fa58c85ea403e24281
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.7.self_attn.k_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1c1c0d753f0e9f33c516c1f01d3c230d219e8f1c74422729cd2e242bdf419338
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.7.self_attn.k_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.7.self_attn.k_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..319204b85f79d2831566430ea47bea7a94c57b9c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.7.self_attn.k_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:520cffe4f3a271fa1b721aed95d07535be7909a68a6107b22a97bf40d4528510
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.7.self_attn.k_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.7.self_attn.k_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..169fe029ae842542229cb040d6df8e273e3d0828
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.7.self_attn.k_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:67052745e6b1f33847e18e1489f639b0bad5d44a1cd7a176d284fcc5396b3ac6
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.7.self_attn.k_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.7.self_attn.k_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.7.self_attn.k_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.7.self_attn.o_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.7.self_attn.o_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1fb81f0a3585b5c7a3a1224c7f93e53f98190ba7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.7.self_attn.o_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ab1dc3d38b96e390d36dcf240b56c8dbe681d20a7a8a8cbe6b96c52d690d10c0
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.7.self_attn.o_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.7.self_attn.o_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7ed195f62f23b01c23b9996f4e5ef6d3af2426d0
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.7.self_attn.o_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a61a5f3104f9c759cb6f545db750e9e6a1c029be917729d84d8bf3765b0f1368
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.7.self_attn.o_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.7.self_attn.o_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c1511b129962d6be042f371ac92c483b626f60cc
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.7.self_attn.o_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:927b2db1985a880d6a8cbb77d70f958e7d9d32989b3f99d9a29de306d0adbffb
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.7.self_attn.o_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.7.self_attn.o_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.7.self_attn.o_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.7.self_attn.o_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.7.self_attn.o_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..15f596fa5e6a472777302652f5890f5fa3a6d6c6
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.7.self_attn.o_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eede545638dd98e22180f0cee23cab7555c29ee13a7cc3da5753933020d318c0
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.7.self_attn.o_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.7.self_attn.o_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8c64f93438fd6b2434642f41f2ecedb5c5ea3409
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.7.self_attn.o_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:abc803377205ae6b50942268889ddce61cd1c42aca24a7fd5a8178ca9e0459f9
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.7.self_attn.o_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.7.self_attn.o_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..56af0daf5a52476128083091704758f6ff0561e0
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.7.self_attn.o_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f5c78a93d35f6dc2b235dfd911d725ecc8b2e4c26454a00d065e5cd005433103
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.7.self_attn.o_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.7.self_attn.o_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.7.self_attn.o_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.7.self_attn.q_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.7.self_attn.q_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b0ec25b9a4a26587c9624b4176b507e9035e0a98
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.7.self_attn.q_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2b655bd3370b8bb67d9ab6c35570b7a8904ad5dd61f4dd8d582fea375ab0b933
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.7.self_attn.q_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.7.self_attn.q_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a4efa043233706eab53965416f01eaacaa53265c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.7.self_attn.q_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7fde4fe4839cbc98c0443f3113bdaada6dec5db859549140de32f724f7e757c5
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.7.self_attn.q_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.7.self_attn.q_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..82a45e01a20b1fab207b1b4ffe3c8336da20d4e8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.7.self_attn.q_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:666d4c9b45ce36ef58ab8a0ae35e383e3d637e0a039cb7b728ba3c8696dd6915
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.7.self_attn.q_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.7.self_attn.q_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.7.self_attn.q_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.7.self_attn.q_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.7.self_attn.q_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d36c97457e6c1301a97d14b68885867e6ce841e1
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.7.self_attn.q_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:87c79acc2cece1b0f1fab22dd4e85ebb8887c8e9a37fc471d743449b5a6beaed
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.7.self_attn.q_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.7.self_attn.q_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5a105fac1eb6c129336c8338f5dfc6c7cf510631
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.7.self_attn.q_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:93eb5ea1272f700bc0b185fc5666fd9088c41c17b4816e594d21afc53ef537db
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.7.self_attn.q_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.7.self_attn.q_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..40f17f4e1c586b17b8c1f0bb4c2fd0ea7e8fa54b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.7.self_attn.q_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:50d6fd9d27b1b03d9829eb7514c484b7a6e0f3d96759295fef85e71053e2b28c
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.7.self_attn.q_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.7.self_attn.q_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.7.self_attn.q_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.7.self_attn.q_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.7.self_attn.q_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5d991006f18321a6bee4574da9c7057b5766a51b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.7.self_attn.q_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd61b54d2f53e07006c82f45e5091b877750a94992cabcf527326fa7b1f57ff5
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.7.self_attn.q_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.7.self_attn.q_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..efe00682d085fc86f17328f2b26e3115f2d1d06c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.7.self_attn.q_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ed9c8f3c26bcc466499e39acdd2060fa5d4a94f364a0125040b1fab93cd81aff
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.7.self_attn.q_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.7.self_attn.q_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2defdd6b8a6fa0f845d7ac5c0a2a285e6c9c5b25
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.7.self_attn.q_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b45f92b28b1ca2000a59116bb47cf67589cb57e16175def923b75c19b32711ce
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.7.self_attn.q_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.7.self_attn.q_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.7.self_attn.q_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.7.self_attn.v_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.7.self_attn.v_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9240f3df4c40d22bb56bcfbc116643434d93213c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.7.self_attn.v_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:807126feeedf849c1dfcbe3cc84bbc8f04ee9c362aa7587df5f8017a4a85442d
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.7.self_attn.v_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.7.self_attn.v_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6ca9d51241e78bd4e6e489a4501f0bcaec66e889
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.7.self_attn.v_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ce1410b1adad5498a48d13fd477e37bc0295803348a8e863210933676470f47e
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.7.self_attn.v_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.7.self_attn.v_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5912ebca3dff8e0f0b57f7b1b2f57e4204042f53
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.7.self_attn.v_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:348fcf286169f5a98e816bf7e7b02108c24f6739976ff6a3eaf80824d2e947be
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.7.self_attn.v_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.7.self_attn.v_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.7.self_attn.v_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.7.self_attn.v_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.7.self_attn.v_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b41579864d4cefd0aa4f970ca42432ac6bf5dd21
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.7.self_attn.v_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4fcf04f2a329b195e8868643ff34ac2a52e21bb2969387ea612e8480fcd21a30
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.7.self_attn.v_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.7.self_attn.v_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d35d62d28dd9f96356824e6857782a8d1be13f65
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.7.self_attn.v_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aef7bafe12a87926ec7659fcbb505ee0b63cce188dbf83fdf9e3cdcce537086f
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.7.self_attn.v_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.7.self_attn.v_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5084cddd6815c4447709cc4bbced606b20e1b228
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.7.self_attn.v_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2574d688a350a64ab23eefb3c7411f9166ccefbcb9133b81386a2e0080c8306e
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.7.self_attn.v_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.7.self_attn.v_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.7.self_attn.v_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.7.self_attn.v_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.7.self_attn.v_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7e956745bc06a619dac8bb8179cc45775430920a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.7.self_attn.v_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:69812afb097d8ea7068adf02c5ede962bbb5943cf7248ce4eef495e58354904c
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.7.self_attn.v_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.7.self_attn.v_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8588bdbb75951349d72b0fd02be55abe95dbabdd
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.7.self_attn.v_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c82dc4c7392fbaa8c5edc9b8c6d888aa8d344f6965a45317ced191bc3d13b7ab
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.7.self_attn.v_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.7.self_attn.v_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..34693e92ef27ba624b01a9e51e0d338c12c9a57d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.7.self_attn.v_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea5f4f535ef36d48581a3c526260202528c88e2b26c87a1631d43754b6ebc407
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.7.self_attn.v_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.7.self_attn.v_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.7.self_attn.v_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.8.down_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.8.down_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..50383304f63a37d928c125582eecd26c4d009584
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.8.down_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3852c0d951dc247f5855a8c4f4e51fa30fc168f29b44e0f4fff84695498684cd
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.8.down_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.8.down_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..835fc41a46562f555974d58d21aa6a978b6d8003
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.8.down_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f4af6f5d8452c10255959fd81aa6d49d7e408a0da61b762b075c3d2d775f592c
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.8.down_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.8.down_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3653cded950e3ad159268f9ba389c0415265d74f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.8.down_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0c2ac20e8f365100df82f41a59f3f932e6b835601a703cae691ba026006ea58a
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.8.down_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.8.down_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.8.down_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.8.gate_up_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.8.gate_up_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2a35d9b58cc5517ed3bf01c13333189349c09441
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.8.gate_up_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cbf832cb8441c9e0b4f8416543bb33a1543d446f4812567dba1fdb9c48ddb6e7
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.8.gate_up_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.8.gate_up_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2dab916860fe7e2bc2f85b47be7b6214e3f18bdb
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.8.gate_up_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:14a2d78f5b1a36c88e5b73fadc44369149f20825d6b456e50096885e80ee96b9
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.8.gate_up_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.8.gate_up_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c80e520f6cebcfa851f4fcf9339a1ae14af4d4b5
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.8.gate_up_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6143193f4ece1867588b0fc820e8f850af815820c35e86d1fa57cb696f73c608
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.8.gate_up_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.8.gate_up_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.8.gate_up_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.8.input_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.8.input_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b2b3fea8f290d1b360c0ff8160ab2583a670ba20
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.8.input_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3c90cba6593a6c526e52007072ffe3b04afb2b772d4c638e954c08f03036dfe5
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.8.input_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.8.input_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6fd27c0e5b5518b5d5a999ddb23e48aeb7b5a659
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.8.input_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:daf4beb8d90f20ce763082d128b2cf2337716253673a6841ee84c863a0ab4d7b
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.8.input_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.8.input_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cd14693823afba0321b19f931d85fae1263ed373
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.8.input_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4c98518bf3a5f4212b44aab054fa6996594ff9b194163f0bc80c39c61b86189
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.8.input_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.8.input_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.8.input_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.8.input_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.8.input_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..672207f1a7ad38fbb7e8e83f3c0795f99eb9aa81
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.8.input_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:16c12a10afc22dbbde640458bf32e6959437addedd5f3e1d982c24281d41a1b0
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.8.input_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.8.input_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3decd051316c069e8da2b77c7c3c1151d8b4a562
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.8.input_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7034298397ca06b87f66912067689721248d24bcc11c7d4c331d6f0dfcd2feb2
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.8.input_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.8.input_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d3337bfe6c18d687c01e4eefe0cc8b3dbd3de73f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.8.input_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29e58c1116061d9d8f7072a10af373c611e740502273e827fc34a0c30b74380f
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.8.input_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.8.input_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.8.input_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.8.mlp.down_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.8.mlp.down_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b90c046cc7b4dfbef6b46fb3011154f35f26f32b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.8.mlp.down_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ba1fededefa23d243445da81e135767f01c16cae399b2cfb927a1bcd47b7117c
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.8.mlp.down_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.8.mlp.down_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6ffc22cfd166fe1c4956ffa6bc712152ecd43c38
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.8.mlp.down_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a33acb30ca110dd51353d6a4c3a9788d7e6bb6fb001f9587c7d32a1c98b585ff
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.8.mlp.down_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.8.mlp.down_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d6e0186a06415ecb6dc0c4f63ec5a874c0fa767f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.8.mlp.down_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3d5d1d7f542c4f04d0719a7ec4b9eb13c98b60678efcb5a86c1892e70c60f46d
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.8.mlp.down_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.8.mlp.down_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.8.mlp.down_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.8.mlp.gate_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.8.mlp.gate_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..37b837c8b09c6d188d2eac986b97dfbe320e8f08
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.8.mlp.gate_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2d57f18ca28b35854a11a98324636f7f4a39d2b9944e5dc259c3f644ab38fcd0
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.8.mlp.gate_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.8.mlp.gate_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fda757edab90f1d9f4b8773b962cdda7239dc1df
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.8.mlp.gate_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b173f61adfaeaa61d7191e8ab58662df5f0e4fe3c46042cac9805296b7377df2
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.8.mlp.gate_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.8.mlp.gate_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2ad5a5fcd8c719deff04da2de84ed27e325a6d48
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.8.mlp.gate_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c2a2f38da1981059cf7765bd0b4af5ddc9aea73756e6e0b4b0d7bef5186796a6
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.8.mlp.gate_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.8.mlp.gate_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.8.mlp.gate_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.8.mlp.up_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.8.mlp.up_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b9a52700b5175604d8d4606c82d883f3e72d76fe
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.8.mlp.up_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:11f6f3009fc4ee4df9b3844a25215ad44dafc9a4fa71a2169994835d84a5c71f
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.8.mlp.up_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.8.mlp.up_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0c44e6ca8f94c214d00fc369b70fcc1b1e964fd4
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.8.mlp.up_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:791ae22bff45ed7055bd887bdfb4b09d4e6923c4381e246bc5b7cd26fc035dab
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.8.mlp.up_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.8.mlp.up_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..77f1c83bee1eb8769fc218fe5015216cbb354ea7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.8.mlp.up_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2d2eb9473abd85a11a6ba45ce2a706dcca703424428f6f7a4e4d3b68dbd265b1
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.8.mlp.up_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.8.mlp.up_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.8.mlp.up_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.8.post_attention_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.8.post_attention_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f7296e2c62701445f008de238a7737b338d7476e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.8.post_attention_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a3c259d8c4c6243b354b987ffc40a0bf9fb54b7a5a54fa1f2861a09ce800179
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.8.post_attention_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.8.post_attention_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4d986a273c59ebf15cabbd852c0bcda2837cf661
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.8.post_attention_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:02124f96448b43afa082ea3591b3a3f01db6e4e8bf180b4802d41aaf3a3de67d
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.8.post_attention_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.8.post_attention_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..85ecbfaa0e9cce8df1c65100724af1bda09d4584
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.8.post_attention_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fa8e228a4adf5a66cf917bd8fd0f680fa4f9177874d10423edea2a772c24859b
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.8.post_attention_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.8.post_attention_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.8.post_attention_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.8.post_attention_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.8.post_attention_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..bd4802deb60cc45a70de7fd783614275e287f305
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.8.post_attention_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bcfdece40e55a3224f04cf8dd8e31c564c9e8913d64381e66a34807d2dc01b66
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.8.post_attention_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.8.post_attention_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7d975d8c3a00e1143c23fd1a5d8d97e889b9b527
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.8.post_attention_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f50d706b6c2591bdfa30b100c2297bfc12ad6bcfc4162b956510ad3b027cba07
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.8.post_attention_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.8.post_attention_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9abd1a12241bf8f0159ec784cb8bfe05b026eac7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.8.post_attention_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:818fecb0e32cebe7d9a56584236b2d4f307b2753fc56eabcf02ff554ad012432
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.8.post_attention_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.8.post_attention_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.8.post_attention_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.8.self_attn.k_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.8.self_attn.k_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..22400f343352337966e29114f870e22695abe843
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.8.self_attn.k_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4e00b888918bc5b0e660079a681dce7defa8c101f00d1b8317ce2c4e5dd4b52e
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.8.self_attn.k_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.8.self_attn.k_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..709127592156a4a18957fbf165ab4a41636a8420
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.8.self_attn.k_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:647aefa2f6e2e66b9d5b51fbc67d2d1fc5f30d53ae14dc78b5fa6618f58528fb
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.8.self_attn.k_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.8.self_attn.k_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..da934ff3f43d409c49b5dd70ce9717dcd39a1c1d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.8.self_attn.k_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4fb23ecc26961414e28e41ac5e4fd433b6526dbff9044af1348460ac14f7ea9e
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.8.self_attn.k_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.8.self_attn.k_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.8.self_attn.k_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.8.self_attn.k_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.8.self_attn.k_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..66fb38c4c226f3b4cb2b7e80e9e33df27875a029
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.8.self_attn.k_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:35ea5b6079b59cc618c3c668829bf4274cbaa7b68165ed1c3758dee7661e49d0
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.8.self_attn.k_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.8.self_attn.k_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f6e52676a3cea002c9c548687d8a1b4a93b7c9c2
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.8.self_attn.k_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a6a072d0667cb4002d01237dab063d9b6d8565ea73a698adaeb6ce32a7d4ef19
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.8.self_attn.k_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.8.self_attn.k_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ff463f66b3f6613283aa8e516b1c5f70c2839aa3
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.8.self_attn.k_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0e53fabbf4c8ed6d44738c498fd4d685e5a72b075e48247828953f2f0dacb5e7
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.8.self_attn.k_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.8.self_attn.k_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.8.self_attn.k_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.8.self_attn.k_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.8.self_attn.k_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..02acb8a4056316e112714dac493ed60433c2aaad
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.8.self_attn.k_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:af80f696f443e89df6529b577dee51f2b236c1edf8a777edfedfd35fdd5c5496
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.8.self_attn.k_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.8.self_attn.k_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9c62d258806fb163ea657b2bf1ce2c2ba46e22d1
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.8.self_attn.k_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7464543002d3efd36a387fecf9796e1f3b18dedfdafae62142a80fa5e8f76ed7
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.8.self_attn.k_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.8.self_attn.k_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6f6407c29df0daf0bdd3da1b68d308dbbce0a90e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.8.self_attn.k_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ae2db3ae5d85809fb918434503242b05cde285395ef5456c4eb075706ef62f7
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.8.self_attn.k_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.8.self_attn.k_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.8.self_attn.k_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.8.self_attn.o_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.8.self_attn.o_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c2b9ff357fac4716458c3edf733a53cba4e46667
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.8.self_attn.o_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bafcbc0467bb0b0b1861d2de339994f2635adfa14fefae775d03c46a59d923d1
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.8.self_attn.o_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.8.self_attn.o_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f86565135ab6639bc2a74ef3238299cbecd30986
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.8.self_attn.o_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef286bcfb0ad9901b312a1add07651f3927d871ce6d1c25ede494f8af484f64e
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.8.self_attn.o_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.8.self_attn.o_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8a08faf137d6d41d717c3b3dae3d1217f33965f1
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.8.self_attn.o_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6683bfce75668d91556048a2c22e86f0ee95b9a0dc9a11ac5cb8eb50993ef115
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.8.self_attn.o_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.8.self_attn.o_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.8.self_attn.o_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.8.self_attn.o_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.8.self_attn.o_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6f180677c61ab25949964a3fa4fbf0045a97e530
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.8.self_attn.o_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0325d22cd04cce3f24bc6ef80bee9200079f650429d4ca7225c587b95e556cc6
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.8.self_attn.o_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.8.self_attn.o_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7bc4600da6b19da794140d8a90d28833a6a6e5af
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.8.self_attn.o_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fc774767c6dade3f17d3e2de143c8c0a74a4560d658b6a048ae710ca3d010cae
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.8.self_attn.o_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.8.self_attn.o_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..121e9f4ab3c0bcb963a498ec0c3a2d73ec357651
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.8.self_attn.o_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cad75f467ff2c5933d2b55e519017aeb02bba80b98c37924f0f0784483e361c1
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.8.self_attn.o_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.8.self_attn.o_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.8.self_attn.o_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.8.self_attn.q_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.8.self_attn.q_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2b113460d6f23e774b7ed88aa50b91fbd235f0d6
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.8.self_attn.q_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:460fdf7e8ac33c79681e08aa00f200a8ef821402cf8d57831e9f71163ed2ef25
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.8.self_attn.q_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.8.self_attn.q_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0d81f3fedda1758a6563ffbb3ddc95a209c53946
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.8.self_attn.q_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61e8d801ffa216c31b52aec78d19e6f692950c50c8b3bc6da99d2d3628da2344
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.8.self_attn.q_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.8.self_attn.q_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..53860c6198b78f9a7eae0916115104fd25bf65e6
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.8.self_attn.q_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7b48b09afdf9e340f36940fce124cad152a22412b581646d5b773f1ccdf625ab
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.8.self_attn.q_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.8.self_attn.q_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.8.self_attn.q_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.8.self_attn.q_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.8.self_attn.q_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..689490fdb3f9fa06a1b48c45eb63f30dc3d13235
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.8.self_attn.q_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c6432dac307a0a9e7c2abec11f9bdc8adacbcefd7eb289a14a84d9a3e74fef13
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.8.self_attn.q_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.8.self_attn.q_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9e87d318cf028e69715d286fa476705e6f45ad75
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.8.self_attn.q_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f9c071196afa88c3c7cbc00bcdd264d4827bc0ac26a6e3175ca7b6e1fda32ec6
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.8.self_attn.q_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.8.self_attn.q_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e949e780862a89e17436e97222ca6a940e63b9b0
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.8.self_attn.q_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e7216cf57ecc2c5dff36671acba438f40c7bac993aff6d7ac8c4aa612b104a0a
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.8.self_attn.q_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.8.self_attn.q_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.8.self_attn.q_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.8.self_attn.q_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.8.self_attn.q_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..49ff5242649bc95aef86a2e85d2081ebc0ab97eb
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.8.self_attn.q_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5ebbf06b0317b953386c7113a8fb38253edcd4fa1f777cd61900e7a368ae351d
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.8.self_attn.q_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.8.self_attn.q_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1562ed92bf33302018076ca31937dfe9b28e589b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.8.self_attn.q_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c0d357c86a16b5bc1ce107f84f647f990cb6a0fda4c2b2c518d9870c0168e36e
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.8.self_attn.q_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.8.self_attn.q_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ae858111dbbd48f6d27a50fff3f67ed01f96cfa3
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.8.self_attn.q_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:540a2dcfd04de483fb5a5b37124c42a077b8f9e642077f3e3ea2904beab9fdb1
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.8.self_attn.q_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.8.self_attn.q_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.8.self_attn.q_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.8.self_attn.v_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.8.self_attn.v_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1af3bf0feeb1e6863d96b642bfae914ddb6f77f4
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.8.self_attn.v_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ecbaa91927c2a2de58fb237d8cb0af16082a221dc67be30b1ac4746ecb6fe65e
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.8.self_attn.v_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.8.self_attn.v_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2a0cbaeb02ad3dfcb3638d2e107a8092a88b2ae1
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.8.self_attn.v_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:772ce798336419aa33c413f0b5e5910de1c2341010a45327b2eee1d335f38c79
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.8.self_attn.v_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.8.self_attn.v_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e016dda6a4a0645564234f192c45f2a02c6cb7d9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.8.self_attn.v_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e90316590a66ca1eea6cd2158e2b1d3729d4cb136541300f02dbf2931a679bad
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.8.self_attn.v_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.8.self_attn.v_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.8.self_attn.v_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.8.self_attn.v_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.8.self_attn.v_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c81b125656196949a01e88cfb92230b1bbc2d506
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.8.self_attn.v_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dea5975de96f9ae1296cbf2761d6efab87de5d34ad2a932751f1ee8aaebfd3fe
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.8.self_attn.v_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.8.self_attn.v_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..945ff676bc73e595ae06f9f699f289bcc1acd293
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.8.self_attn.v_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e1713d3c0d4398f1eafce2ccd66a2a0b8db7ec82f2ff7983289120f38db2d92e
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.8.self_attn.v_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.8.self_attn.v_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4637a56be1844110bf92358f62f25a4307a1305f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.8.self_attn.v_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:59b4e0959f47489750c6dc66fcebd551ffd394ce11c5f95e3c58b6977ebf3e7f
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.8.self_attn.v_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.8.self_attn.v_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.8.self_attn.v_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.8.self_attn.v_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.8.self_attn.v_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2c77ff7eebf0f8e8fe8993da1965b15517b272dc
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.8.self_attn.v_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aaffa42cca8ed5bace75fb1a6417667216ba2eed5566728b5dc158b0f03f0d53
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.8.self_attn.v_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.8.self_attn.v_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..86198570275aed802e91b727b0b948f4a9f4562b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.8.self_attn.v_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a345470fbe92b399a5b01e6ddd51280b8859abbf66298bfa3f3285303d59781
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.8.self_attn.v_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.8.self_attn.v_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e8ee284a13555263fcec8410cbc5383318b19382
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.8.self_attn.v_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f1d5c3c6aa20573d7f15205f755cd36f86843e79aa5b8c6dd7c3f3a70e1aa029
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.8.self_attn.v_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.8.self_attn.v_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.8.self_attn.v_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.9.down_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.9.down_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a9d56c4eb9d7bd6f82246bad982e6098649dc269
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.9.down_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3c410078108a4da60216ad82047ded8211addf4343130d135b9c5ebb7102a2a6
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.9.down_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.9.down_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..31a54349a213a444d2c07f3ab455cadaa9dfd9e3
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.9.down_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3badfc13e1b6a072ca8982a76e04de7a9410e32f35d4afef25665dca1659429d
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.9.down_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.9.down_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..55ed677a51e2ae5178043464cbeb9162af80ebf5
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.9.down_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9b77327939d6cba6e0995c482c4c2f77b9e9ebc6efb203b08ecbcee9587a20
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.9.down_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.9.down_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.9.down_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.9.gate_up_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.9.gate_up_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2c36fc130baefbe40f44c6418c219139d11251e8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.9.gate_up_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:38838e91dccce52cbbafe1c54b0c4e276e1af75833fa64883679fd727cc7e461
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.9.gate_up_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.9.gate_up_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f16f544b2885be371a99442de4f8fd8253ed8ab5
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.9.gate_up_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fc2100df12aed30f362f483383dd58754931412800d6295911aa6a620a721307
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.9.gate_up_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.9.gate_up_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..44a48331ee9a3d49667bb52d15bf823818f5545d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.9.gate_up_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1cf068151429e7730fb0f23e834bb1504e5a8c930508e368b99d09d13342841f
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.9.gate_up_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.9.gate_up_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.9.gate_up_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.9.input_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.9.input_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b1f97c74bdd781f91b8467848bfbf135494af2f2
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.9.input_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a976b8d34504573a7207eb35cbbfa3143e8d8d168e4edcc3a846bd21db5dc85a
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.9.input_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.9.input_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c23f60a136ac1d01ee24a48909a5a0a38dfea61e
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.9.input_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a310c8675e70f17d34c1bb1a55ae1412aaee6fa9a94a35cd01c39380b822f327
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.9.input_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.9.input_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4d7f158eec8966cb7501e65fd9a32b2582614d59
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.9.input_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:08e8881306b7888f6397090c3af345f1a1df612359549430802a7c392a701fe0
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.9.input_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.9.input_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.9.input_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.9.input_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.9.input_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8c9c54fa15363f19225e556d7cc54700a0809091
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.9.input_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5f95edcbcf3cff18532343ae46bfde4c2b41e1fa1ef960989521c8be789d22d9
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.9.input_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.9.input_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a37cc79c071c71a77fe8d93c452acabd87af8707
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.9.input_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b336ad0d03b5b2462e4508dc50eb9cf068279fc7320c42b13bb044f2b6e17cae
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.9.input_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.9.input_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b2984c475b6b5fa7da8a2c33fb57e7652aa4d55c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.9.input_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c06813f116f0ed750317f020e339c48888f2102f4dcba89901106cd853862be9
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.9.input_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.9.input_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.9.input_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.9.mlp.down_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.9.mlp.down_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..564a6c9da09e56b31fed527cbc3e3c3448310789
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.9.mlp.down_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1f96a0c9a05aba26d2ee09e5533fea7461a15abb34ff42e834bef1367996bd43
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.9.mlp.down_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.9.mlp.down_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0dc446bd70b9074b7c231d3d57151897e28df642
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.9.mlp.down_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:39e7440bb5eb4fe8aa36f6d1a058607464d72457268a1d1092350cdd8d051643
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.9.mlp.down_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.9.mlp.down_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c6dca777dbad974c2183d4a71ddfd9d2906ce55b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.9.mlp.down_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0ae0c85c96cf30a5502e9f09c03e2adc2572d87c97b5fe361a7da5d6723960ed
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.9.mlp.down_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.9.mlp.down_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.9.mlp.down_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.9.mlp.gate_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.9.mlp.gate_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ef7190e5b2fab6189748d16f22b4c31f8db3c5fe
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.9.mlp.gate_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0c12aee9ae8641b6e4a5fda16da813f775fabe043c8b174d400337b9af545967
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.9.mlp.gate_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.9.mlp.gate_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..24c65f5a463f69b939531ef5d11c71f7117e7f26
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.9.mlp.gate_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:370c36678b44e0ebc5bcfce43145271e84611effb54eaf1b1a9910f73358dc80
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.9.mlp.gate_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.9.mlp.gate_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..91b6de56a1feb46462beb220f91c8492624d8d54
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.9.mlp.gate_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f6429c0e66f78b94dab20ed3d3558b3900fb6f824a0cc9c165b8b2cf60ba391a
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.9.mlp.gate_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.9.mlp.gate_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.9.mlp.gate_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.9.mlp.up_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.9.mlp.up_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..711af8760ba81b8bf9a80ad74d488cd0ec66f1a0
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.9.mlp.up_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:46e5e745a2bd668272ecf96f30596ea0ee1ec13778282cf01517621d21e74228
+size 36865244
diff --git a/global_step194526/_universal/zero/model.layers.9.mlp.up_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.9.mlp.up_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d989335b58d5e66eee7a176c51a67caf0dbf8e43
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.9.mlp.up_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:015481bb4dbe263aa8b3277fc79deef3de925c2ac9623f245f54496d96522ae4
+size 36865259
diff --git a/global_step194526/_universal/zero/model.layers.9.mlp.up_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.9.mlp.up_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..bb1f006101d3e6daff38ef234d61d9b6538f1180
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.9.mlp.up_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:42b9f97ddf6dc75f5f0e32fe61634fcffe23db0b3dd8ee9a66d5e0e690e5823c
+size 36865165
diff --git a/global_step194526/_universal/zero/model.layers.9.mlp.up_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.9.mlp.up_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.9.mlp.up_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.9.post_attention_layernorm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.9.post_attention_layernorm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3446d1f1afd0d8b50c48c183de6e4a25bcc7697f
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.9.post_attention_layernorm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:398fe5997bd2644181f7db8b4dc2e50debe46e31e8e393b9be2ed52b5b3d38c3
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.9.post_attention_layernorm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.9.post_attention_layernorm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5ce7edf330cc468e629de1d88c9b3a84a4e5987d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.9.post_attention_layernorm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:62e0dd0316716711aeca3b8abe47f9995a3f12b07233408ec8c27df2a352247d
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.9.post_attention_layernorm.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.9.post_attention_layernorm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9ce3ef751f5d6a4f3135aeaf9cd0f5286b04b2c7
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.9.post_attention_layernorm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:14b5f56dd5a3901f15d7b1f920822b5f495371f562b9e6f87235c59554a23df0
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.9.post_attention_layernorm.weight/step.pt b/global_step194526/_universal/zero/model.layers.9.post_attention_layernorm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.9.post_attention_layernorm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.9.post_attention_layernorm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.9.post_attention_layernorm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fc7af8e5e86deb78894fd0b43f52f5667fd1269a
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.9.post_attention_layernorm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c8daa52c756ce1e6655bd9b5f2f9da963afd925cf1a977f45377754df75725ad
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.9.post_attention_layernorm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.9.post_attention_layernorm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..090b70b131c7517ec56ebd1cb19262c35316ce93
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.9.post_attention_layernorm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5f6bbe63bcf40ba170c61c3254b5e293a6bbbe67ced746b99bebf457c8270db3
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.9.post_attention_layernorm_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.9.post_attention_layernorm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3a5a34c41ccaaef35e199a1f1f55179c1f4d5be8
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.9.post_attention_layernorm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b29b6a941e254baaaffb63a25820c82b87ea145db663084a5ef7d635ce786360
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.9.post_attention_layernorm_alpha/step.pt b/global_step194526/_universal/zero/model.layers.9.post_attention_layernorm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.9.post_attention_layernorm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.9.self_attn.k_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.9.self_attn.k_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f054c3ca70a2e07fb669e1aed84c5a8d83b634c2
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.9.self_attn.k_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1dd00559727381243646065609b68d55d5921d61afe9305bedecfadbb133ada6
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.9.self_attn.k_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.9.self_attn.k_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e417a6f727247322f1de0975f05b3c4247feb02b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.9.self_attn.k_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3b451b4041fc9a749f6225f595e30e3e75790da6ec48bb7c47dd252dc3ce210b
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.9.self_attn.k_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.9.self_attn.k_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6b69ac6b69c3f13d939f8665ea6358cbbc623fb5
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.9.self_attn.k_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a5e2bc8c94e6ef253edf06cefc6f1213f4141117d69a6dbf0b7041d46c55a80c
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.9.self_attn.k_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.9.self_attn.k_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.9.self_attn.k_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.9.self_attn.k_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.9.self_attn.k_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d63d39673f3b46de499455699520fb84a0b6761b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.9.self_attn.k_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4270ebd28b725418e69f5ac0112313e1067674e441c3a6a1344ea607812ef5e4
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.9.self_attn.k_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.9.self_attn.k_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0861b1a163d2a658dba149c3766a1bb12cd5332d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.9.self_attn.k_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ee1474a769e963bd92418bdade80543ea13770523f9deaff9df7e190a4a42955
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.9.self_attn.k_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.9.self_attn.k_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fe869a21afa90755c71ab24a5b605f0310f7c9b5
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.9.self_attn.k_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5730bcb963213534fd4551cf776502c06f06f603e12b18dd6a2103b5e1c3f609
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.9.self_attn.k_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.9.self_attn.k_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.9.self_attn.k_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.9.self_attn.k_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.9.self_attn.k_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..bbe32d76b3574d85623cd27473bd1853028d5363
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.9.self_attn.k_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:487905f49f33844de6ab79adb88d030ee75a30a57819f496aa969dd7cbc780c0
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.9.self_attn.k_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.9.self_attn.k_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..057bb59706558e2c6bbfd980677aad009308354d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.9.self_attn.k_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:85d89a778724cad6e3d5b3152420954f02636bbe6b050ba76a0a4a7188fdf9d4
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.9.self_attn.k_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.9.self_attn.k_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..afbda38e525577b67eb7e47d224b2d6a43440a41
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.9.self_attn.k_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2677b16d5d730dd679d900740674b741996377096fa60359821713b5491f1b86
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.9.self_attn.k_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.9.self_attn.k_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.9.self_attn.k_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.9.self_attn.o_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.9.self_attn.o_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fe0832c30d55f03209d62c14c3f2007b3f43fe30
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.9.self_attn.o_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7bc8c1dacfed5a791dd6e8782774c613ba4073cc34b90fb60810d50353619bc9
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.9.self_attn.o_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.9.self_attn.o_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a94bfdd002eb3550d755d31187007240f1f1568d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.9.self_attn.o_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:884b4360aeafaf74da41e421bbac45ea72280ca480bf51ddf911bd8f018a3af1
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.9.self_attn.o_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.9.self_attn.o_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2094fec12e22fef5ccdeef02e6dccc7f32d300e9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.9.self_attn.o_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2dd979c810e097f4e1d16113f3c5114e6d533667522ad8d112e8e3e4458cbd85
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.9.self_attn.o_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.9.self_attn.o_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.9.self_attn.o_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.9.self_attn.o_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.9.self_attn.o_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..efd3f57cccde7ceddd8f57b6042fecba1d91cc87
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.9.self_attn.o_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0b1f918d98e7775cb03f6fce4c61f23ea4c7393c5eec13647b8eca056595a199
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.9.self_attn.o_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.9.self_attn.o_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e9655effbc3a9a18b3aaab0fd3baefebccc9d94b
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.9.self_attn.o_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:207ef80754f1069dd07471085e622ee13467a91e336b28106dfb7e0bb0461771
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.9.self_attn.o_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.9.self_attn.o_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..08b19434f4ea39ce36d8168a7f20581c2b806149
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.9.self_attn.o_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1a03f89451322757e1d1d606017860af3ad6bc2188b53b1c2d5d4e4c51b576c6
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.9.self_attn.o_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.9.self_attn.o_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.9.self_attn.o_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.9.self_attn.q_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.9.self_attn.q_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..696d9166d21729ae1e216e6d5268d713e9ded223
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.9.self_attn.q_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8406f61b121d715916cc5cf7828c45e8a79e914afa807a20c0f41d5771a20be5
+size 8860
diff --git a/global_step194526/_universal/zero/model.layers.9.self_attn.q_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.9.self_attn.q_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5b61a6da6410a36914d6b4e500994c146e44d194
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.9.self_attn.q_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e9e09a9dac7f96da5d75f281b89612d364aff81cef2a1d083718ce11e4f88b2f
+size 8875
diff --git a/global_step194526/_universal/zero/model.layers.9.self_attn.q_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.9.self_attn.q_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..49e7017d1459345b374dc708cb9a74b988296a1c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.9.self_attn.q_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:458c688036e981e6a14b6e8e1e44cf25f678464f0600eebd80c923455f55941f
+size 8781
diff --git a/global_step194526/_universal/zero/model.layers.9.self_attn.q_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.9.self_attn.q_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.9.self_attn.q_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.9.self_attn.q_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.9.self_attn.q_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..87d815a1433942a4c9912ea088e6bb168c4a0a5c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.9.self_attn.q_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fdaf7f4a78b7ee4ae15e2659240a2cc3ed341f39cb2c4fb07735c69cb642b3c2
+size 14746844
diff --git a/global_step194526/_universal/zero/model.layers.9.self_attn.q_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.9.self_attn.q_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1c73d369de3f6f0e42a3923c4f43913f1066a164
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.9.self_attn.q_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:308c7fe95e15773300b940527241a134da2276b89d20c5344d277f3ec72b71de
+size 14746859
diff --git a/global_step194526/_universal/zero/model.layers.9.self_attn.q_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.9.self_attn.q_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e7249f1080c0d2e1c17d109da010f42455237826
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.9.self_attn.q_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3ce42177e5cdb49cd474c80882810da9695e2e77b473c06ff30c129a5ef1ff2b
+size 14746765
diff --git a/global_step194526/_universal/zero/model.layers.9.self_attn.q_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.9.self_attn.q_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.9.self_attn.q_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.9.self_attn.q_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.9.self_attn.q_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..01eb97fd4938e5d59da8c874cf777e50f1c5d6c3
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.9.self_attn.q_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2157749f3ba15c6cec11d3dea24d2759970fcd5e60c4593dfcd8344704ad2ef1
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.9.self_attn.q_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.9.self_attn.q_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5cefd0bc24cfb93a3923edf9fb2c8973e6df43a9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.9.self_attn.q_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9212083201846a12afa932d1b23226315e3e5336361eb68383e97dc70f6fb839
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.9.self_attn.q_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.9.self_attn.q_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d512266a46bba79012d35d58cd1924f5cb29863d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.9.self_attn.q_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0cb1e5fd5b90890be6b127555a502df21958c2b33a5c727e0c5714b970bc703e
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.9.self_attn.q_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.9.self_attn.q_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.9.self_attn.q_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.9.self_attn.v_proj.bias/exp_avg.pt b/global_step194526/_universal/zero/model.layers.9.self_attn.v_proj.bias/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..48eff86735cfae16c8c5301ca006e285ee078394
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.9.self_attn.v_proj.bias/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:080d76d8233acf94aedd55d637b6017baa706c417bc14f27deac7b3daf40bcb7
+size 2716
diff --git a/global_step194526/_universal/zero/model.layers.9.self_attn.v_proj.bias/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.9.self_attn.v_proj.bias/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..539639e24f7a36b43f7ed70e2853cb5a3cc8c29d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.9.self_attn.v_proj.bias/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2aede12cb7c32be565a9cbd841df06de7896ac2b44719e2b2bf51df9a995750e
+size 2731
diff --git a/global_step194526/_universal/zero/model.layers.9.self_attn.v_proj.bias/fp32.pt b/global_step194526/_universal/zero/model.layers.9.self_attn.v_proj.bias/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f4a67c3b97dc4f79c9707c7eab4ee321354bea5c
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.9.self_attn.v_proj.bias/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2ad75d8e79e4a0e6c0056e85314b29e0889043835e9531ec12e0c34054fcf0f5
+size 2637
diff --git a/global_step194526/_universal/zero/model.layers.9.self_attn.v_proj.bias/step.pt b/global_step194526/_universal/zero/model.layers.9.self_attn.v_proj.bias/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.9.self_attn.v_proj.bias/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.9.self_attn.v_proj.weight/exp_avg.pt b/global_step194526/_universal/zero/model.layers.9.self_attn.v_proj.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7bbe26108ab3e65e03459a323b989df4beec91ce
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.9.self_attn.v_proj.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9f269a20e03380dc97cbf4a28f08f2ce48180edd056c3aeeeadbc638287d1acc
+size 2950364
diff --git a/global_step194526/_universal/zero/model.layers.9.self_attn.v_proj.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.9.self_attn.v_proj.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..418e06aa1c99d5a2e16f2c3e5e829de9fa6e7239
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.9.self_attn.v_proj.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe51d109a76550b9f1837d0754b5157f500b92d759d07570665d6b5abd7fe283
+size 2950379
diff --git a/global_step194526/_universal/zero/model.layers.9.self_attn.v_proj.weight/fp32.pt b/global_step194526/_universal/zero/model.layers.9.self_attn.v_proj.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8cb40e3f1ebf227a94744395f100b5808a839db9
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.9.self_attn.v_proj.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b4d7436708ec1a12d413195eb97d4a8fedaf1d4068a876b2fa9e5e16e26f292
+size 2950285
diff --git a/global_step194526/_universal/zero/model.layers.9.self_attn.v_proj.weight/step.pt b/global_step194526/_universal/zero/model.layers.9.self_attn.v_proj.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.9.self_attn.v_proj.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.layers.9.self_attn.v_proj_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.layers.9.self_attn.v_proj_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3359eb3255fd6ffb22962f44dafaec25d210b235
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.9.self_attn.v_proj_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:530ddbefcccee7e73484b93e8f2b4dd433b01120b8c6944f1af63f1eacb9ec11
+size 1180
diff --git a/global_step194526/_universal/zero/model.layers.9.self_attn.v_proj_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.layers.9.self_attn.v_proj_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f74c57b1ae65da833d88f3b553d03e984d2315df
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.9.self_attn.v_proj_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c887043ee08ff52d9b4eea6f94b670b7408f0a0a3526c00c3609475c5e3c5670
+size 1195
diff --git a/global_step194526/_universal/zero/model.layers.9.self_attn.v_proj_alpha/fp32.pt b/global_step194526/_universal/zero/model.layers.9.self_attn.v_proj_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..bfb8f2cd3c748383cdb77b9d6b8f395a59b4163d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.9.self_attn.v_proj_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3537c1d30bb1ed1279fb8e302ee38eee5eec13a9fb85a1f118caba2153fe384f
+size 1165
diff --git a/global_step194526/_universal/zero/model.layers.9.self_attn.v_proj_alpha/step.pt b/global_step194526/_universal/zero/model.layers.9.self_attn.v_proj_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.layers.9.self_attn.v_proj_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.norm.weight/exp_avg.pt b/global_step194526/_universal/zero/model.norm.weight/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..13fc999d5237bf17820d55d8de849b3456a3eb27
--- /dev/null
+++ b/global_step194526/_universal/zero/model.norm.weight/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0c0887aca8eb06bbf40a7ba62251db04fe004db69abbd9118f0e4336c2448ea5
+size 8860
diff --git a/global_step194526/_universal/zero/model.norm.weight/exp_avg_sq.pt b/global_step194526/_universal/zero/model.norm.weight/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b1e009bfa7270feb5276fe590ace5c34b6c509f5
--- /dev/null
+++ b/global_step194526/_universal/zero/model.norm.weight/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c657fcb919239192ea60c7c97cae6402db172c498c5cdc439358e24682b84517
+size 8875
diff --git a/global_step194526/_universal/zero/model.norm.weight/fp32.pt b/global_step194526/_universal/zero/model.norm.weight/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..26060ca08c6086589aed2eba167c3cf92de28027
--- /dev/null
+++ b/global_step194526/_universal/zero/model.norm.weight/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b3489fa5572e2fc66be7ab8d84b354b3fc69f09e18f9d41b139ae2fbb2b45439
+size 8781
diff --git a/global_step194526/_universal/zero/model.norm.weight/step.pt b/global_step194526/_universal/zero/model.norm.weight/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.norm.weight/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/model.norm_alpha/exp_avg.pt b/global_step194526/_universal/zero/model.norm_alpha/exp_avg.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9079d7a4c2d7e3d9201c0a37a87216f7f255bfcd
--- /dev/null
+++ b/global_step194526/_universal/zero/model.norm_alpha/exp_avg.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:074ac466b6940b8a69e5dc7b063afbc45434c0926319035d87484bad5541de35
+size 1180
diff --git a/global_step194526/_universal/zero/model.norm_alpha/exp_avg_sq.pt b/global_step194526/_universal/zero/model.norm_alpha/exp_avg_sq.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9a19d14bfd612c6deb6ff365959764d52e8c722d
--- /dev/null
+++ b/global_step194526/_universal/zero/model.norm_alpha/exp_avg_sq.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9359f665e780e19fcf622f93bf47cefe827c9d23773d77c19e4624c029cc0ae5
+size 1195
diff --git a/global_step194526/_universal/zero/model.norm_alpha/fp32.pt b/global_step194526/_universal/zero/model.norm_alpha/fp32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..bbb0951ff812ceab9f258aa6dc76c28b70b70e02
--- /dev/null
+++ b/global_step194526/_universal/zero/model.norm_alpha/fp32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:626e93ab0400ad233390a3a56870a47f7b69966c58252afe7e873de1edfdb580
+size 1165
diff --git a/global_step194526/_universal/zero/model.norm_alpha/step.pt b/global_step194526/_universal/zero/model.norm_alpha/step.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bab45d6272f6b32c58f8ddc876051f9ad1d2ed
--- /dev/null
+++ b/global_step194526/_universal/zero/model.norm_alpha/step.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98de81b139a3cfe7aafa6e330e41025546ce093c429bdcefae707145f15f29f
+size 852
diff --git a/global_step194526/_universal/zero/optimizer_state.pt b/global_step194526/_universal/zero/optimizer_state.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f743b4fabcce3fbfade2c1cae8ed646ac969b710
--- /dev/null
+++ b/global_step194526/_universal/zero/optimizer_state.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:96338ce820582ed28a33569a634202f79f59fbce6b829a32a04849508bf6d562
+size 1664
diff --git a/global_step194526/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/global_step194526/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5321c14fd3e0d054d085da3f2d99a4a48afd687d
--- /dev/null
+++ b/global_step194526/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:886322714a729090864129b07b72efb52b031b08ef4b9dc90e9b45beb7f1d5ce
+size 558554482
diff --git a/global_step194526/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt b/global_step194526/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3708717163018abc9f7c0f0b60594be91924ebd2
--- /dev/null
+++ b/global_step194526/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4152bf862de11596b2077bf9566743b0a5080b380c6083d3f1a9ce3452dd18cd
+size 558554306
diff --git a/global_step194526/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt b/global_step194526/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8e907d343effea4e63fdab62e1c264ffa9aa97f1
--- /dev/null
+++ b/global_step194526/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e0209b44f0241483a3de4beeebc1b6c002291cc0b2bf8e45c43ee1ea768cc543
+size 558554306
diff --git a/global_step194526/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt b/global_step194526/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..555c340730ce193e393ec71cb1bac9643c08bfe4
--- /dev/null
+++ b/global_step194526/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ba8350f3207e3c77d62396a62dd6eed55780fbab3d0c557241d4fda32b7797fa
+size 558554370
diff --git a/global_step194526/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt b/global_step194526/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9b080ad2461640e7dae37ecbd95090ea8a2fca85
--- /dev/null
+++ b/global_step194526/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1cf197ffa06f13d271062ce806a0586e481f764b3dcbabc05449c7b504dc20a5
+size 558554434
diff --git a/global_step194526/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt b/global_step194526/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5d322c252500826f2c023e26cb622c3190eb647d
--- /dev/null
+++ b/global_step194526/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ff32be0e74d41ca4fbc0babc646feac2e4a4f16f8e5f03c922a136d4a2f57ba2
+size 558554306
diff --git a/global_step194526/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt b/global_step194526/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fc26f886ce0bbf9500c7acdddd770e1f70d9a1e5
--- /dev/null
+++ b/global_step194526/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:037a31e7e721f8a664723b6774dad4bce4f0bd92590bc2c73f004ef7b8774d50
+size 558554370
diff --git a/global_step194526/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt b/global_step194526/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..88513125bd268506158ca309c9918dc8c6aec473
--- /dev/null
+++ b/global_step194526/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5c1b48efe0a99da31971a866d37c7c7a6a5a70968b73b997e1e160623fd4b711
+size 558554306
diff --git a/global_step194526/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt b/global_step194526/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d6152f5bb2a76c0271bb6cbf29c95eb98669098f
--- /dev/null
+++ b/global_step194526/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8b0297fc34822fa26a7b754c8a9995deb5b7e2c838847cdcadc7addb55473ff6
+size 558554306
diff --git a/global_step194526/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt b/global_step194526/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..587b57b83d5ce3afc5810a2aa71bc5159ed6c3c5
--- /dev/null
+++ b/global_step194526/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6c76a8607ba0157582922c17bae67f33e980b131518376abbaa09fe284d11008
+size 558554370
diff --git a/global_step194526/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt b/global_step194526/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3d6e367f040e0a74ca2c81dcd21b1572355db43a
--- /dev/null
+++ b/global_step194526/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d93b24940fc6a12fcff15595e36775b33d100a31dd9e3324c9673d2800d535dc
+size 558554434
diff --git a/global_step194526/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/global_step194526/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..15be5be6e498940affa089327c0ded4b59a0afc8
--- /dev/null
+++ b/global_step194526/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e439738bc4b403ac7473e8f00741eed7eacc38a90e6dcbf520012ff4004cb89e
+size 558554418
diff --git a/global_step194526/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt b/global_step194526/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..acce3f729f9f7b0eea3c7388c468084a04d784f6
--- /dev/null
+++ b/global_step194526/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cf509c7f13ad056a1657eb50a64c1711e4998a8175a89d0cc80f726c965218a9
+size 558554306
diff --git a/global_step194526/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt b/global_step194526/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..885bad33501d37e987a7b2f19d9ae76aeb13ba25
--- /dev/null
+++ b/global_step194526/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8a57187c0b28b3eaf2183066ebe406196d607a241a36ba33ddaf285ef3e39953
+size 558554370
diff --git a/global_step194526/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt b/global_step194526/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b6ca02f1127e7f3dc609679f8c8753d5e8ed2aca
--- /dev/null
+++ b/global_step194526/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:32ae9056c3fe7d40804483f80770d89719cc7881afe3d259181cdd645d7a7f15
+size 558554306
diff --git a/global_step194526/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt b/global_step194526/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b9e68231035f876f179ee7f3304ca38aa9761511
--- /dev/null
+++ b/global_step194526/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a03d9b9e653fad620707dc640766bef5e0fa39b2403a419fd239f049956effb8
+size 558554306
diff --git a/global_step194526/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt b/global_step194526/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6e43739d4b31ca06634e6d59a770e62b4e33121d
--- /dev/null
+++ b/global_step194526/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ebd113aac0be28cd50e966ed070b4e4d00c62446a866630de5c5570f99109486
+size 558554370
diff --git a/global_step194526/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt b/global_step194526/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..764804abdd64be837bafb92af270194cce4629b2
--- /dev/null
+++ b/global_step194526/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:418605082c1c52ff42d032b7215c009be7af7fbf2c2a149d26fc54c313d0cc6b
+size 558554434
diff --git a/global_step194526/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt b/global_step194526/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..01182f0c6184e5eede37d40397c156f358cd866f
--- /dev/null
+++ b/global_step194526/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:41c8c38ed1202b9e15bf670a71573e3a7a9490e1b283192eeb489c12d8984068
+size 558554306
diff --git a/global_step194526/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt b/global_step194526/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b2b3b6c5923c33220e921e7a4b236ab97dc52eb1
--- /dev/null
+++ b/global_step194526/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:da5e9569e283be8fa81a36357c1b6a989fcbbe45a72ae660a18ff0577eeeedb8
+size 558554370
diff --git a/global_step194526/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt b/global_step194526/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fdcc9fe13fc5b994ed71065dc4ab86b61c33a352
--- /dev/null
+++ b/global_step194526/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0ff9431d72677496b3cef352fe4cbbff040bb4103f5e1eb8d44a12f1ce0917cc
+size 558554306
diff --git a/global_step194526/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt b/global_step194526/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..348e6b845312026fa16e8d28f73a025e59c4de98
--- /dev/null
+++ b/global_step194526/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2d63346ecd6b6a4b74cae7f6d4fdf756efce047ac905cf52a28a630d9ef3562f
+size 558554306
diff --git a/global_step194526/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt b/global_step194526/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d2a575b0b240191d1ff94b4868696fcdc0475010
--- /dev/null
+++ b/global_step194526/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0c369b86e22c3d531c68e0886d9767be5e658c2ec8e4c71cdd13fcdc0b94b682
+size 558554290
diff --git a/global_step194526/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt b/global_step194526/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fac44c487d5ee02c38d6000cd362278b447c675b
--- /dev/null
+++ b/global_step194526/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b247dd273ad02db9de300b3d9ddd683e4e11cf4003888c0f51a900bf9ac7996c
+size 558554370
diff --git a/global_step194526/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt b/global_step194526/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6e8d875bd554431ac46b201610fb7e114dd8a38f
--- /dev/null
+++ b/global_step194526/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aff0dbafa1d60e887bf34d6dc5805bfd150fb4f0e69963c307ca7d5f6f0a3010
+size 558554434
diff --git a/global_step194526/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt b/global_step194526/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ac504a7feb2fa4b6d3d40ed853979b2f8428f5f9
--- /dev/null
+++ b/global_step194526/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:478f19d790343e3c97b347d071bc611bbfb0586165567867257f19751467efa5
+size 558554306
diff --git a/global_step194526/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt b/global_step194526/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..051bf8c6d910d1c5ca621a2a4a5d162076ca4e24
--- /dev/null
+++ b/global_step194526/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a70ecddd92ba249be54eca09bbe5958476103fd76d8915ab9302a6c2baf4fae8
+size 558554370
diff --git a/global_step194526/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt b/global_step194526/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2c706e687698b6278d2c1e59d106cfbd6647c1ed
--- /dev/null
+++ b/global_step194526/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e9c1614eb63c35882cd5aed2c26a744c7cde1470899d79d158b81b23563fd02c
+size 558554306
diff --git a/global_step194526/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt b/global_step194526/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..871340c229d899590d48620188cbdf88bdef4b82
--- /dev/null
+++ b/global_step194526/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aa9ac1dc6838f481f651f58ee2caf3537dd0666a2ee00d0ad54728b737a244a9
+size 558554306
diff --git a/global_step194526/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt b/global_step194526/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5747e9daecd091b898408cdc2e8e2e71e180d796
--- /dev/null
+++ b/global_step194526/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:59e6a1aa3deb3335242f1a78606a1200a09288349bf51c9cad727b8884775b5c
+size 558554370
diff --git a/global_step194526/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt b/global_step194526/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..baa1bb84f9b69fbbeedda010c7d853eadf0047ff
--- /dev/null
+++ b/global_step194526/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a35eb64ceea30542105651af7e57b3378e8d1bb81813b8db310de1ee0556a901
+size 558554434
diff --git a/global_step194526/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt b/global_step194526/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..10296a8b15daf66809153eebb84cd5616b825590
--- /dev/null
+++ b/global_step194526/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:72465897c3ad591af2414d06ff2ff917a64e26242e8a42b89423a4895caaa3d0
+size 558554306
diff --git a/global_step194526/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt b/global_step194526/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6a6e994e604d9ba165fb0e997306475ddf7c4e37
--- /dev/null
+++ b/global_step194526/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d31f495807897a192839c196036a16ebbd1ae5330ed1d9bfd1ad73871960b289
+size 558554370
diff --git a/global_step194526/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt b/global_step194526/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..457435d815e47c2acae3c0e1525ffd336a4dfb2b
--- /dev/null
+++ b/global_step194526/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2c642b6678390ae79a26b3599560c0c7110202fba3b8a6553e73b0c5223c9095
+size 558554354
diff --git a/global_step194526/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt b/global_step194526/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fe1afc66c1cfb9178ed29585c4bf16ea2c3bbb5e
--- /dev/null
+++ b/global_step194526/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bad59af8006c2d499f35d6a6930bdb6ae2fad260483fc3420969e78abaf8254a
+size 558554306
diff --git a/global_step194526/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt b/global_step194526/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c05b215c9a03e38cc4b7520b4e53812e66a650a9
--- /dev/null
+++ b/global_step194526/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4976bab64e1d3a9fc3cd556d6bfaade21ec6f4ef87f6eb50aff1e2666f75aa48
+size 558554306
diff --git a/global_step194526/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt b/global_step194526/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..dee459e826608fde10f7d8a680d9ff17de48ec83
--- /dev/null
+++ b/global_step194526/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:15795a23cf8657ec2b349b783cae783ae425304d997b9fba290405e1423c4ba7
+size 558554370
diff --git a/global_step194526/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt b/global_step194526/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ffe9d33dbb55b7379c1a7b7bbcec1756bcd9f7f2
--- /dev/null
+++ b/global_step194526/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5dc390355bba3d710f2efefbfb204ffb28c69bef68363f42bb700521a5b90ffd
+size 558554434
diff --git a/global_step194526/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt b/global_step194526/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..eeac428c59ce7acb607b93892e625f455ef7d14f
--- /dev/null
+++ b/global_step194526/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cedaa87e3b5bb79c46d554432b44aebea2e14996dda950327045ad87ab3c908f
+size 558554306
diff --git a/global_step194526/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt b/global_step194526/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0c58ab6c35b6666dedce9a1bf2b915939096fad0
--- /dev/null
+++ b/global_step194526/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:68f9f3b782cbca61f7ddc2e6bf2dbe8f04356d5dc657dbe9b3c7101349f2f375
+size 558554370
diff --git a/global_step194526/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt b/global_step194526/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7669c3de9f58812635231ca5b518acb10818d8e6
--- /dev/null
+++ b/global_step194526/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:962df37a8623bbe0c2bff69030992dfa094cec57a699d29c3be0e5ead761da5e
+size 558554306
diff --git a/global_step194526/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt b/global_step194526/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d48c3d82ccb76ad9944a7edea7f126a417a3b656
--- /dev/null
+++ b/global_step194526/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:efb69b8fd57e3f4ffc248b0cec773689b888f3639a2f05d9ad10f8441bee79bd
+size 558610626
diff --git a/global_step194526/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt b/global_step194526/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ce0f0e08446397bf0055a35e155801350075d2ca
--- /dev/null
+++ b/global_step194526/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7c8afdf335469a164ee6dc71adfbbbb2183388ed5507fd8a0c32a9c0d654e4ff
+size 558554290
diff --git a/global_step194526/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt b/global_step194526/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..634e5a8c6a3fd41a89abfb0264169774e08fdab6
--- /dev/null
+++ b/global_step194526/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4070e0c2ec006b827345d2772f7291e2f137362f351a413f0a1fc45095af689f
+size 558554290
diff --git a/global_step194526/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt b/global_step194526/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..70ce633a729dea2129ff8ed04190d6a10c2a3ab3
--- /dev/null
+++ b/global_step194526/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c16df19b4e5364d630cf6215ce2ea5973fe9b787a5333931e2bd76c9afb6858e
+size 558554354
diff --git a/global_step194526/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt b/global_step194526/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3afa7cffe14ff3619c14af8a6e42822aa08b595a
--- /dev/null
+++ b/global_step194526/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a53a3b907743b0cb02147e64df67c82b0faa737dedcc65681d97617797f56f7d
+size 558554418
diff --git a/global_step194526/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt b/global_step194526/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2f46f0177601b2aa4ee12351cbbca5cd73f13e59
--- /dev/null
+++ b/global_step194526/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3302d5a788dd96f8daa50890fc946f62440e5f7020864023169791cc1afc4412
+size 558554290
diff --git a/global_step194526/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt b/global_step194526/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d6ed44f432b59f3a385f2e49409aa04a6b49cd2c
--- /dev/null
+++ b/global_step194526/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:96c72d64024a79b6b07e611e5e66badf210122b68cf8205ee24d489199426663
+size 558554354
diff --git a/global_step194526/latest_universal b/global_step194526/latest_universal
new file mode 100644
index 0000000000000000000000000000000000000000..67b76d9e890df41a113d2ca47071d30d1c501daa
--- /dev/null
+++ b/global_step194526/latest_universal
@@ -0,0 +1 @@
+_universal
\ No newline at end of file
diff --git a/global_step194526/mp_rank_00_model_states.pt b/global_step194526/mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3951d6e6a69d875eac133f13760e5359d93d7ad5
--- /dev/null
+++ b/global_step194526/mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:85186ad6e40f2b727e0fdd1a1db6e2ebc9cfe7b6c6ad89f5fc521017fe415fda
+size 4468641136