diff --git a/1b_layer0_mlp.down_proj_input_pretrain_representation.pth b/1b_layer0_mlp.down_proj_input_pretrain_representation.pth
new file mode 100644
index 0000000000000000000000000000000000000000..96492fd9ef4083df694644e7b26ff50718bf636f
--- /dev/null
+++ b/1b_layer0_mlp.down_proj_input_pretrain_representation.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fa3d10570fd39f5790dfdb51ad9b493d1c914ccd2cd3c38366aab49963c1017e
+size 266798594
diff --git a/1b_layer0_self_attn.o_proj_input_pretrain_representation.pth b/1b_layer0_self_attn.o_proj_input_pretrain_representation.pth
new file mode 100644
index 0000000000000000000000000000000000000000..f4e9f0d6956478a011f4d76a1a0456151c6fd547
--- /dev/null
+++ b/1b_layer0_self_attn.o_proj_input_pretrain_representation.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d5c9ac9ae9716701ae1d66328671f5fb600fd5a16e49064df7323823eb40569e
+size 66700881
diff --git a/1b_layer10_self_attn.o_proj_input_pretrain_representation.pth b/1b_layer10_self_attn.o_proj_input_pretrain_representation.pth
new file mode 100644
index 0000000000000000000000000000000000000000..cba5fa1988cd483f779d3c50f17b8762decc31ca
--- /dev/null
+++ b/1b_layer10_self_attn.o_proj_input_pretrain_representation.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3d2e8eed0869e3a197cf1af2d8cfc6f1b72f97c21104364e7cfdb27304b0677a
+size 66700886
diff --git a/1b_layer10_self_attn.v_proj_output_pretrain_representation.pth b/1b_layer10_self_attn.v_proj_output_pretrain_representation.pth
new file mode 100644
index 0000000000000000000000000000000000000000..06b8b9642fca72e2d97d53ec1f8e67eb5ff752cc
--- /dev/null
+++ b/1b_layer10_self_attn.v_proj_output_pretrain_representation.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d77ba4d561aa059c4d38e90441bea1faba3bcf1b9390c48e743f5431c69c387f
+size 16676443
diff --git a/1b_layer11_mlp.down_proj_output_pretrain_representation.pth b/1b_layer11_mlp.down_proj_output_pretrain_representation.pth
new file mode 100644
index 0000000000000000000000000000000000000000..b5aeac4408f010d8f8a9ae4c0c22921c4337e8fe
--- /dev/null
+++ b/1b_layer11_mlp.down_proj_output_pretrain_representation.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:06efd48332cd284922166dbce3147175b0840a6244ec249a594f5356f9d0a4a6
+size 66700876
diff --git a/1b_layer11_self_attn.o_proj_output_pretrain_representation.pth b/1b_layer11_self_attn.o_proj_output_pretrain_representation.pth
new file mode 100644
index 0000000000000000000000000000000000000000..f7a414ad0aad213cc146ddae04bdf55016e786c6
--- /dev/null
+++ b/1b_layer11_self_attn.o_proj_output_pretrain_representation.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:531c5adb40f59c60e1e3ea527e96b57024768c91bc541ea6c363f607e20a215e
+size 66700891
diff --git a/1b_layer11_self_attn.q_proj_output_pretrain_representation.pth b/1b_layer11_self_attn.q_proj_output_pretrain_representation.pth
new file mode 100644
index 0000000000000000000000000000000000000000..02e0beff3c06c992b9a20f40665bb3672b73274a
--- /dev/null
+++ b/1b_layer11_self_attn.q_proj_output_pretrain_representation.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4b3f50e903b2e071b61fdb64b9071dd72ce0c6fbd8550a89250633a487fe2822
+size 66700891
diff --git a/1b_layer12_mlp.up_proj_output_pretrain_representation.pth b/1b_layer12_mlp.up_proj_output_pretrain_representation.pth
new file mode 100644
index 0000000000000000000000000000000000000000..95d0a933fac856a20d9dd07ccb82b6f980a01eeb
--- /dev/null
+++ b/1b_layer12_mlp.up_proj_output_pretrain_representation.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1bf06125f9c92ed604ebb1c7862898b437bb35ce29814e850ae555c26ccdd74d
+size 266798594
diff --git a/1b_layer13_mlp.gate_proj_output_pretrain_representation.pth b/1b_layer13_mlp.gate_proj_output_pretrain_representation.pth
new file mode 100644
index 0000000000000000000000000000000000000000..0dcbc033b4f7b1c1d921c566db92c5a14cfd6e5a
--- /dev/null
+++ b/1b_layer13_mlp.gate_proj_output_pretrain_representation.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea74193c6891104e1e612126e7ac87a351fabd373b9e700f7d6ead24a0785f11
+size 266798668
diff --git a/1b_layer13_self_attn.k_proj_input_pretrain_representation.pth b/1b_layer13_self_attn.k_proj_input_pretrain_representation.pth
new file mode 100644
index 0000000000000000000000000000000000000000..49de17c2e5ae51f1e290e88b6ad349073d692072
--- /dev/null
+++ b/1b_layer13_self_attn.k_proj_input_pretrain_representation.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2e08f31fdaf22c034e4945c0e5b2a50435276a2eee35cf73146457be135b11e7
+size 66700886
diff --git a/1b_layer13_self_attn.o_proj_input_pretrain_representation.pth b/1b_layer13_self_attn.o_proj_input_pretrain_representation.pth
new file mode 100644
index 0000000000000000000000000000000000000000..e9f05a4f6ff82d489a0055b9ec48e7db77c83172
--- /dev/null
+++ b/1b_layer13_self_attn.o_proj_input_pretrain_representation.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a52c72c4158a5ab51f6ecad73487162dbe00c3fba46b42f504f983ea8cebd73e
+size 66700886
diff --git a/1b_layer13_self_attn.o_proj_output_pretrain_representation.pth b/1b_layer13_self_attn.o_proj_output_pretrain_representation.pth
new file mode 100644
index 0000000000000000000000000000000000000000..4522b7de5324bdc6e6aa58075f56a252efe8c653
--- /dev/null
+++ b/1b_layer13_self_attn.o_proj_output_pretrain_representation.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2563804ac69879a097ccab3b43192845960e20ffd1f7a644d86ae33f42e143bf
+size 66700891
diff --git a/1b_layer14_mlp.up_proj_input_pretrain_representation.pth b/1b_layer14_mlp.up_proj_input_pretrain_representation.pth
new file mode 100644
index 0000000000000000000000000000000000000000..135d51cf7b8fb16f8d54a7526d2139f16c8812e6
--- /dev/null
+++ b/1b_layer14_mlp.up_proj_input_pretrain_representation.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6c5c78fa4ac600102c7f3062bfbd4d8e3e0ab34b4c3fcbd5bc0bfc3fb65b9213
+size 66700797
diff --git a/1b_layer14_self_attn.o_proj_output_pretrain_representation.pth b/1b_layer14_self_attn.o_proj_output_pretrain_representation.pth
new file mode 100644
index 0000000000000000000000000000000000000000..561d5007cfc447af3bfa18bf14f68aa529b3d462
--- /dev/null
+++ b/1b_layer14_self_attn.o_proj_output_pretrain_representation.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d607d26873fd84cbcf123ad9b649bc516778de8a57a7a7b13465f45f0262335
+size 66700891
diff --git a/1b_layer15_mlp.gate_proj_output_pretrain_representation.pth b/1b_layer15_mlp.gate_proj_output_pretrain_representation.pth
new file mode 100644
index 0000000000000000000000000000000000000000..8d9ddf5b30fe5f9ae20b2c743022439ed1a55ac2
--- /dev/null
+++ b/1b_layer15_mlp.gate_proj_output_pretrain_representation.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:919f2de693262560d56ef8d62b9f671c15c88e3a8e96d8f0a2ce29e40e272637
+size 266798668
diff --git a/1b_layer1_mlp.up_proj_input_pretrain_representation.pth b/1b_layer1_mlp.up_proj_input_pretrain_representation.pth
new file mode 100644
index 0000000000000000000000000000000000000000..010688ffbef3351b11b929ef5f4e1a39a0bda888
--- /dev/null
+++ b/1b_layer1_mlp.up_proj_input_pretrain_representation.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:998422a2d6ec03226ecaca8bc3e88a54e5c77310fa00007348958af5c788900f
+size 66700792
diff --git a/1b_layer2_self_attn.v_proj_input_pretrain_representation.pth b/1b_layer2_self_attn.v_proj_input_pretrain_representation.pth
new file mode 100644
index 0000000000000000000000000000000000000000..0deb0f006b2ff767fde58ca322e29a5b30d5a13c
--- /dev/null
+++ b/1b_layer2_self_attn.v_proj_input_pretrain_representation.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d521e7da26096fb394a8e56bca93546630cb8152073d6d28c23e904e231cdaa6
+size 66700881
diff --git a/1b_layer3_mlp.gate_proj_output_pretrain_representation.pth b/1b_layer3_mlp.gate_proj_output_pretrain_representation.pth
new file mode 100644
index 0000000000000000000000000000000000000000..8e3b8eec98a891ebaa06c265e3bc345ce0e2ffe4
--- /dev/null
+++ b/1b_layer3_mlp.gate_proj_output_pretrain_representation.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:22da8a28539a96323700ff2fe4e7e399a3de138be49cc6ca660fc495bed9bb36
+size 266798663
diff --git a/1b_layer3_mlp.up_proj_output_pretrain_representation.pth b/1b_layer3_mlp.up_proj_output_pretrain_representation.pth
new file mode 100644
index 0000000000000000000000000000000000000000..99e7fab08b5805d7b9598f046b570f7bc152bb92
--- /dev/null
+++ b/1b_layer3_mlp.up_proj_output_pretrain_representation.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4755b23e9f7f03f15ae581234df6219d5ef264e153dcbed89a2b672138c9ee63
+size 266798589
diff --git a/1b_layer3_self_attn.o_proj_output_pretrain_representation.pth b/1b_layer3_self_attn.o_proj_output_pretrain_representation.pth
new file mode 100644
index 0000000000000000000000000000000000000000..b298dfd887a10cd39c0ccd4510a2dbc9012fb2de
--- /dev/null
+++ b/1b_layer3_self_attn.o_proj_output_pretrain_representation.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8ebf1e0de706d684bf1463c84d424d3222aec89ce8794dc3a4349e977c71b3d1
+size 66700886
diff --git a/1b_layer3_self_attn.q_proj_input_pretrain_representation.pth b/1b_layer3_self_attn.q_proj_input_pretrain_representation.pth
new file mode 100644
index 0000000000000000000000000000000000000000..c5ffe6e9c97175b12f4501aac36ce1873bee1357
--- /dev/null
+++ b/1b_layer3_self_attn.q_proj_input_pretrain_representation.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:164b5133ea703f53dd243d991b0dbd1b0b7117fb631318e787c529b77a390ecf
+size 66700881
diff --git a/1b_layer4_mlp.down_proj_output_pretrain_representation.pth b/1b_layer4_mlp.down_proj_output_pretrain_representation.pth
new file mode 100644
index 0000000000000000000000000000000000000000..d716d77e6da0d9abefa9230853a0040d18905716
--- /dev/null
+++ b/1b_layer4_mlp.down_proj_output_pretrain_representation.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:47b52104ec010414ad54e95b9963fbd0058799fa2479b9baf64f753a5d97d17f
+size 66700871
diff --git a/1b_layer4_self_attn.k_proj_output_pretrain_representation.pth b/1b_layer4_self_attn.k_proj_output_pretrain_representation.pth
new file mode 100644
index 0000000000000000000000000000000000000000..7e8add8c7669143a910a1cc037fa43011677ec5d
--- /dev/null
+++ b/1b_layer4_self_attn.k_proj_output_pretrain_representation.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:77fc5c5a12f4c1f5e3b840f866db1df2153e5fa5a5fb5a63b462002533fbbd85
+size 16676438
diff --git a/1b_layer4_self_attn.q_proj_output_pretrain_representation.pth b/1b_layer4_self_attn.q_proj_output_pretrain_representation.pth
new file mode 100644
index 0000000000000000000000000000000000000000..2ded17f8860e918da4026eb349a26dca5d99944f
--- /dev/null
+++ b/1b_layer4_self_attn.q_proj_output_pretrain_representation.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:603bd5bc3303939a2a8f4bb700c7f98f583c89e7aec18bda97fc4c58258d8d9b
+size 66700886
diff --git a/1b_layer4_self_attn.v_proj_output_pretrain_representation.pth b/1b_layer4_self_attn.v_proj_output_pretrain_representation.pth
new file mode 100644
index 0000000000000000000000000000000000000000..317707b78680b0155885603216a45fb6cbd40428
--- /dev/null
+++ b/1b_layer4_self_attn.v_proj_output_pretrain_representation.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ab7643ac51fdae811b3d0c4d34c3e1e981a594b27165c63856026a7526ce710e
+size 16676438
diff --git a/1b_layer5_mlp.up_proj_output_pretrain_representation.pth b/1b_layer5_mlp.up_proj_output_pretrain_representation.pth
new file mode 100644
index 0000000000000000000000000000000000000000..5495986fdac45944b4d4a5200797e54b33f0d757
--- /dev/null
+++ b/1b_layer5_mlp.up_proj_output_pretrain_representation.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fad78085e4e4e91909c368549b10c733b098ea12599bae468133f388ac958d69
+size 266798589
diff --git a/1b_layer5_self_attn.k_proj_input_pretrain_representation.pth b/1b_layer5_self_attn.k_proj_input_pretrain_representation.pth
new file mode 100644
index 0000000000000000000000000000000000000000..a7536209138b7e498f87182ba44573ca5193e121
--- /dev/null
+++ b/1b_layer5_self_attn.k_proj_input_pretrain_representation.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:21e2826928d2f35e89ba2f7f2aa9199b105c62334a1bb1d42af24cadc32ccd72
+size 66700881
diff --git a/1b_layer5_self_attn.o_proj_input_pretrain_representation.pth b/1b_layer5_self_attn.o_proj_input_pretrain_representation.pth
new file mode 100644
index 0000000000000000000000000000000000000000..ee4d24097872066c24b4e66b38ebe4a42d1abee2
--- /dev/null
+++ b/1b_layer5_self_attn.o_proj_input_pretrain_representation.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:91a3c246c5ab4349d56be6d8df3c616e4cef503721c2f01bb85487ccb81cf1a6
+size 66700881
diff --git a/1b_layer6_self_attn.k_proj_input_pretrain_representation.pth b/1b_layer6_self_attn.k_proj_input_pretrain_representation.pth
new file mode 100644
index 0000000000000000000000000000000000000000..10a7040dca4d9f462fc7424c3c7753cccbf602e8
--- /dev/null
+++ b/1b_layer6_self_attn.k_proj_input_pretrain_representation.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3b4597dff9b9340ff1f36c45e9a1c1324519b6b2d8b6f09e0e47237e5c6fc3fd
+size 66700881
diff --git a/1b_layer7_self_attn.k_proj_input_pretrain_representation.pth b/1b_layer7_self_attn.k_proj_input_pretrain_representation.pth
new file mode 100644
index 0000000000000000000000000000000000000000..0e702df84fef3a4908ac55aa7728d53b719fe469
--- /dev/null
+++ b/1b_layer7_self_attn.k_proj_input_pretrain_representation.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b0464f3233944b43345de7996eea520041ee9a76b6f8921a3b4d2fb7f80a5a5c
+size 66700881
diff --git a/1b_layer7_self_attn.k_proj_output_pretrain_representation.pth b/1b_layer7_self_attn.k_proj_output_pretrain_representation.pth
new file mode 100644
index 0000000000000000000000000000000000000000..8f35510d673d0b31b8ca6e7beec263029c6a8143
--- /dev/null
+++ b/1b_layer7_self_attn.k_proj_output_pretrain_representation.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aeb8de1b019de22c148364c3b89daffd0e3b11f8ab4a09876c6b0a13ee7ecd26
+size 16676438
diff --git a/1b_layer7_self_attn.v_proj_input_pretrain_representation.pth b/1b_layer7_self_attn.v_proj_input_pretrain_representation.pth
new file mode 100644
index 0000000000000000000000000000000000000000..386ec285ddca1624d984bd61d4578d2a340588f3
--- /dev/null
+++ b/1b_layer7_self_attn.v_proj_input_pretrain_representation.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b27dd2eb792d226f40ae7ad09a3f6c2e04cb1a7f8a12ce631bb32d1df48613e2
+size 66700881
diff --git a/1b_layer8_mlp.gate_proj_input_pretrain_representation.pth b/1b_layer8_mlp.gate_proj_input_pretrain_representation.pth
new file mode 100644
index 0000000000000000000000000000000000000000..4de089986d2ede93a25cf480f0884c350b47c05e
--- /dev/null
+++ b/1b_layer8_mlp.gate_proj_input_pretrain_representation.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2ae151c3664616bea7ee4fbbd398b3cbaab22f3dbbed99919a24eec7e9f9d48a
+size 66700802
diff --git a/1b_layer8_self_attn.o_proj_input_pretrain_representation.pth b/1b_layer8_self_attn.o_proj_input_pretrain_representation.pth
new file mode 100644
index 0000000000000000000000000000000000000000..33731f4db9c3805cfe62699ee8fdebe694a75c8c
--- /dev/null
+++ b/1b_layer8_self_attn.o_proj_input_pretrain_representation.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ab74d7d32bedfb0fb411094fa29fd429d1e10d02df137159ee33463603065423
+size 66700881
diff --git a/1b_layer8_self_attn.q_proj_output_pretrain_representation.pth b/1b_layer8_self_attn.q_proj_output_pretrain_representation.pth
new file mode 100644
index 0000000000000000000000000000000000000000..8e74198513b4ae8865b85a21b9f77587d0afe163
--- /dev/null
+++ b/1b_layer8_self_attn.q_proj_output_pretrain_representation.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1c9a0156b51dacd9c14bf3fb42f3652599cfb0b67e09c66c4e8909381364d009
+size 66700886
diff --git a/1b_layer8_self_attn.v_proj_input_pretrain_representation.pth b/1b_layer8_self_attn.v_proj_input_pretrain_representation.pth
new file mode 100644
index 0000000000000000000000000000000000000000..e4582c6d0a77c02cf133e85c30732aa95a5dd5c3
--- /dev/null
+++ b/1b_layer8_self_attn.v_proj_input_pretrain_representation.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e2daec00902f80bc26e18ddf887a47c4f7a600a4fe8a6a7bc45a4f00299b77ee
+size 66700881
diff --git a/1b_layer8_self_attn.v_proj_output_pretrain_representation.pth b/1b_layer8_self_attn.v_proj_output_pretrain_representation.pth
new file mode 100644
index 0000000000000000000000000000000000000000..304431fc5fb3b5b3b158822edd28b5e1d4e9c4d3
--- /dev/null
+++ b/1b_layer8_self_attn.v_proj_output_pretrain_representation.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6698c0338da014d9235bf6659deff861235763deedcc5123caf603513be0b036
+size 16676438
diff --git a/1b_layer9_self_attn.o_proj_input_pretrain_representation.pth b/1b_layer9_self_attn.o_proj_input_pretrain_representation.pth
new file mode 100644
index 0000000000000000000000000000000000000000..050bff6dc7e61432b00e11681bf50bcefec5826e
--- /dev/null
+++ b/1b_layer9_self_attn.o_proj_input_pretrain_representation.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c27a2791d7c7081b00bc9fce5ef38759371e847b2dbccaeee7ed949143b61ddd
+size 66700881
diff --git a/3b_layer0_mlp.down_proj_output_pretrain_representation.pth b/3b_layer0_mlp.down_proj_output_pretrain_representation.pth
new file mode 100644
index 0000000000000000000000000000000000000000..72b9f111e1dba2cfa74253fef5709b310a093410
--- /dev/null
+++ b/3b_layer0_mlp.down_proj_output_pretrain_representation.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dbc0c6217c790d30ed63395d7b01850e3b793c8a78a2d0a1badddc4a77272633
+size 133391943
diff --git a/3b_layer0_self_attn.v_proj_output_pretrain_representation.pth b/3b_layer0_self_attn.v_proj_output_pretrain_representation.pth
new file mode 100644
index 0000000000000000000000000000000000000000..17a4b8bab903d074b8e06844ef95741d1ff0d364
--- /dev/null
+++ b/3b_layer0_self_attn.v_proj_output_pretrain_representation.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e3aced4d600548358fabfb1496c44b4ec95cb069b04ad5b41c6930f3c467ddc0
+size 33349206
diff --git a/3b_layer10_self_attn.o_proj_input_pretrain_representation.pth b/3b_layer10_self_attn.o_proj_input_pretrain_representation.pth
new file mode 100644
index 0000000000000000000000000000000000000000..aa9c956c67b9b30049626da6fa0379015b68c896
--- /dev/null
+++ b/3b_layer10_self_attn.o_proj_input_pretrain_representation.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:db9783642606625764b7ed1477f41f57bd4190f3d68d04e3ea00a1a0048a9ee5
+size 133391958
diff --git a/3b_layer11_mlp.down_proj_input_pretrain_representation.pth b/3b_layer11_mlp.down_proj_input_pretrain_representation.pth
new file mode 100644
index 0000000000000000000000000000000000000000..63aa634d4f08baabf157f99c4a8f61aa1a893624
--- /dev/null
+++ b/3b_layer11_mlp.down_proj_input_pretrain_representation.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e47bb30215ebd7c91878cb3a8395f5268e4764b0171758c138bb4fe7fa678ede
+size 533562951
diff --git a/3b_layer11_self_attn.k_proj_output_pretrain_representation.pth b/3b_layer11_self_attn.k_proj_output_pretrain_representation.pth
new file mode 100644
index 0000000000000000000000000000000000000000..aa286c9c4c1b6ef2378db869c6133304872680d6
--- /dev/null
+++ b/3b_layer11_self_attn.k_proj_output_pretrain_representation.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6394bf6649088b10c4dbfffd233d10b148ae4a4d21356fe6678c574f4e2e51e5
+size 33349211
diff --git a/3b_layer12_mlp.down_proj_output_pretrain_representation.pth b/3b_layer12_mlp.down_proj_output_pretrain_representation.pth
new file mode 100644
index 0000000000000000000000000000000000000000..3be00406efcefe5421247cba31f29301457d1d4d
--- /dev/null
+++ b/3b_layer12_mlp.down_proj_output_pretrain_representation.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:59732b6846f8f2c96de6eab0226e1865db6a98bf935bc367366e65663be30d9e
+size 133391948
diff --git a/3b_layer12_self_attn.o_proj_input_pretrain_representation.pth b/3b_layer12_self_attn.o_proj_input_pretrain_representation.pth
new file mode 100644
index 0000000000000000000000000000000000000000..e62495b50a9fe7a17130aafbe12f15bfed6daaa3
--- /dev/null
+++ b/3b_layer12_self_attn.o_proj_input_pretrain_representation.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:30f0e6b666e6fa119f442b90f3a995abae0ca1ee4150a951b1ab9bea91126c65
+size 133391958
diff --git a/3b_layer13_mlp.gate_proj_input_pretrain_representation.pth b/3b_layer13_mlp.gate_proj_input_pretrain_representation.pth
new file mode 100644
index 0000000000000000000000000000000000000000..cfcb189eac8cfbbba770c44e65d5c7153a258141
--- /dev/null
+++ b/3b_layer13_mlp.gate_proj_input_pretrain_representation.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d8a521e261fbf05e70fa8224100122d9a5096975dc6244b9d9d6a6cf43c1c31f
+size 133391943
diff --git a/3b_layer13_self_attn.k_proj_input_pretrain_representation.pth b/3b_layer13_self_attn.k_proj_input_pretrain_representation.pth
new file mode 100644
index 0000000000000000000000000000000000000000..822493827958a2e53cabf3702c92bc685adab89a
--- /dev/null
+++ b/3b_layer13_self_attn.k_proj_input_pretrain_representation.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:56a7cb64a53e2ac573c0caab7c8d8933482acfda09ae824ab0b8f639bed7b7f6
+size 133391958
diff --git a/3b_layer13_self_attn.k_proj_output_pretrain_representation.pth b/3b_layer13_self_attn.k_proj_output_pretrain_representation.pth
new file mode 100644
index 0000000000000000000000000000000000000000..d767c6e8df3bea05c13a39c919581a16b284cb52
--- /dev/null
+++ b/3b_layer13_self_attn.k_proj_output_pretrain_representation.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a7c40523e2c67eff044c9badc802c1d7c4e130d4ed822170b323a1d82035b89
+size 33349211
diff --git a/3b_layer13_self_attn.o_proj_output_pretrain_representation.pth b/3b_layer13_self_attn.o_proj_output_pretrain_representation.pth
new file mode 100644
index 0000000000000000000000000000000000000000..a69a79a3c76fa9fdea03dfab32a247d75b4afc5a
--- /dev/null
+++ b/3b_layer13_self_attn.o_proj_output_pretrain_representation.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:04c05698cc5766f0822eaece6821983a2cd43413935e2dffa729a09e56b389ab
+size 133391963
diff --git a/3b_layer13_self_attn.v_proj_input_pretrain_representation.pth b/3b_layer13_self_attn.v_proj_input_pretrain_representation.pth
new file mode 100644
index 0000000000000000000000000000000000000000..ff1ef6fe32010978b5ea8ec9dfb51745654d58c6
--- /dev/null
+++ b/3b_layer13_self_attn.v_proj_input_pretrain_representation.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:906d9ff80960db4c8949f3ce60f83e9f9d7ce150d80ce344e491f8956dfc1e02
+size 133391958
diff --git a/3b_layer14_mlp.gate_proj_output_pretrain_representation.pth b/3b_layer14_mlp.gate_proj_output_pretrain_representation.pth
new file mode 100644
index 0000000000000000000000000000000000000000..5fa137d9c53782de3b81192cca0ef8b0b2371ef9
--- /dev/null
+++ b/3b_layer14_mlp.gate_proj_output_pretrain_representation.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7affea8c8708a41624cbec521184e4a47d3441b919e5e08404357448316f3bfa
+size 533562956
diff --git a/3b_layer15_self_attn.q_proj_input_pretrain_representation.pth b/3b_layer15_self_attn.q_proj_input_pretrain_representation.pth
new file mode 100644
index 0000000000000000000000000000000000000000..98bdbb2a531c0020dd591e1c0f0113c633695107
--- /dev/null
+++ b/3b_layer15_self_attn.q_proj_input_pretrain_representation.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6c75b7fd811b9609a4d9351c4af043f23e5051f342ec006d65bcaef7edd526ca
+size 133391958
diff --git a/3b_layer15_self_attn.v_proj_output_pretrain_representation.pth b/3b_layer15_self_attn.v_proj_output_pretrain_representation.pth
new file mode 100644
index 0000000000000000000000000000000000000000..fa3ab0ec83d970c11abb2b13878cc543d8ac8d10
--- /dev/null
+++ b/3b_layer15_self_attn.v_proj_output_pretrain_representation.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e7eec64584f40f7e6de74971de493896ff64a3e23bebd768d4ddd42ed0e576c8
+size 33349211
diff --git a/3b_layer1_mlp.down_proj_output_pretrain_representation.pth b/3b_layer1_mlp.down_proj_output_pretrain_representation.pth
new file mode 100644
index 0000000000000000000000000000000000000000..4062c6b8105f5b13cb82115e367c2a1643f9e256
--- /dev/null
+++ b/3b_layer1_mlp.down_proj_output_pretrain_representation.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:33f9d30d6ee09d7192335985e7f65ba500082190b45b1bf719731febdb93d5e0
+size 133391943
diff --git a/3b_layer1_mlp.gate_proj_input_pretrain_representation.pth b/3b_layer1_mlp.gate_proj_input_pretrain_representation.pth
new file mode 100644
index 0000000000000000000000000000000000000000..4ad242f382aed572e6eb1f62e99b1fd3148055b1
--- /dev/null
+++ b/3b_layer1_mlp.gate_proj_input_pretrain_representation.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0ff58d67d640b264d6d8446467e9ea49dd296aaa2bfd68dc0895dd9c8eec879c
+size 133391874
diff --git a/3b_layer1_self_attn.k_proj_input_pretrain_representation.pth b/3b_layer1_self_attn.k_proj_input_pretrain_representation.pth
new file mode 100644
index 0000000000000000000000000000000000000000..c4c4155ec825c3ce155117236b3c516d5943d055
--- /dev/null
+++ b/3b_layer1_self_attn.k_proj_input_pretrain_representation.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1bddb6fa92d0a971e1eda5a43526c10ad2bcc602d578b88102f21d460ba3e8cc
+size 133391953
diff --git a/3b_layer1_self_attn.o_proj_input_pretrain_representation.pth b/3b_layer1_self_attn.o_proj_input_pretrain_representation.pth
new file mode 100644
index 0000000000000000000000000000000000000000..d6dc1f7deaebf8f351d4ce443c185cca3e554976
--- /dev/null
+++ b/3b_layer1_self_attn.o_proj_input_pretrain_representation.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7451748a06585fb9757466ae37f99aacc6a40f7492f2ea3e6e6ff576fc5e1de3
+size 133391953
diff --git a/3b_layer1_self_attn.q_proj_output_pretrain_representation.pth b/3b_layer1_self_attn.q_proj_output_pretrain_representation.pth
new file mode 100644
index 0000000000000000000000000000000000000000..4ed5823b83ee5fcc2e79ad74f256a9b534f9a573
--- /dev/null
+++ b/3b_layer1_self_attn.q_proj_output_pretrain_representation.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:317ce498ee85f2e579d7f92a079a251027dac343655440a18d94529cfa86b07b
+size 133391958
diff --git a/3b_layer1_self_attn.v_proj_input_pretrain_representation.pth b/3b_layer1_self_attn.v_proj_input_pretrain_representation.pth
new file mode 100644
index 0000000000000000000000000000000000000000..eb67fa71be47e56668f6672328d41b96416de594
--- /dev/null
+++ b/3b_layer1_self_attn.v_proj_input_pretrain_representation.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c22952ff1168e96e59c42bc418e6ec9b58495c7622880f4b0474c5d89a95549d
+size 133391953
diff --git a/3b_layer2_mlp.gate_proj_output_pretrain_representation.pth b/3b_layer2_mlp.gate_proj_output_pretrain_representation.pth
new file mode 100644
index 0000000000000000000000000000000000000000..af0cb431fb35b196c0a0cf2cf440680968ed08be
--- /dev/null
+++ b/3b_layer2_mlp.gate_proj_output_pretrain_representation.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c5614229635646a2ddfdab72963b091fa344aefb742014525da2260d54c52558
+size 533562951
diff --git a/3b_layer2_self_attn.k_proj_input_pretrain_representation.pth b/3b_layer2_self_attn.k_proj_input_pretrain_representation.pth
new file mode 100644
index 0000000000000000000000000000000000000000..1a48f0273706fc802f923816607f11ea83fc3c41
--- /dev/null
+++ b/3b_layer2_self_attn.k_proj_input_pretrain_representation.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3e21025508959b068958587500aa3a5cd52eed589c7ab9b35fcf62450f53e0b5
+size 133391953
diff --git a/3b_layer2_self_attn.q_proj_output_pretrain_representation.pth b/3b_layer2_self_attn.q_proj_output_pretrain_representation.pth
new file mode 100644
index 0000000000000000000000000000000000000000..f5e90bdd4bc72f79c291c0c5b11284f625268768
--- /dev/null
+++ b/3b_layer2_self_attn.q_proj_output_pretrain_representation.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:25ef9364f93adef49d8774518af7ce47b6b1deb19a534a4c462c457caf57ad52
+size 133391958
diff --git a/3b_layer3_mlp.down_proj_output_pretrain_representation.pth b/3b_layer3_mlp.down_proj_output_pretrain_representation.pth
new file mode 100644
index 0000000000000000000000000000000000000000..61956a3a5df2edac7946b1a86cd093736279cba0
--- /dev/null
+++ b/3b_layer3_mlp.down_proj_output_pretrain_representation.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:55cf4c17d7f5407c4914e14475837447d4b82eeec80364b9a5afd78b891e6799
+size 133391943
diff --git a/3b_layer3_mlp.gate_proj_input_pretrain_representation.pth b/3b_layer3_mlp.gate_proj_input_pretrain_representation.pth
new file mode 100644
index 0000000000000000000000000000000000000000..74451dce2278f05ed7ed6b0fd60f1a2f09c86c59
--- /dev/null
+++ b/3b_layer3_mlp.gate_proj_input_pretrain_representation.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b796c5cf60e7c2114043b3f5588693931c22dee1a2be54a547a03704af7dbea9
+size 133391874
diff --git a/3b_layer3_self_attn.q_proj_input_pretrain_representation.pth b/3b_layer3_self_attn.q_proj_input_pretrain_representation.pth
new file mode 100644
index 0000000000000000000000000000000000000000..c3324def0d558db4948298623999227a78080ab0
--- /dev/null
+++ b/3b_layer3_self_attn.q_proj_input_pretrain_representation.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4c690855784a1641a131500e4dcf3e52ebdf6d8a108e027682361ac8dee19b38
+size 133391953
diff --git a/3b_layer3_self_attn.q_proj_output_pretrain_representation.pth b/3b_layer3_self_attn.q_proj_output_pretrain_representation.pth
new file mode 100644
index 0000000000000000000000000000000000000000..6e68aa345785deedcde1a34b84a3d63d903908ff
--- /dev/null
+++ b/3b_layer3_self_attn.q_proj_output_pretrain_representation.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9760f6c57cd6f80833ef18e7330886fe1182ebd4c86c346dc6f2f7528c861270
+size 133391958
diff --git a/3b_layer4_self_attn.q_proj_input_pretrain_representation.pth b/3b_layer4_self_attn.q_proj_input_pretrain_representation.pth
new file mode 100644
index 0000000000000000000000000000000000000000..2d4d1754ec5941b6862de52d562c0e50f067f930
--- /dev/null
+++ b/3b_layer4_self_attn.q_proj_input_pretrain_representation.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0ebfe2651cab2987f7b0ee572408c97fa52a10201d1de2eb25b5314acb593321
+size 133391953
diff --git a/3b_layer6_mlp.down_proj_input_pretrain_representation.pth b/3b_layer6_mlp.down_proj_input_pretrain_representation.pth
new file mode 100644
index 0000000000000000000000000000000000000000..4aefa8c166abb958a8a57960ce934b37081f8dca
--- /dev/null
+++ b/3b_layer6_mlp.down_proj_input_pretrain_representation.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:83a98c82957fe9342e8f07247a3a4ee0742e05ac3734f704bc53754792a8c14e
+size 533562882
diff --git a/3b_layer6_mlp.gate_proj_output_pretrain_representation.pth b/3b_layer6_mlp.gate_proj_output_pretrain_representation.pth
new file mode 100644
index 0000000000000000000000000000000000000000..c88b18184f3ecd7ee4bbe4e164ddcef51b94f99f
--- /dev/null
+++ b/3b_layer6_mlp.gate_proj_output_pretrain_representation.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bf21ccfb6f5a367ff4ee2e1aa22a67101eee4301ab9a3f3430e70d6d574c4114
+size 533562951
diff --git a/3b_layer6_self_attn.q_proj_input_pretrain_representation.pth b/3b_layer6_self_attn.q_proj_input_pretrain_representation.pth
new file mode 100644
index 0000000000000000000000000000000000000000..4ab89c4dbfd543ed1282670cfa654d1436571e63
--- /dev/null
+++ b/3b_layer6_self_attn.q_proj_input_pretrain_representation.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e04e287529b73177f337c4819ebdc446babba6f8a15abdd02a1c413b10c3492d
+size 133391953
diff --git a/3b_layer7_mlp.down_proj_output_pretrain_representation.pth b/3b_layer7_mlp.down_proj_output_pretrain_representation.pth
new file mode 100644
index 0000000000000000000000000000000000000000..86519c68c6f09155f0a7ab1bf26468e04404ba7b
--- /dev/null
+++ b/3b_layer7_mlp.down_proj_output_pretrain_representation.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1a9c68464b8f2c32d23b690d2696a465c0986b0ed21a8a5c98852f392d722f9b
+size 133391943
diff --git a/3b_layer7_self_attn.q_proj_input_pretrain_representation.pth b/3b_layer7_self_attn.q_proj_input_pretrain_representation.pth
new file mode 100644
index 0000000000000000000000000000000000000000..1b9eb723bbc897210b7753f9d27404a692b284be
--- /dev/null
+++ b/3b_layer7_self_attn.q_proj_input_pretrain_representation.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3363ccf9af262f5a408b0c923887c7214721e6212afa6e4476180e91c22f397e
+size 133391953
diff --git a/3b_layer7_self_attn.q_proj_output_pretrain_representation.pth b/3b_layer7_self_attn.q_proj_output_pretrain_representation.pth
new file mode 100644
index 0000000000000000000000000000000000000000..7892e7481bbeb1853bb4763a68ea6d653975872a
--- /dev/null
+++ b/3b_layer7_self_attn.q_proj_output_pretrain_representation.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6f763cff36b75a10b031ef037d9c9b22af0c2fc3bac48b8f087d209754fd4392
+size 133391958
diff --git a/3b_layer8_mlp.gate_proj_input_pretrain_representation.pth b/3b_layer8_mlp.gate_proj_input_pretrain_representation.pth
new file mode 100644
index 0000000000000000000000000000000000000000..a133fcf12e5df11245b43faeae8c0ba769628bdf
--- /dev/null
+++ b/3b_layer8_mlp.gate_proj_input_pretrain_representation.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9adb2c4a1f6bdb50f35deb37fcb70f3b453287bfca9016f3c0c828afda09cf18
+size 133391874
diff --git a/3b_layer8_self_attn.q_proj_input_pretrain_representation.pth b/3b_layer8_self_attn.q_proj_input_pretrain_representation.pth
new file mode 100644
index 0000000000000000000000000000000000000000..61ee754bd69ba956b01b8c9740f1a80f34e5e1f9
--- /dev/null
+++ b/3b_layer8_self_attn.q_proj_input_pretrain_representation.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ce03c1e8807288a2b0e91219c2f1ee1554ec253dcfafe8a7095c0502d1e0b06f
+size 133391953
diff --git a/3b_layer8_self_attn.q_proj_output_pretrain_representation.pth b/3b_layer8_self_attn.q_proj_output_pretrain_representation.pth
new file mode 100644
index 0000000000000000000000000000000000000000..7c9011d9c77d7a10f96ef038afcb86fe43808f08
--- /dev/null
+++ b/3b_layer8_self_attn.q_proj_output_pretrain_representation.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3ad10dcb05c83a7e4163596d6c546e7827a72bc6d4484a50ba2f211746580064
+size 133391958
diff --git a/3b_layer8_self_attn.v_proj_input_pretrain_representation.pth b/3b_layer8_self_attn.v_proj_input_pretrain_representation.pth
new file mode 100644
index 0000000000000000000000000000000000000000..b1cc3cc2da3edfe9a61442bd58129bad0df9c540
--- /dev/null
+++ b/3b_layer8_self_attn.v_proj_input_pretrain_representation.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6520493cb11e9ee8ac4f4d832bf481555d53164167cec8f0c3f58eac123bacdb
+size 133391953
diff --git a/3b_layer9_mlp.up_proj_input_pretrain_representation.pth b/3b_layer9_mlp.up_proj_input_pretrain_representation.pth
new file mode 100644
index 0000000000000000000000000000000000000000..0832c9a2e169a5fbb55db8d0ec87dec924174659
--- /dev/null
+++ b/3b_layer9_mlp.up_proj_input_pretrain_representation.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e2e1aecbeeb1625e2129a0a71619cab91a07d5e4a73ba173f5d8d9082385b0c6
+size 133391864
diff --git a/3b_layer9_mlp.up_proj_output_pretrain_representation.pth b/3b_layer9_mlp.up_proj_output_pretrain_representation.pth
new file mode 100644
index 0000000000000000000000000000000000000000..324d2a541d0c04f67c49d8ed2621789ef52bd7ae
--- /dev/null
+++ b/3b_layer9_mlp.up_proj_output_pretrain_representation.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3472079a59a409ce75155b7667b5c59241deea5dd5f0275d291cb773771f188d
+size 533562877