truncated :/ because no tmux
45 2 | 8 self_attn q | 125,829,120 | -0.00002
46 1 | 8 self_attn [k, v] | 10,485,760 | -0.00010
46 2 | 8 self_attn [k, v] | 20,971,520 | -0.00017
47 1 | 8 self_attn o | 62,914,560 | +0.00007
47 2 | 8 self_attn o | 125,829,120 | +0.00003
48 1 | 8 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00012
48 2 | 8 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00009
49 1 | 8 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 2,516,582,400 | +0.00002
49 2 | 8 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 5,033,164,800 | -0.00004
50 1 | 8 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 1,258,291,200 | -0.00007
50 2 | 8 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 2,516,582,400 | -0.00005
51 1 | 9 self_attn q | 62,914,560 | -0.00004
51 2 | 9 self_attn q | 125,829,120 | -0.00000
52 1 | 9 self_attn [k, v] | 10,485,760 | +0.00007
52 2 | 9 self_attn [k, v] | 20,971,520 | +0.00002
53 1 | 9 self_attn o | 62,914,560 | -0.00005
53 2 | 9 self_attn o | 125,829,120 | -0.00012
54 1 | 9 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00008
54 2 | 9 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00007
55 1 | 9 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 2,516,582,400 | -0.00004
55 2 | 9 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 5,033,164,800 | -0.00008
56 1 | 9 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 1,258,291,200 | -0.00003
56 2 | 9 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 2,516,582,400 | +0.00001
57 1 | 10 self_attn q | 62,914,560 | -0.00004
57 2 | 10 self_attn q | 125,829,120 | +0.00000
58 1 | 10 self_attn [k, v] | 10,485,760 | +0.00011
58 2 | 10 self_attn [k, v] | 20,971,520 | -0.00013
59 1 | 10 self_attn o | 62,914,560 | -0.00020
59 2 | 10 self_attn o | 125,829,120 | -0.00016
60 1 | 10 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00002
60 2 | 10 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00001
61 1 | 10 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00005
61 2 | 10 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00009
62 1 | 10 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00005
62 2 | 10 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00007
63 1 | 11 self_attn q | 62,914,560 | -0.00008
63 2 | 11 self_attn q | 125,829,120 | -0.00003
64 1 | 11 self_attn [k, v] | 10,485,760 | +0.00008
64 2 | 11 self_attn [k, v] | 20,971,520 | +0.00008
65 1 | 11 self_attn o | 62,914,560 | -0.00031
65 2 | 11 self_attn o | 125,829,120 | -0.00032
66 1 | 11 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00004
66 2 | 11 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00000
67 1 | 11 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00006
67 2 | 11 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00004
68 1 | 11 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00003
68 2 | 11 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00004
69 1 | 12 self_attn q | 62,914,560 | +0.00002
69 2 | 12 self_attn q | 125,829,120 | -0.00001
70 1 | 12 self_attn [k, v] | 10,485,760 | +0.00014
70 2 | 12 self_attn [k, v] | 20,971,520 | +0.00014
71 1 | 12 self_attn o | 62,914,560 | +0.00005
71 2 | 12 self_attn o | 125,829,120 | +0.00006
72 1 | 12 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00010
72 2 | 12 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00011
73 1 | 12 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00006
73 2 | 12 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00007
74 1 | 12 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00005
74 2 | 12 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00006
75 1 | 13 self_attn q | 62,914,560 | -0.00010
75 2 | 13 self_attn q | 125,829,120 | -0.00009
76 1 | 13 self_attn [k, v] | 10,485,760 | -0.00004
76 2 | 13 self_attn [k, v] | 20,971,520 | +0.00005
77 1 | 13 self_attn o | 62,914,560 | -0.00009
77 2 | 13 self_attn o | 125,829,120 | -0.00009
78 1 | 13 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00017
78 2 | 13 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00020
79 1 | 13 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00007
79 2 | 13 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00015
80 1 | 13 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00013
80 2 | 13 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00018
81 1 | 14 self_attn q | 62,914,560 | +0.00002
81 2 | 14 self_attn q | 125,829,120 | -0.00003
82 1 | 14 self_attn [k, v] | 10,485,760 | +0.00011
82 2 | 14 self_attn [k, v] | 20,971,520 | +0.00010
83 1 | 14 self_attn o | 62,914,560 | -0.00024
83 2 | 14 self_attn o | 125,829,120 | -0.00023
84 1 | 14 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00003
84 2 | 14 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00004
85 1 | 14 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00010
85 2 | 14 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00010
86 1 | 14 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00010
86 2 | 14 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00014
87 1 | 15 self_attn q | 62,914,560 | -0.00000
87 2 | 15 self_attn q | 125,829,120 | -0.00002
88 1 | 15 self_attn [k, v] | 10,485,760 | -0.00009
88 2 | 15 self_attn [k, v] | 20,971,520 | -0.00007
89 1 | 15 self_attn o | 62,914,560 | -0.00006
89 2 | 15 self_attn o | 125,829,120 | -0.00012
90 1 | 15 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00001
90 2 | 15 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00002
91 1 | 15 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00001
91 2 | 15 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00003
92 1 | 15 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00007
92 2 | 15 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00005
93 1 | 16 self_attn q | 62,914,560 | +0.00004
93 2 | 16 self_attn q | 125,829,120 | +0.00001
94 1 | 16 self_attn [k, v] | 10,485,760 | -0.00002
94 2 | 16 self_attn [k, v] | 20,971,520 | -0.00004
95 1 | 16 self_attn o | 62,914,560 | -0.00015
95 2 | 16 self_attn o | 125,829,120 | -0.00012
96 1 | 16 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00006
96 2 | 16 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00006
97 1 | 16 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00018
97 2 | 16 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00019
98 1 | 16 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00016
98 2 | 16 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00011
99 1 | 17 self_attn q | 62,914,560 | -0.00005
99 2 | 17 self_attn q | 125,829,120 | -0.00001
100 1 | 17 self_attn [k, v] | 10,485,760 | +0.00001
100 2 | 17 self_attn [k, v] | 20,971,520 | +0.00002
101 1 | 17 self_attn o | 62,914,560 | -0.00002
101 2 | 17 self_attn o | 125,829,120 | -0.00004
102 1 | 17 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00006
102 2 | 17 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00010
103 1 | 17 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00009
103 2 | 17 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00010
104 1 | 17 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00006
104 2 | 17 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00004
105 1 | 18 self_attn q | 62,914,560 | +0.00011
105 2 | 18 self_attn q | 125,829,120 | +0.00003
106 1 | 18 self_attn [k, v] | 10,485,760 | +0.00004
106 2 | 18 self_attn [k, v] | 20,971,520 | +0.00004
107 1 | 18 self_attn o | 62,914,560 | +0.00003
107 2 | 18 self_attn o | 125,829,120 | -0.00007
108 1 | 18 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00024
108 2 | 18 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00023
109 1 | 18 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00008
109 2 | 18 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00004
110 1 | 18 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00004
110 2 | 18 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00009
111 1 | 19 self_attn q | 62,914,560 | -0.00005
111 2 | 19 self_attn q | 125,829,120 | -0.00005
112 1 | 19 self_attn [k, v] | 10,485,760 | -0.00000
112 2 | 19 self_attn [k, v] | 20,971,520 | -0.00007
113 1 | 19 self_attn o | 62,914,560 | -0.00006
113 2 | 19 self_attn o | 125,829,120 | -0.00005
114 1 | 19 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00020
114 2 | 19 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00020
115 1 | 19 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00007
115 2 | 19 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00007
116 1 | 19 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00002
116 2 | 19 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00006
117 1 | 20 self_attn q | 62,914,560 | +0.00002
117 2 | 20 self_attn q | 125,829,120 | +0.00002
118 1 | 20 self_attn [k, v] | 10,485,760 | -0.00009
118 2 | 20 self_attn [k, v] | 20,971,520 | -0.00003
119 1 | 20 self_attn o | 62,914,560 | -0.00026
119 2 | 20 self_attn o | 125,829,120 | -0.00020
120 1 | 20 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00023
120 2 | 20 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00027
121 1 | 20 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00011
121 2 | 20 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00014
122 1 | 20 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00000
122 2 | 20 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00003
123 1 | 21 self_attn q | 62,914,560 | -0.00006
123 2 | 21 self_attn q | 125,829,120 | -0.00004
124 1 | 21 self_attn [k, v] | 10,485,760 | +0.00004
124 2 | 21 self_attn [k, v] | 20,971,520 | -0.00004
125 1 | 21 self_attn o | 62,914,560 | +0.00011
125 2 | 21 self_attn o | 125,829,120 | +0.00011
126 1 | 21 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00022
126 2 | 21 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00017
127 1 | 21 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00010
127 2 | 21 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00009
128 1 | 21 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00014
128 2 | 21 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00011
129 1 | 22 self_attn q | 62,914,560 | -0.00003
129 2 | 22 self_attn q | 125,829,120 | +0.00001
130 1 | 22 self_attn [k, v] | 10,485,760 | -0.00008
130 2 | 22 self_attn [k, v] | 20,971,520 | +0.00003
131 1 | 22 self_attn o | 62,914,560 | +0.00004
131 2 | 22 self_attn o | 125,829,120 | +0.00011
132 1 | 22 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00003
132 2 | 22 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00002
133 1 | 22 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00007
133 2 | 22 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00012
134 1 | 22 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00008
134 2 | 22 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00007
135 1 | 23 self_attn q | 62,914,560 | -0.00004
135 2 | 23 self_attn q | 125,829,120 | -0.00001
136 1 | 23 self_attn [k, v] | 10,485,760 | -0.00010
136 2 | 23 self_attn [k, v] | 20,971,520 | -0.00004
137 1 | 23 self_attn o | 62,914,560 | +0.00008
137 2 | 23 self_attn o | 125,829,120 | +0.00008
138 1 | 23 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00009
138 2 | 23 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00005
139 1 | 23 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00012
139 2 | 23 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00012
140 1 | 23 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00004
140 2 | 23 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00003
141 1 | 24 self_attn q | 62,914,560 | -0.00010
141 2 | 24 self_attn q | 125,829,120 | -0.00010
142 1 | 24 self_attn [k, v] | 10,485,760 | -0.00006
142 2 | 24 self_attn [k, v] | 20,971,520 | -0.00013
143 1 | 24 self_attn o | 62,914,560 | -0.00002
143 2 | 24 self_attn o | 125,829,120 | -0.00015
144 1 | 24 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00004
144 2 | 24 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00013
145 1 | 24 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00000
145 2 | 24 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00004
146 1 | 24 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00005
146 2 | 24 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00006
147 1 | 25 self_attn q | 62,914,560 | -0.00007
147 2 | 25 self_attn q | 125,829,120 | -0.00009
148 1 | 25 self_attn [k, v] | 10,485,760 | -0.00008
148 2 | 25 self_attn [k, v] | 20,971,520 | -0.00004
149 1 | 25 self_attn o | 62,914,560 | -0.00003
149 2 | 25 self_attn o | 125,829,120 | -0.00005
150 1 | 25 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00009
150 2 | 25 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00003
151 1 | 25 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00008
151 2 | 25 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00007
152 1 | 25 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00003
152 2 | 25 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00002
153 1 | 26 self_attn q | 62,914,560 | -0.00001
153 2 | 26 self_attn q | 125,829,120 | -0.00007
154 1 | 26 self_attn [k, v] | 10,485,760 | -0.00013
154 2 | 26 self_attn [k, v] | 20,971,520 | -0.00022
155 1 | 26 self_attn o | 62,914,560 | -0.00023
155 2 | 26 self_attn o | 125,829,120 | -0.00021
156 1 | 26 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00004
156 2 | 26 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00006
157 1 | 26 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00005
157 2 | 26 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00006
158 1 | 26 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00000
158 2 | 26 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00004
159 1 | 27 self_attn q | 62,914,560 | +0.00004
159 2 | 27 self_attn q | 125,829,120 | -0.00002
160 1 | 27 self_attn [k, v] | 10,485,760 | -0.00003
160 2 | 27 self_attn [k, v] | 20,971,520 | -0.00001
161 1 | 27 self_attn o | 62,914,560 | -0.00000
161 2 | 27 self_attn o | 125,829,120 | -0.00013
162 1 | 27 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00004
162 2 | 27 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00008
163 1 | 27 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00001
163 2 | 27 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00002
164 1 | 27 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00004
164 2 | 27 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00006
165 1 | 28 self_attn q | 62,914,560 | -0.00005
165 2 | 28 self_attn q | 125,829,120 | -0.00004
166 1 | 28 self_attn [k, v] | 10,485,760 | -0.00017
166 2 | 28 self_attn [k, v] | 20,971,520 | -0.00018
167 1 | 28 self_attn o | 62,914,560 | -0.00010
167 2 | 28 self_attn o | 125,829,120 | -0.00011
168 1 | 28 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00000
168 2 | 28 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00014
169 1 | 28 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00000
169 2 | 28 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00003
170 1 | 28 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00004
170 2 | 28 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00004
171 1 | 29 self_attn q | 62,914,560 | -0.00003
171 2 | 29 self_attn q | 125,829,120 | -0.00001
172 1 | 29 self_attn [k, v] | 10,485,760 | +0.00009
172 2 | 29 self_attn [k, v] | 20,971,520 | +0.00001
173 1 | 29 self_attn o | 62,914,560 | -0.00006
173 2 | 29 self_attn o | 125,829,120 | -0.00003
174 1 | 29 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00003
174 2 | 29 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00004
175 1 | 29 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00000
175 2 | 29 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00004
176 1 | 29 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00000
176 2 | 29 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00000
177 1 | 30 self_attn q | 62,914,560 | -0.00000
177 2 | 30 self_attn q | 125,829,120 | -0.00000
178 1 | 30 self_attn [k, v] | 10,485,760 | -0.00000
178 2 | 30 self_attn [k, v] | 20,971,520 | -0.00005
179 1 | 30 self_attn o | 62,914,560 | -0.00005
179 2 | 30 self_attn o | 125,829,120 | -0.00011
180 1 | 30 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00012
180 2 | 30 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00007
181 1 | 30 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00007
181 2 | 30 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00007
182 1 | 30 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00002
182 2 | 30 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00003
183 1 | 31 self_attn q | 62,914,560 | +0.00005
183 2 | 31 self_attn q | 125,829,120 | -0.00001
184 1 | 31 self_attn [k, v] | 10,485,760 | -0.00031
184 2 | 31 self_attn [k, v] | 20,971,520 | -0.00015
185 1 | 31 self_attn o | 62,914,560 | -0.00013
185 2 | 31 self_attn o | 125,829,120 | -0.00004
186 1 | 31 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00029
186 2 | 31 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00038
187 1 | 31 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00005
187 2 | 31 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00006
188 1 | 31 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00008
188 2 | 31 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00009
189 1 | 32 self_attn q | 62,914,560 | +0.00002
189 2 | 32 self_attn q | 125,829,120 | -0.00008
190 1 | 32 self_attn [k, v] | 10,485,760 | +0.00007
190 2 | 32 self_attn [k, v] | 20,971,520 | -0.00000
191 1 | 32 self_attn o | 62,914,560 | +0.00012
191 2 | 32 self_attn o | 125,829,120 | +0.00013
192 1 | 32 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00002
192 2 | 32 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00013
193 1 | 32 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00005
193 2 | 32 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00007
194 1 | 32 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00002
194 2 | 32 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00005
195 1 | 33 self_attn q | 62,914,560 | +0.00002
195 2 | 33 self_attn q | 125,829,120 | +0.00002
196 1 | 33 self_attn [k, v] | 10,485,760 | -0.00004
196 2 | 33 self_attn [k, v] | 20,971,520 | -0.00004
197 1 | 33 self_attn o | 62,914,560 | -0.00021
197 2 | 33 self_attn o | 125,829,120 | -0.00015
198 1 | 33 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00029
198 2 | 33 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00019
199 1 | 33 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00004
199 2 | 33 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00007
200 1 | 33 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00006
200 2 | 33 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00005
201 1 | 34 self_attn q | 62,914,560 | -0.00010
201 2 | 34 self_attn q | 125,829,120 | -0.00008
202 1 | 34 self_attn [k, v] | 10,485,760 | -0.00013
202 2 | 34 self_attn [k, v] | 20,971,520 | -0.00009
203 1 | 34 self_attn o | 62,914,560 | +0.00009
203 2 | 34 self_attn o | 125,829,120 | +0.00008
204 1 | 34 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00022
204 2 | 34 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00022
205 1 | 34 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00007
205 2 | 34 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00005
206 1 | 34 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00004
206 2 | 34 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00005
207 1 | 35 self_attn q | 62,914,560 | -0.00007
207 2 | 35 self_attn q | 125,829,120 | -0.00004
208 1 | 35 self_attn [k, v] | 10,485,760 | +0.00018
208 2 | 35 self_attn [k, v] | 20,971,520 | +0.00018
209 1 | 35 self_attn o | 62,914,560 | -0.00018
209 2 | 35 self_attn o | 125,829,120 | -0.00017
210 1 | 35 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00011
210 2 | 35 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00004
211 1 | 35 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00005
211 2 | 35 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00010
212 1 | 35 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00005
212 2 | 35 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00004
213 1 | 36 self_attn q | 62,914,560 | -0.00000
213 2 | 36 self_attn q | 125,829,120 | -0.00013
214 1 | 36 self_attn [k, v] | 10,485,760 | -0.00060
214 2 | 36 self_attn [k, v] | 20,971,520 | -0.00055
215 1 | 36 self_attn o | 62,914,560 | -0.00012
215 2 | 36 self_attn o | 125,829,120 | -0.00012
216 1 | 36 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00006
216 2 | 36 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00001
217 1 | 36 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00007
217 2 | 36 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00006
218 1 | 36 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00006
218 2 | 36 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00007
219 1 | 37 self_attn q | 62,914,560 | +0.00006
219 2 | 37 self_attn q | 125,829,120 | +0.00005
220 1 | 37 self_attn [k, v] | 10,485,760 | -0.00015
220 2 | 37 self_attn [k, v] | 20,971,520 | -0.00004
221 1 | 37 self_attn o | 62,914,560 | +0.00006
221 2 | 37 self_attn o | 125,829,120 | +0.00006
222 1 | 37 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00002
222 2 | 37 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00008
223 1 | 37 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00001
223 2 | 37 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00002
224 1 | 37 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00001
224 2 | 37 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | +0.00000
225 1 | 38 self_attn q | 62,914,560 | -0.00011
225 2 | 38 self_attn q | 125,829,120 | -0.00011
226 1 | 38 self_attn [k, v] | 10,485,760 | +0.00001
226 2 | 38 self_attn [k, v] | 20,971,520 | +0.00001
227 1 | 38 self_attn o | 62,914,560 | -0.00009
227 2 | 38 self_attn o | 125,829,120 | -0.00015
228 1 | 38 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00002
228 2 | 38 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00004
229 1 | 38 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00008
229 2 | 38 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00011
230 1 | 38 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00002
230 2 | 38 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00004
231 1 | 39 self_attn q | 62,914,560 | -0.00008
231 2 | 39 self_attn q | 125,829,120 | -0.00005
232 1 | 39 self_attn [k, v] | 10,485,760 | -0.00040
232 2 | 39 self_attn [k, v] | 20,971,520 | -0.00042
233 1 | 39 self_attn o | 62,914,560 | +0.00013
233 2 | 39 self_attn o | 125,829,120 | +0.00005
234 1 | 39 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00015
234 2 | 39 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00019
235 1 | 39 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00003
235 2 | 39 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00003
236 1 | 39 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00003
236 2 | 39 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00003
237 1 | 40 self_attn q | 62,914,560 | -0.00001
237 2 | 40 self_attn q | 125,829,120 | -0.00002
238 1 | 40 self_attn [k, v] | 10,485,760 | -0.00018
238 2 | 40 self_attn [k, v] | 20,971,520 | -0.00011
239 1 | 40 self_attn o | 62,914,560 | -0.00014
239 2 | 40 self_attn o | 125,829,120 | -0.00021
240 1 | 40 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00007
240 2 | 40 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00010
241 1 | 40 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00002
241 2 | 40 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00003
242 1 | 40 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00007
242 2 | 40 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00005
243 1 | 41 self_attn q | 62,914,560 | -0.00001
243 2 | 41 self_attn q | 125,829,120 | +0.00001
244 1 | 41 self_attn [k, v] | 10,485,760 | -0.00008
244 2 | 41 self_attn [k, v] | 20,971,520 | -0.00011
245 1 | 41 self_attn o | 62,914,560 | -0.00002
245 2 | 41 self_attn o | 125,829,120 | -0.00004
246 1 | 41 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00009
246 2 | 41 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00003
247 1 | 41 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00003
247 2 | 41 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00004
248 1 | 41 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | +0.00001
248 2 | 41 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | +0.00005
249 1 | 42 self_attn q | 62,914,560 | -0.00000
249 2 | 42 self_attn q | 125,829,120 | -0.00001
250 1 | 42 self_attn [k, v] | 10,485,760 | -0.00025
250 2 | 42 self_attn [k, v] | 20,971,520 | -0.00026
251 1 | 42 self_attn o | 62,914,560 | +0.00005
251 2 | 42 self_attn o | 125,829,120 | +0.00008
252 1 | 42 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00001
252 2 | 42 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00002
253 1 | 42 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00002
253 2 | 42 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00004
254 1 | 42 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00003
254 2 | 42 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00003
255 1 | 43 self_attn q | 62,914,560 | -0.00008
255 2 | 43 self_attn q | 125,829,120 | -0.00007
256 1 | 43 self_attn [k, v] | 10,485,760 | +0.00000
256 2 | 43 self_attn [k, v] | 20,971,520 | +0.00001
257 1 | 43 self_attn o | 62,914,560 | -0.00010
257 2 | 43 self_attn o | 125,829,120 | -0.00013
258 1 | 43 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00002
258 2 | 43 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00002
259 1 | 43 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00001
259 2 | 43 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00002
260 1 | 43 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | +0.00001
260 2 | 43 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | +0.00001
261 1 | 44 self_attn q | 62,914,560 | +0.00000
261 2 | 44 self_attn q | 125,829,120 | -0.00003
262 1 | 44 self_attn [k, v] | 10,485,760 | +0.00009
262 2 | 44 self_attn [k, v] | 20,971,520 | +0.00014
263 1 | 44 self_attn o | 62,914,560 | -0.00008
263 2 | 44 self_attn o | 125,829,120 | -0.00008
264 1 | 44 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00009
264 2 | 44 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00006
265 1 | 44 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00001
265 2 | 44 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00001
266 1 | 44 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | +0.00000
266 2 | 44 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00001
267 1 | 45 self_attn q | 62,914,560 | +0.00005
267 2 | 45 self_attn q | 125,829,120 | +0.00005
268 1 | 45 self_attn [k, v] | 10,485,760 | -0.00006
268 2 | 45 self_attn [k, v] | 20,971,520 | -0.00003
269 1 | 45 self_attn o | 62,914,560 | +0.00003
269 2 | 45 self_attn o | 125,829,120 | -0.00001
270 1 | 45 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00001
270 2 | 45 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00003
271 1 | 45 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | +0.00002
271 2 | 45 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | +0.00001
272 1 | 45 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00002
272 2 | 45 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00002
273 1 | 46 self_attn q | 62,914,560 | -0.00012
273 2 | 46 self_attn q | 125,829,120 | -0.00007
274 1 | 46 self_attn [k, v] | 10,485,760 | +0.00001
274 2 | 46 self_attn [k, v] | 20,971,520 | -0.00001
275 1 | 46 self_attn o | 62,914,560 | +0.00001
275 2 | 46 self_attn o | 125,829,120 | -0.00000
276 1 | 46 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00003
276 2 | 46 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00005
277 1 | 46 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00001
277 2 | 46 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00003
278 1 | 46 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | +0.00003
278 2 | 46 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00000
279 1 | 47 self_attn q | 62,914,560 | -0.00001
279 2 | 47 self_attn q | 125,829,120 | +0.00002
280 1 | 47 self_attn [k, v] | 10,485,760 | +0.00006
280 2 | 47 self_attn [k, v] | 20,971,520 | +0.00002
281 1 | 47 self_attn o | 62,914,560 | -0.00005
281 2 | 47 self_attn o | 125,829,120 | -0.00002
282 1 | 47 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00003
282 2 | 47 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00003
283 1 | 47 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | +0.00002
283 2 | 47 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00000
284 1 | 47 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00002
284 2 | 47 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00003
285 1 | 48 self_attn q | 62,914,560 | -0.00007
285 2 | 48 self_attn q | 125,829,120 | -0.00004
286 1 | 48 self_attn [k, v] | 10,485,760 | -0.00014
286 2 | 48 self_attn [k, v] | 20,971,520 | -0.00013
287 1 | 48 self_attn o | 62,914,560 | -0.00002
287 2 | 48 self_attn o | 125,829,120 | +0.00001
288 1 | 48 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00006
288 2 | 48 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00005
289 1 | 48 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | +0.00001
289 2 | 48 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00001
290 1 | 48 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00003
290 2 | 48 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00001
291 1 | 49 self_attn q | 62,914,560 | -0.00000
291 2 | 49 self_attn q | 125,829,120 | +0.00002
292 1 | 49 self_attn [k, v] | 10,485,760 | -0.00007
292 2 | 49 self_attn [k, v] | 20,971,520 | -0.00007
293 1 | 49 self_attn o | 62,914,560 | +0.00003
293 2 | 49 self_attn o | 125,829,120 | +0.00004
294 1 | 49 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00003
294 2 | 49 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00001
295 1 | 49 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00003
295 2 | 49 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00001
296 1 | 49 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | +0.00000
296 2 | 49 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00000
297 1 | 50 self_attn q | 62,914,560 | +0.00002
297 2 | 50 self_attn q | 125,829,120 | +0.00002
298 1 | 50 self_attn [k, v] | 10,485,760 | -0.00007
298 2 | 50 self_attn [k, v] | 20,971,520 | -0.00010
299 1 | 50 self_attn o | 62,914,560 | -0.00004
299 2 | 50 self_attn o | 125,829,120 | -0.00003
300 1 | 50 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00001
300 2 | 50 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00000
301 1 | 50 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00004
301 2 | 50 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00004
302 1 | 50 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | +0.00001
302 2 | 50 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00000
303 1 | 51 self_attn q | 62,914,560 | -0.00000
303 2 | 51 self_attn q | 125,829,120 | +0.00001
304 1 | 51 self_attn [k, v] | 10,485,760 | -0.00002
304 2 | 51 self_attn [k, v] | 20,971,520 | -0.00006
305 1 | 51 self_attn o | 62,914,560 | +0.00006
305 2 | 51 self_attn o | 125,829,120 | +0.00005
306 1 | 51 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00005
306 2 | 51 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00004
307 1 | 51 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00003
307 2 | 51 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00004
308 1 | 51 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | +0.00001
308 2 | 51 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00002
309 1 | 52 self_attn q | 62,914,560 | +0.00004
309 2 | 52 self_attn q | 125,829,120 | -0.00000
310 1 | 52 self_attn [k, v] | 10,485,760 | +0.00010
310 2 | 52 self_attn [k, v] | 20,971,520 | +0.00003
311 1 | 52 self_attn o | 62,914,560 | -0.00002
311 2 | 52 self_attn o | 125,829,120 | -0.00004
312 1 | 52 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00001
312 2 | 52 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00000
313 1 | 52 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00002
313 2 | 52 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00000
314 1 | 52 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00001
314 2 | 52 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | +0.00000
315 1 | 53 self_attn q | 62,914,560 | -0.00003
315 2 | 53 self_attn q | 125,829,120 | -0.00001
316 1 | 53 self_attn [k, v] | 10,485,760 | +0.00002
316 2 | 53 self_attn [k, v] | 20,971,520 | +0.00000
317 1 | 53 self_attn o | 62,914,560 | -0.00005
317 2 | 53 self_attn o | 125,829,120 | -0.00005
318 1 | 53 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00002
318 2 | 53 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00002
319 1 | 53 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | +0.00000
319 2 | 53 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | +0.00001
320 1 | 53 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00001
320 2 | 53 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00001
321 1 | 54 self_attn q | 62,914,560 | -0.00007
321 2 | 54 self_attn q | 125,829,120 | -0.00007
322 1 | 54 self_attn [k, v] | 10,485,760 | -0.00008
322 2 | 54 self_attn [k, v] | 20,971,520 | -0.00008
323 1 | 54 self_attn o | 62,914,560 | +0.00004
323 2 | 54 self_attn o | 125,829,120 | +0.00001
324 1 | 54 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00001
324 2 | 54 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00003
325 1 | 54 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00001
325 2 | 54 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00000
326 1 | 54 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00000
326 2 | 54 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | +0.00001
327 1 | 55 self_attn q | 62,914,560 | +0.00004
327 2 | 55 self_attn q | 125,829,120 | +0.00003
328 1 | 55 self_attn [k, v] | 10,485,760 | +0.00005
328 2 | 55 self_attn [k, v] | 20,971,520 | +0.00006
329 1 | 55 self_attn o | 62,914,560 | -0.00001
329 2 | 55 self_attn o | 125,829,120 | -0.00000
330 1 | 55 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00002
330 2 | 55 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00001
331 1 | 55 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00001
331 2 | 55 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00002
332 1 | 55 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00000
332 2 | 55 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | +0.00000
333 1 | 56 self_attn q | 62,914,560 | +0.00001
333 2 | 56 self_attn q | 125,829,120 | -0.00001
334 1 | 56 self_attn [k, v] | 10,485,760 | +0.00005
334 2 | 56 self_attn [k, v] | 20,971,520 | +0.00006
335 1 | 56 self_attn o | 62,914,560 | +0.00001
335 2 | 56 self_attn o | 125,829,120 | +0.00001
336 1 | 56 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00002
336 2 | 56 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00001
337 1 | 56 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | +0.00002
337 2 | 56 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00000
338 1 | 56 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | +0.00000
338 2 | 56 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | +0.00000
339 1 | 57 self_attn q | 62,914,560 | +0.00002
339 2 | 57 self_attn q | 125,829,120 | +0.00001
340 1 | 57 self_attn [k, v] | 10,485,760 | -0.00001
340 2 | 57 self_attn [k, v] | 20,971,520 | -0.00000
341 1 | 57 self_attn o | 62,914,560 | +0.00000
341 2 | 57 self_attn o | 125,829,120 | -0.00001
342 1 | 57 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00002
342 2 | 57 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00001
343 1 | 57 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | +0.00001
343 2 | 57 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | +0.00000
344 1 | 57 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00002
344 2 | 57 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00002
345 1 | 58 self_attn q | 62,914,560 | +0.00002
345 2 | 58 self_attn q | 125,829,120 | -0.00001
346 1 | 58 self_attn [k, v] | 10,485,760 | +0.00000
346 2 | 58 self_attn [k, v] | 20,971,520 | -0.00001
347 1 | 58 self_attn o | 62,914,560 | -0.00001
347 2 | 58 self_attn o | 125,829,120 | -0.00001
348 1 | 58 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00005
348 2 | 58 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00004
349 1 | 58 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | +0.00001
349 2 | 58 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | +0.00002
350 1 | 58 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | +0.00001
350 2 | 58 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00001
351 1 | 59 self_attn q | 62,914,560 | +0.00000
351 2 | 59 self_attn q | 125,829,120 | +0.00003
352 1 | 59 self_attn [k, v] | 10,485,760 | -0.00004
352 2 | 59 self_attn [k, v] | 20,971,520 | -0.00003
353 1 | 59 self_attn o | 62,914,560 | -0.00002
353 2 | 59 self_attn o | 125,829,120 | -0.00002
354 1 | 59 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00005
354 2 | 59 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00004
355 1 | 59 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | +0.00002
355 2 | 59 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | +0.00002
356 1 | 59 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00001
356 2 | 59 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00001
357 1 | 60 self_attn q | 62,914,560 | -0.00003
357 2 | 60 self_attn q | 125,829,120 | -0.00002
358 1 | 60 self_attn [k, v] | 10,485,760 | -0.00005
358 2 | 60 self_attn [k, v] | 20,971,520 | -0.00008
359 1 | 60 self_attn o | 62,914,560 | -0.00001
359 2 | 60 self_attn o | 125,829,120 | +0.00002
360 1 | 60 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00003
360 2 | 60 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00004
361 1 | 60 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | +0.00001
361 2 | 60 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00000
362 1 | 60 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00001
362 2 | 60 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00000
363 1 | 61 self_attn q | 62,914,560 | +0.00002
363 2 | 61 self_attn q | 125,829,120 | +0.00002
364 1 | 61 self_attn [k, v] | 10,485,760 | -0.00000
364 2 | 61 self_attn [k, v] | 20,971,520 | -0.00000
365 1 | 61 self_attn o | 62,914,560 | -0.00001
365 2 | 61 self_attn o | 125,829,120 | -0.00002
366 1 | 61 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00002
366 2 | 61 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00001
367 1 | 61 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00002
367 2 | 61 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00003
368 1 | 61 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | +0.00002
368 2 | 61 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00001
369 1 | 62 self_attn q | 62,914,560 | +0.00001
369 2 | 62 self_attn q | 125,829,120 | +0.00001
370 1 | 62 self_attn [k, v] | 10,485,760 | +0.00002
370 2 | 62 self_attn [k, v] | 20,971,520 | +0.00002
371 1 | 62 self_attn o | 62,914,560 | +0.00001
371 2 | 62 self_attn o | 125,829,120 | +0.00001
372 1 | 62 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00002
372 2 | 62 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00001
373 1 | 62 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00001
373 2 | 62 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00001
374 1 | 62 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | +0.00000
374 2 | 62 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00001
375 1 | 63 self_attn q | 62,914,560 | -0.00002
375 2 | 63 self_attn q | 125,829,120 | -0.00003
376 1 | 63 self_attn [k, v] | 10,485,760 | +0.00001
376 2 | 63 self_attn [k, v] | 20,971,520 | -0.00002
377 1 | 63 self_attn o | 62,914,560 | +0.00001
377 2 | 63 self_attn o | 125,829,120 | +0.00001
378 1 | 63 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00002
378 2 | 63 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00000
379 1 | 63 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00000
379 2 | 63 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00000
380 1 | 63 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | +0.00002
380 2 | 63 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | +0.00002
381 1 | 64 self_attn q | 62,914,560 | +0.00002
381 2 | 64 self_attn q | 125,829,120 | +0.00001
382 1 | 64 self_attn [k, v] | 10,485,760 | +0.00001
382 2 | 64 self_attn [k, v] | 20,971,520 | +0.00001
383 1 | 64 self_attn o | 62,914,560 | +0.00001
383 2 | 64 self_attn o | 125,829,120 | +0.00000
384 1 | 64 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00000
384 2 | 64 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00001
385 1 | 64 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00003
385 2 | 64 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00002
386 1 | 64 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00001
386 2 | 64 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00000
387 1 | 65 self_attn q | 62,914,560 | +0.00002
387 2 | 65 self_attn q | 125,829,120 | +0.00002
388 1 | 65 self_attn [k, v] | 10,485,760 | -0.00005
388 2 | 65 self_attn [k, v] | 20,971,520 | -0.00002
389 1 | 65 self_attn o | 62,914,560 | -0.00001
389 2 | 65 self_attn o | 125,829,120 | -0.00001
390 1 | 65 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00001
390 2 | 65 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00001
391 1 | 65 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | +0.00000
391 2 | 65 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00001
392 1 | 65 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | +0.00002
392 2 | 65 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | +0.00002
393 1 | 66 self_attn q | 62,914,560 | -0.00008
393 2 | 66 self_attn q | 125,829,120 | -0.00008
394 1 | 66 self_attn [k, v] | 10,485,760 | -0.00001
394 2 | 66 self_attn [k, v] | 20,971,520 | +0.00000
395 1 | 66 self_attn o | 62,914,560 | -0.00001
395 2 | 66 self_attn o | 125,829,120 | +0.00001
396 1 | 66 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00005
396 2 | 66 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00003
397 1 | 66 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | +0.00000
397 2 | 66 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | +0.00001
398 1 | 66 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00001
398 2 | 66 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00000
399 1 | 67 self_attn q | 62,914,560 | -0.00002
399 2 | 67 self_attn q | 125,829,120 | -0.00003
400 1 | 67 self_attn [k, v] | 10,485,760 | +0.00003
400 2 | 67 self_attn [k, v] | 20,971,520 | +0.00002
401 1 | 67 self_attn o | 62,914,560 | +0.00002
401 2 | 67 self_attn o | 125,829,120 | +0.00002
402 1 | 67 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00004
402 2 | 67 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00002
403 1 | 67 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00001
403 2 | 67 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | +0.00002
404 1 | 67 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | +0.00000
404 2 | 67 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | +0.00000
405 1 | 68 self_attn q | 62,914,560 | -0.00000
405 2 | 68 self_attn q | 125,829,120 | -0.00001
406 1 | 68 self_attn [k, v] | 10,485,760 | +0.00007
406 2 | 68 self_attn [k, v] | 20,971,520 | +0.00003
407 1 | 68 self_attn o | 62,914,560 | -0.00003
407 2 | 68 self_attn o | 125,829,120 | -0.00005
408 1 | 68 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00001
408 2 | 68 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00002
409 1 | 68 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | +0.00002
409 2 | 68 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | +0.00002
410 1 | 68 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00002
410 2 | 68 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00002
411 1 | 69 self_attn q | 62,914,560 | +0.00001
411 2 | 69 self_attn q | 125,829,120 | +0.00001
412 1 | 69 self_attn [k, v] | 10,485,760 | -0.00002
412 2 | 69 self_attn [k, v] | 20,971,520 | -0.00003
413 1 | 69 self_attn o | 62,914,560 | +0.00002
413 2 | 69 self_attn o | 125,829,120 | +0.00002
414 1 | 69 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00004
414 2 | 69 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00002
415 1 | 69 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00001
415 2 | 69 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00002
416 1 | 69 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00002
416 2 | 69 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00000
417 1 | 70 self_attn q | 62,914,560 | +0.00001
417 2 | 70 self_attn q | 125,829,120 | -0.00000
418 1 | 70 self_attn [k, v] | 10,485,760 | +0.00002
418 2 | 70 self_attn [k, v] | 20,971,520 | +0.00001
419 1 | 70 self_attn o | 62,914,560 | -0.00000
419 2 | 70 self_attn o | 125,829,120 | -0.00001
420 1 | 70 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00000
420 2 | 70 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00002
421 1 | 70 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | +0.00001
421 2 | 70 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00000
422 1 | 70 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00001
422 2 | 70 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00001
423 1 | 71 self_attn q | 62,914,560 | +0.00003
423 2 | 71 self_attn q | 125,829,120 | +0.00003
424 1 | 71 self_attn [k, v] | 10,485,760 | +0.00002
424 2 | 71 self_attn [k, v] | 20,971,520 | -0.00001
425 1 | 71 self_attn o | 62,914,560 | -0.00001
425 2 | 71 self_attn o | 125,829,120 | -0.00002
426 1 | 71 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00004
426 2 | 71 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00004
427 1 | 71 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00003
427 2 | 71 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00002
428 1 | 71 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00002
428 2 | 71 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00002
429 1 | 72 self_attn q | 62,914,560 | +0.00001
429 2 | 72 self_attn q | 125,829,120 | +0.00001
430 1 | 72 self_attn [k, v] | 10,485,760 | -0.00000
430 2 | 72 self_attn [k, v] | 20,971,520 | -0.00002
431 1 | 72 self_attn o | 62,914,560 | -0.00002
431 2 | 72 self_attn o | 125,829,120 | -0.00002
432 1 | 72 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00002
432 2 | 72 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00000
433 1 | 72 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00004
433 2 | 72 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00005
434 1 | 72 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00002
434 2 | 72 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00000
435 1 | 73 self_attn q | 62,914,560 | +0.00001
435 2 | 73 self_attn q | 125,829,120 | +0.00001
436 1 | 73 self_attn [k, v] | 10,485,760 | +0.00003
436 2 | 73 self_attn [k, v] | 20,971,520 | +0.00003
437 1 | 73 self_attn o | 62,914,560 | +0.00001
437 2 | 73 self_attn o | 125,829,120 | +0.00002
438 1 | 73 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00002
438 2 | 73 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00003
439 1 | 73 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | +0.00001
439 2 | 73 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | +0.00000
440 1 | 73 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | +0.00001
440 2 | 73 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00001
441 1 | 74 self_attn q | 62,914,560 | +0.00001
441 2 | 74 self_attn q | 125,829,120 | +0.00000
442 1 | 74 self_attn [k, v] | 10,485,760 | +0.00001
442 2 | 74 self_attn [k, v] | 20,971,520 | -0.00001
443 1 | 74 self_attn o | 62,914,560 | -0.00002
443 2 | 74 self_attn o | 125,829,120 | -0.00000
444 1 | 74 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00003
444 2 | 74 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00004
445 1 | 74 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | +0.00000
445 2 | 74 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00001
446 1 | 74 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00003
446 2 | 74 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00003
447 1 | 75 self_attn q | 62,914,560 | -0.00003
447 2 | 75 self_attn q | 125,829,120 | -0.00001
448 1 | 75 self_attn [k, v] | 10,485,760 | -0.00000
448 2 | 75 self_attn [k, v] | 20,971,520 | +0.00000
449 1 | 75 self_attn o | 62,914,560 | +0.00000
449 2 | 75 self_attn o | 125,829,120 | -0.00001
450 1 | 75 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00005
450 2 | 75 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00004
451 1 | 75 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00003
451 2 | 75 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00003
452 1 | 75 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00000
452 2 | 75 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | +0.00000
453 1 | 76 self_attn q | 62,914,560 | -0.00011
453 2 | 76 self_attn q | 125,829,120 | -0.00011
454 1 | 76 self_attn [k, v] | 10,485,760 | -0.00004
454 2 | 76 self_attn [k, v] | 20,971,520 | -0.00004
455 1 | 76 self_attn o | 62,914,560 | -0.00001
455 2 | 76 self_attn o | 125,829,120 | -0.00002
456 1 | 76 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00005
456 2 | 76 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00004
457 1 | 76 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00002
457 2 | 76 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00002
458 1 | 76 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00002
458 2 | 76 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | +0.00000
459 1 | 77 self_attn q | 62,914,560 | -0.00002
459 2 | 77 self_attn q | 125,829,120 | -0.00003
460 1 | 77 self_attn [k, v] | 10,485,760 | -0.00007
460 2 | 77 self_attn [k, v] | 20,971,520 | -0.00007
461 1 | 77 self_attn o | 62,914,560 | -0.00001
461 2 | 77 self_attn o | 125,829,120 | -0.00002
462 1 | 77 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00002
462 2 | 77 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00003
463 1 | 77 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00001
463 2 | 77 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00000
464 1 | 77 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | +0.00001
464 2 | 77 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00001
465 1 | 78 self_attn q | 62,914,560 | +0.00001
465 2 | 78 self_attn q | 125,829,120 | +0.00002
466 1 | 78 self_attn [k, v] | 10,485,760 | +0.00000
466 2 | 78 self_attn [k, v] | 20,971,520 | -0.00002
467 1 | 78 self_attn o | 62,914,560 | +0.00002
467 2 | 78 self_attn o | 125,829,120 | +0.00004
468 1 | 78 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00001
468 2 | 78 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00004
469 1 | 78 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | +0.00001
469 2 | 78 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | +0.00000
470 1 | 78 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00000
470 2 | 78 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00002
471 1 | 79 self_attn q | 62,914,560 | +0.00002
471 2 | 79 self_attn q | 125,829,120 | +0.00001
472 1 | 79 self_attn [k, v] | 10,485,760 | -0.00016
472 2 | 79 self_attn [k, v] | 20,971,520 | -0.00019
473 1 | 79 self_attn o | 62,914,560 | -0.00002
473 2 | 79 self_attn o | 125,829,120 | +0.00000
474 1 | 79 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00004
474 2 | 79 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00002
475 1 | 79 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00006
475 2 | 79 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00005
476 1 | 79 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | +0.00003
476 2 | 79 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | +0.00002
477 1 | 80 self_attn q | 62,914,560 | +0.00019
477 2 | 80 self_attn q | 125,829,120 | +0.00008
478 1 | 80 self_attn [k, v] | 10,485,760 | +0.00005
478 2 | 80 self_attn [k, v] | 20,971,520 | +0.00000
479 1 | 80 self_attn o | 62,914,560 | -0.00000
479 2 | 80 self_attn o | 125,829,120 | +0.00000
480 1 | 80 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00009
480 2 | 80 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00006
481 1 | 80 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | +0.00001
481 2 | 80 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | +0.00000
482 1 | 80 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00000
482 2 | 80 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00001
483 1 | 81 self_attn q | 62,914,560 | +0.00009
483 2 | 81 self_attn q | 125,829,120 | +0.00004
484 1 | 81 self_attn [k, v] | 10,485,760 | +0.00005
484 2 | 81 self_attn [k, v] | 20,971,520 | +0.00006
485 1 | 81 self_attn o | 62,914,560 | -0.00002
485 2 | 81 self_attn o | 125,829,120 | -0.00002
486 1 | 81 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00002
486 2 | 81 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00006
487 1 | 81 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00001
487 2 | 81 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00002
488 1 | 81 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00001
488 2 | 81 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00001
489 1 | 82 self_attn q | 62,914,560 | -0.00002
489 2 | 82 self_attn q | 125,829,120 | -0.00002
490 1 | 82 self_attn [k, v] | 10,485,760 | -0.00009
490 2 | 82 self_attn [k, v] | 20,971,520 | -0.00008
491 1 | 82 self_attn o | 62,914,560 | -0.00007
491 2 | 82 self_attn o | 125,829,120 | -0.00007
492 1 | 82 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00003
492 2 | 82 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00002
493 1 | 82 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | +0.00002
493 2 | 82 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | +0.00001
494 1 | 82 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | +0.00001
494 2 | 82 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | +0.00001
495 1 | 83 self_attn q | 62,914,560 | +0.00001
495 2 | 83 self_attn q | 125,829,120 | -0.00000
496 1 | 83 self_attn [k, v] | 10,485,760 | -0.00013
496 2 | 83 self_attn [k, v] | 20,971,520 | -0.00014
497 1 | 83 self_attn o | 62,914,560 | -0.00005
497 2 | 83 self_attn o | 125,829,120 | -0.00003
498 1 | 83 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00003
498 2 | 83 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00004
499 1 | 83 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | +0.00000
499 2 | 83 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00002
500 1 | 83 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00001
500 2 | 83 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00000
501 1 | 84 self_attn q | 62,914,560 | -0.00007
501 2 | 84 self_attn q | 125,829,120 | -0.00010
502 1 | 84 self_attn [k, v] | 10,485,760 | -0.00012
502 2 | 84 self_attn [k, v] | 20,971,520 | -0.00009
503 1 | 84 self_attn o | 62,914,560 | +0.00001
503 2 | 84 self_attn o | 125,829,120 | -0.00000
504 1 | 84 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00007
504 2 | 84 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00009
505 1 | 84 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00002
505 2 | 84 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00003
506 1 | 84 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00000
506 2 | 84 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00001
507 1 | 85 self_attn q | 62,914,560 | -0.00002
507 2 | 85 self_attn q | 125,829,120 | -0.00008
508 1 | 85 self_attn [k, v] | 10,485,760 | -0.00025
508 2 | 85 self_attn [k, v] | 20,971,520 | -0.00027
509 1 | 85 self_attn o | 62,914,560 | -0.00007
509 2 | 85 self_attn o | 125,829,120 | -0.00007
510 1 | 85 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00003
510 2 | 85 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00004
511 1 | 85 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | +0.00004
511 2 | 85 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | +0.00001
512 1 | 85 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | +0.00001
512 2 | 85 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00001
513 1 | 86 self_attn q | 62,914,560 | +0.00000
513 2 | 86 self_attn q | 125,829,120 | -0.00003
514 1 | 86 self_attn [k, v] | 10,485,760 | -0.00011
514 2 | 86 self_attn [k, v] | 20,971,520 | -0.00013
515 1 | 86 self_attn o | 62,914,560 | -0.00013
515 2 | 86 self_attn o | 125,829,120 | -0.00011
516 1 | 86 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00001
516 2 | 86 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00001
517 1 | 86 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00003
517 2 | 86 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00004
518 1 | 86 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00001
518 2 | 86 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00000
519 1 | 87 self_attn q | 62,914,560 | +0.00001
519 2 | 87 self_attn q | 125,829,120 | +0.00000
520 1 | 87 self_attn [k, v] | 10,485,760 | -0.00013
520 2 | 87 self_attn [k, v] | 20,971,520 | -0.00035
521 1 | 87 self_attn o | 62,914,560 | +0.00000
521 2 | 87 self_attn o | 125,829,120 | +0.00000
522 1 | 87 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00001
522 2 | 87 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00005
523 1 | 87 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00001
523 2 | 87 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00001
524 1 | 87 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00001
524 2 | 87 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00001
525 1 | 88 self_attn q | 62,914,560 | -0.00002
525 2 | 88 self_attn q | 125,829,120 | -0.00001
526 1 | 88 self_attn [k, v] | 10,485,760 | +0.00002
526 2 | 88 self_attn [k, v] | 20,971,520 | -0.00003
527 1 | 88 self_attn o | 62,914,560 | -0.00005
527 2 | 88 self_attn o | 125,829,120 | -0.00004
528 1 | 88 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00001
528 2 | 88 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00003
529 1 | 88 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00000
529 2 | 88 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00001
530 1 | 88 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | +0.00000
530 2 | 88 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00000
531 1 | 89 self_attn q | 62,914,560 | -0.00004
531 2 | 89 self_attn q | 125,829,120 | -0.00004
532 1 | 89 self_attn [k, v] | 10,485,760 | -0.00010
532 2 | 89 self_attn [k, v] | 20,971,520 | -0.00008
533 1 | 89 self_attn o | 62,914,560 | -0.00002
533 2 | 89 self_attn o | 125,829,120 | -0.00002
534 1 | 89 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00004
534 2 | 89 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00006
535 1 | 89 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00000
535 2 | 89 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00000
536 1 | 89 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | +0.00000
536 2 | 89 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00001
537 1 | 90 self_attn q | 62,914,560 | -0.00001
537 2 | 90 self_attn q | 125,829,120 | +0.00000
538 1 | 90 self_attn [k, v] | 10,485,760 | -0.00005
538 2 | 90 self_attn [k, v] | 20,971,520 | -0.00005
539 1 | 90 self_attn o | 62,914,560 | +0.00001
539 2 | 90 self_attn o | 125,829,120 | +0.00000
540 1 | 90 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00007
540 2 | 90 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00005
541 1 | 90 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00000
541 2 | 90 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00000
542 1 | 90 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | +0.00001
542 2 | 90 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | +0.00000
543 1 | 91 self_attn q | 62,914,560 | +0.00003
543 2 | 91 self_attn q | 125,829,120 | +0.00003
544 1 | 91 self_attn [k, v] | 10,485,760 | -0.00011
544 2 | 91 self_attn [k, v] | 20,971,520 | -0.00013
545 1 | 91 self_attn o | 62,914,560 | -0.00008
545 2 | 91 self_attn o | 125,829,120 | -0.00008
546 1 | 91 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00020
546 2 | 91 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00016
547 1 | 91 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | +0.00000
547 2 | 91 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00001
548 1 | 91 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | +0.00000
548 2 | 91 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00000