Hugging Face's logo

dlab-cmu
/
sf-map-ridge

quantization
codebook
sf
ridge
Model card Files
xet
 Community
tim-dettmers commited on
Commit
39c65a6
·
verified ·
1 Parent(s): 1c391a3

Add sf-map-ridge-v1 selector

Browse files
Files changed (1) hide show
  1. sf-map-ridge.json +238 -0
sf-map-ridge.json ADDED
@@ -0,0 +1,238 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "schema": "sf-map-ridge-v1",
3
+ "repo": "dlab-cmu/sf-map-ridge",
4
+ "target": "log_mean_direct_kl",
5
+ "mode": "type",
6
+ "lam": 1.0,
7
+ "n": 1760,
8
+ "intercept": -6.096409487550104,
9
+ "features": [
10
+ "log_hidden",
11
+ "log_intermediate",
12
+ "log_n_blocks",
13
+ "has_deltanet",
14
+ "has_moe",
15
+ "is_deltanet",
16
+ "is_moe",
17
+ "is_shared_expert_gate",
18
+ "q_proj",
19
+ "k_proj",
20
+ "v_proj",
21
+ "o_proj",
22
+ "gate_proj",
23
+ "up_proj",
24
+ "down_proj",
25
+ "out_proj",
26
+ "in_proj_qkv",
27
+ "in_proj_z",
28
+ "in_proj_b",
29
+ "in_proj_a",
30
+ "shared_expert_gate"
31
+ ],
32
+ "feature_mean": [
33
+ 8.273532109875925,
34
+ 9.230560591865997,
35
+ 3.8855296220840874,
36
+ 0.7619318181818182,
37
+ 0.19886363636363635,
38
+ 0.35795454545454547,
39
+ 0.09090909090909091,
40
+ 0.022727272727272728,
41
+ 0.05795454545454545,
42
+ 0.05795454545454545,
43
+ 0.05795454545454545,
44
+ 0.05795454545454545,
45
+ 0.12954545454545455,
46
+ 0.12954545454545455,
47
+ 0.12840909090909092,
48
+ 0.0715909090909091,
49
+ 0.0715909090909091,
50
+ 0.0715909090909091,
51
+ 0.0715909090909091,
52
+ 0.0715909090909091,
53
+ 0.022727272727272728
54
+ ],
55
+ "feature_std": [
56
+ 0.3617200903994945,
57
+ 0.8327864061903807,
58
+ 0.3258661508165429,
59
+ 0.4259010714050337,
60
+ 0.3991451997655323,
61
+ 0.47939867421904503,
62
+ 0.287479787288034,
63
+ 0.1490326937341364,
64
+ 0.2336574760535212,
65
+ 0.2336574760535212,
66
+ 0.2336574760535212,
67
+ 0.2336574760535212,
68
+ 0.33580266489721156,
69
+ 0.33580266489721156,
70
+ 0.3345447597571874,
71
+ 0.2578093303711958,
72
+ 0.2578093303711958,
73
+ 0.25780933037119586,
74
+ 0.25780933037119586,
75
+ 0.25780933037119586,
76
+ 0.1490326937341364
77
+ ],
78
+ "weights": [
79
+ -0.16136731580940464,
80
+ -0.12497605346070649,
81
+ -0.21465072720678854,
82
+ -0.16689063963743547,
83
+ 0.05314806242157213,
84
+ -0.15781975127473868,
85
+ -0.40588865223183473,
86
+ -0.006448746937497545,
87
+ -0.10390388737809762,
88
+ -0.21552913509170615,
89
+ 0.10654495823752747,
90
+ 0.015220171823522142,
91
+ 0.0654735997100454,
92
+ 0.12668799883615095,
93
+ 0.17420047963195334,
94
+ 0.0880146150847331,
95
+ 0.10737333439917397,
96
+ 0.049599289846695586,
97
+ -0.2674454249577563,
98
+ -0.2710090035351432,
99
+ -0.006448746940487679
100
+ ],
101
+ "types": [
102
+ "q_proj",
103
+ "k_proj",
104
+ "v_proj",
105
+ "o_proj",
106
+ "gate_proj",
107
+ "up_proj",
108
+ "down_proj",
109
+ "out_proj",
110
+ "in_proj_qkv",
111
+ "in_proj_z",
112
+ "in_proj_b",
113
+ "in_proj_a",
114
+ "shared_expert_gate"
115
+ ],
116
+ "type_means": {
117
+ "q_proj": -6.030737484177871,
118
+ "k_proj": -6.50872345475221,
119
+ "v_proj": -5.1295829713035355,
120
+ "o_proj": -5.520641155412461,
121
+ "gate_proj": -5.910868972952471,
122
+ "up_proj": -5.728486041760329,
123
+ "down_proj": -5.587067951821876,
124
+ "out_proj": -6.066408508543807,
125
+ "in_proj_qkv": -5.9912796095587115,
126
+ "in_proj_z": -6.215493840192537,
127
+ "in_proj_b": -7.445906863151851,
128
+ "in_proj_a": -7.4597366899403506,
129
+ "shared_expert_gate": -6.683762662283681
130
+ },
131
+ "global_mean": -6.096409487550172,
132
+ "moe_residual": {
133
+ "type_means": {
134
+ "q_proj": 0.4778202789639603,
135
+ "k_proj": 0.23697090622885114,
136
+ "v_proj": -0.21278645170363397,
137
+ "o_proj": 0.5484346163993171,
138
+ "gate_proj": -0.03176503975089837,
139
+ "up_proj": -0.05422902707833601,
140
+ "down_proj": 0.08310097409720282,
141
+ "out_proj": 0.1621999601730064,
142
+ "in_proj_qkv": -0.06886859919994333,
143
+ "in_proj_z": -0.27820501294154526,
144
+ "in_proj_b": -0.06875325163530918,
145
+ "in_proj_a": -0.091922927112007,
146
+ "shared_expert_gate": -2.40268532061938e-05
147
+ },
148
+ "global_mean": 6.061084000421602e-05,
149
+ "n": 350
150
+ },
151
+ "predict": "score = intercept + sum((x_i - mean_i) / std_i * weight_i); if has_moe, add moe_residual.type_means[layer_type]",
152
+ "trained_on": [
153
+ "meta-llama/Llama-3.1-8B",
154
+ "meta-llama/Llama-3.2-3B",
155
+ "Qwen/Qwen3.6-27B",
156
+ "Qwen/Qwen3.8-27B",
157
+ "Qwen/Qwen3.6-35B-A3B"
158
+ ],
159
+ "leave_one_model": [
160
+ {
161
+ "held": "Qwen/Qwen3.6-27B",
162
+ "mode": "type",
163
+ "n_train": 1265,
164
+ "n_test": 495,
165
+ "spearman": 0.6110909605391778,
166
+ "mae": 0.3989224913369916
167
+ },
168
+ {
169
+ "held": "Qwen/Qwen3.6-35B-A3B",
170
+ "mode": "type",
171
+ "n_train": 1410,
172
+ "n_test": 350,
173
+ "spearman": 0.08936259537978734,
174
+ "mae": 0.85024000770386
175
+ },
176
+ {
177
+ "held": "Qwen/Qwen3.8-27B",
178
+ "mode": "type",
179
+ "n_train": 1264,
180
+ "n_test": 496,
181
+ "spearman": 0.6093684295410784,
182
+ "mae": 0.3970483370198125
183
+ },
184
+ {
185
+ "held": "meta-llama/Llama-3.1-8B",
186
+ "mode": "type",
187
+ "n_train": 1537,
188
+ "n_test": 223,
189
+ "spearman": 0.6420769343975621,
190
+ "mae": 0.5257405388740487
191
+ },
192
+ {
193
+ "held": "meta-llama/Llama-3.2-3B",
194
+ "mode": "type",
195
+ "n_train": 1564,
196
+ "n_test": 196,
197
+ "spearman": 0.6907776777247409,
198
+ "mae": 0.4610140557482808
199
+ }
200
+ ],
201
+ "in_sample_with_residual": [
202
+ {
203
+ "model": "meta-llama/Llama-3.1-8B",
204
+ "n": 223,
205
+ "spearman": 0.7684847724085392,
206
+ "mae": 0.4536931557416361,
207
+ "moe": false
208
+ },
209
+ {
210
+ "model": "meta-llama/Llama-3.2-3B",
211
+ "n": 196,
212
+ "spearman": 0.762396704093586,
213
+ "mae": 0.40142727850839277,
214
+ "moe": false
215
+ },
216
+ {
217
+ "model": "Qwen/Qwen3.6-27B",
218
+ "n": 495,
219
+ "spearman": 0.619066926282546,
220
+ "mae": 0.36812811095602516,
221
+ "moe": false
222
+ },
223
+ {
224
+ "model": "Qwen/Qwen3.8-27B",
225
+ "n": 496,
226
+ "spearman": 0.6172235760604684,
227
+ "mae": 0.3698431368020365,
228
+ "moe": false
229
+ },
230
+ {
231
+ "model": "Qwen/Qwen3.6-35B-A3B",
232
+ "n": 350,
233
+ "spearman": 0.7791595733143011,
234
+ "mae": 0.4092630347573207,
235
+ "moe": true
236
+ }
237
+ ]
238
+ }