Hugging Face's logo

z-lab
/
Meta-Llama-3-8B-Instruct-SparseLoRA

Safetensors
Model card Files
xet
 Community
zhijianliu commited on
Commit
5f0efdf
·
verified ·
1 Parent(s): 0304219

Upload SparseLoRA predictor checkpoint

Browse files
Files changed (2) hide show
  1. config.json +142 -0
  2. model.safetensors +3 -0
config.json ADDED
@@ -0,0 +1,142 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "base_model": "NousResearch/Meta-Llama-3-8B-Instruct",
3
+ "predictor_rank": 8,
4
+ "modes": {
5
+ "o1": {
6
+ "layer_sparsity": {
7
+ "model.layers.0.mlp": 0,
8
+ "model.layers.0.self_attn": 0,
9
+ "model.layers.1.mlp": 0,
10
+ "model.layers.1.self_attn": 0,
11
+ "model.layers.2.mlp": 0,
12
+ "model.layers.2.self_attn": 0,
13
+ "model.layers.3.mlp": 0,
14
+ "model.layers.3.self_attn": 0,
15
+ "model.layers.4.mlp": 0,
16
+ "model.layers.4.self_attn": 0,
17
+ "model.layers.5.mlp": 0,
18
+ "model.layers.5.self_attn": 0,
19
+ "model.layers.6.mlp": 0,
20
+ "model.layers.6.self_attn": 0,
21
+ "model.layers.7.mlp": 0,
22
+ "model.layers.7.self_attn": 0,
23
+ "model.layers.8.mlp": 0,
24
+ "model.layers.8.self_attn": 0,
25
+ "model.layers.9.mlp": 0,
26
+ "model.layers.9.self_attn": 0,
27
+ "model.layers.10.mlp": 0,
28
+ "model.layers.10.self_attn": 0,
29
+ "model.layers.11.mlp": 0,
30
+ "model.layers.11.self_attn": 0,
31
+ "model.layers.12.mlp": 0,
32
+ "model.layers.12.self_attn": 0,
33
+ "model.layers.13.mlp": 0,
34
+ "model.layers.13.self_attn": 0,
35
+ "model.layers.14.mlp": 0,
36
+ "model.layers.14.self_attn": 0,
37
+ "model.layers.15.mlp": 0,
38
+ "model.layers.15.self_attn": 0,
39
+ "model.layers.16.mlp": 0,
40
+ "model.layers.16.self_attn": 0,
41
+ "model.layers.17.mlp": 0.97,
42
+ "model.layers.17.self_attn": 0.2,
43
+ "model.layers.18.mlp": 0.97,
44
+ "model.layers.18.self_attn": 0.2,
45
+ "model.layers.19.mlp": 0.97,
46
+ "model.layers.19.self_attn": 0.2,
47
+ "model.layers.20.mlp": 0.97,
48
+ "model.layers.20.self_attn": 0,
49
+ "model.layers.21.mlp": 0.97,
50
+ "model.layers.21.self_attn": 0.2,
51
+ "model.layers.22.mlp": 0.97,
52
+ "model.layers.22.self_attn": 0.2,
53
+ "model.layers.23.mlp": 0.97,
54
+ "model.layers.23.self_attn": 0.2,
55
+ "model.layers.24.mlp": 0.97,
56
+ "model.layers.24.self_attn": 0,
57
+ "model.layers.25.mlp": 0.97,
58
+ "model.layers.25.self_attn": 0.2,
59
+ "model.layers.26.mlp": 0.97,
60
+ "model.layers.26.self_attn": 0.2,
61
+ "model.layers.27.mlp": 0.97,
62
+ "model.layers.27.self_attn": 0.2,
63
+ "model.layers.28.mlp": 0.97,
64
+ "model.layers.28.self_attn": 0.2,
65
+ "model.layers.29.mlp": 0.97,
66
+ "model.layers.29.self_attn": 0.2,
67
+ "model.layers.30.mlp": 0.97,
68
+ "model.layers.30.self_attn": 0,
69
+ "model.layers.31.mlp": 0,
70
+ "model.layers.31.self_attn": 0
71
+ }
72
+ },
73
+ "o2": {
74
+ "layer_sparsity": {
75
+ "model.layers.0.mlp": 0,
76
+ "model.layers.0.self_attn": 0,
77
+ "model.layers.1.mlp": 0,
78
+ "model.layers.1.self_attn": 0,
79
+ "model.layers.2.mlp": 0,
80
+ "model.layers.2.self_attn": 0,
81
+ "model.layers.3.mlp": 0.99,
82
+ "model.layers.3.self_attn": 0,
83
+ "model.layers.4.mlp": 0.99,
84
+ "model.layers.4.self_attn": 0,
85
+ "model.layers.5.mlp": 0.99,
86
+ "model.layers.5.self_attn": 0,
87
+ "model.layers.6.mlp": 0.99,
88
+ "model.layers.6.self_attn": 0,
89
+ "model.layers.7.mlp": 0.99,
90
+ "model.layers.7.self_attn": 0,
91
+ "model.layers.8.mlp": 0.99,
92
+ "model.layers.8.self_attn": 0,
93
+ "model.layers.9.mlp": 0.99,
94
+ "model.layers.9.self_attn": 0,
95
+ "model.layers.10.mlp": 0.99,
96
+ "model.layers.10.self_attn": 0,
97
+ "model.layers.11.mlp": 0.99,
98
+ "model.layers.11.self_attn": 0,
99
+ "model.layers.12.mlp": 0.99,
100
+ "model.layers.12.self_attn": 0,
101
+ "model.layers.13.mlp": 0.99,
102
+ "model.layers.13.self_attn": 0,
103
+ "model.layers.14.mlp": 0.99,
104
+ "model.layers.14.self_attn": 0.75,
105
+ "model.layers.15.mlp": 0.99,
106
+ "model.layers.15.self_attn": 0.75,
107
+ "model.layers.16.mlp": 0.99,
108
+ "model.layers.16.self_attn": 0.75,
109
+ "model.layers.17.mlp": 0.99,
110
+ "model.layers.17.self_attn": 0.75,
111
+ "model.layers.18.mlp": 0.99,
112
+ "model.layers.18.self_attn": 0.75,
113
+ "model.layers.19.mlp": 0.99,
114
+ "model.layers.19.self_attn": 0.75,
115
+ "model.layers.20.mlp": 0.99,
116
+ "model.layers.20.self_attn": 0,
117
+ "model.layers.21.mlp": 0.99,
118
+ "model.layers.21.self_attn": 0.75,
119
+ "model.layers.22.mlp": 0.99,
120
+ "model.layers.22.self_attn": 0.75,
121
+ "model.layers.23.mlp": 0.99,
122
+ "model.layers.23.self_attn": 0.75,
123
+ "model.layers.24.mlp": 0.99,
124
+ "model.layers.24.self_attn": 0,
125
+ "model.layers.25.mlp": 0.99,
126
+ "model.layers.25.self_attn": 0.75,
127
+ "model.layers.26.mlp": 0.99,
128
+ "model.layers.26.self_attn": 0.75,
129
+ "model.layers.27.mlp": 0.99,
130
+ "model.layers.27.self_attn": 0.75,
131
+ "model.layers.28.mlp": 0.99,
132
+ "model.layers.28.self_attn": 0.75,
133
+ "model.layers.29.mlp": 0.99,
134
+ "model.layers.29.self_attn": 0.75,
135
+ "model.layers.30.mlp": 0.99,
136
+ "model.layers.30.self_attn": 0,
137
+ "model.layers.31.mlp": 0,
138
+ "model.layers.31.self_attn": 0
139
+ }
140
+ }
141
+ }
142
+ }
model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8f12a7f11100f773b8dff4b9782e94f9ee782467fbe87f25ef4d928ba2fe95df
3
+ size 28345000