Hugging Face's logo

Cactus-Compute
/
needle-pebble-ft

Model card Files
xet
 Community
jakmro commited on
Commit
0e1355e
·
verified ·
1 Parent(s): b1e66b7

Upload finetuned needle INT4 weights

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .gitattributes +58 -0
  2. weights/needle-pebble-ft-int4/config.txt +18 -0
  3. weights/needle-pebble-ft-int4/contrastive_proj.weights +0 -0
  4. weights/needle-pebble-ft-int4/encoder_layer_0_attn_gate.weights +0 -0
  5. weights/needle-pebble-ft-int4/encoder_layer_0_attn_k.weights +0 -0
  6. weights/needle-pebble-ft-int4/encoder_layer_0_attn_k_norm.weights +0 -0
  7. weights/needle-pebble-ft-int4/encoder_layer_0_attn_output.weights +3 -0
  8. weights/needle-pebble-ft-int4/encoder_layer_0_attn_q.weights +3 -0
  9. weights/needle-pebble-ft-int4/encoder_layer_0_attn_q_norm.weights +0 -0
  10. weights/needle-pebble-ft-int4/encoder_layer_0_attn_v.weights +0 -0
  11. weights/needle-pebble-ft-int4/encoder_layer_0_input_norm.weights +0 -0
  12. weights/needle-pebble-ft-int4/encoder_layer_10_attn_gate.weights +0 -0
  13. weights/needle-pebble-ft-int4/encoder_layer_10_attn_k.weights +0 -0
  14. weights/needle-pebble-ft-int4/encoder_layer_10_attn_k_norm.weights +0 -0
  15. weights/needle-pebble-ft-int4/encoder_layer_10_attn_output.weights +3 -0
  16. weights/needle-pebble-ft-int4/encoder_layer_10_attn_q.weights +3 -0
  17. weights/needle-pebble-ft-int4/encoder_layer_10_attn_q_norm.weights +0 -0
  18. weights/needle-pebble-ft-int4/encoder_layer_10_attn_v.weights +0 -0
  19. weights/needle-pebble-ft-int4/encoder_layer_10_input_norm.weights +0 -0
  20. weights/needle-pebble-ft-int4/encoder_layer_11_attn_gate.weights +0 -0
  21. weights/needle-pebble-ft-int4/encoder_layer_11_attn_k.weights +0 -0
  22. weights/needle-pebble-ft-int4/encoder_layer_11_attn_k_norm.weights +0 -0
  23. weights/needle-pebble-ft-int4/encoder_layer_11_attn_output.weights +3 -0
  24. weights/needle-pebble-ft-int4/encoder_layer_11_attn_q.weights +3 -0
  25. weights/needle-pebble-ft-int4/encoder_layer_11_attn_q_norm.weights +0 -0
  26. weights/needle-pebble-ft-int4/encoder_layer_11_attn_v.weights +0 -0
  27. weights/needle-pebble-ft-int4/encoder_layer_11_input_norm.weights +0 -0
  28. weights/needle-pebble-ft-int4/encoder_layer_1_attn_gate.weights +0 -0
  29. weights/needle-pebble-ft-int4/encoder_layer_1_attn_k.weights +0 -0
  30. weights/needle-pebble-ft-int4/encoder_layer_1_attn_k_norm.weights +0 -0
  31. weights/needle-pebble-ft-int4/encoder_layer_1_attn_output.weights +3 -0
  32. weights/needle-pebble-ft-int4/encoder_layer_1_attn_q.weights +3 -0
  33. weights/needle-pebble-ft-int4/encoder_layer_1_attn_q_norm.weights +0 -0
  34. weights/needle-pebble-ft-int4/encoder_layer_1_attn_v.weights +0 -0
  35. weights/needle-pebble-ft-int4/encoder_layer_1_input_norm.weights +0 -0
  36. weights/needle-pebble-ft-int4/encoder_layer_2_attn_gate.weights +0 -0
  37. weights/needle-pebble-ft-int4/encoder_layer_2_attn_k.weights +0 -0
  38. weights/needle-pebble-ft-int4/encoder_layer_2_attn_k_norm.weights +0 -0
  39. weights/needle-pebble-ft-int4/encoder_layer_2_attn_output.weights +3 -0
  40. weights/needle-pebble-ft-int4/encoder_layer_2_attn_q.weights +3 -0
  41. weights/needle-pebble-ft-int4/encoder_layer_2_attn_q_norm.weights +0 -0
  42. weights/needle-pebble-ft-int4/encoder_layer_2_attn_v.weights +0 -0
  43. weights/needle-pebble-ft-int4/encoder_layer_2_input_norm.weights +0 -0
  44. weights/needle-pebble-ft-int4/encoder_layer_3_attn_gate.weights +0 -0
  45. weights/needle-pebble-ft-int4/encoder_layer_3_attn_k.weights +0 -0
  46. weights/needle-pebble-ft-int4/encoder_layer_3_attn_k_norm.weights +0 -0
  47. weights/needle-pebble-ft-int4/encoder_layer_3_attn_output.weights +3 -0
  48. weights/needle-pebble-ft-int4/encoder_layer_3_attn_q.weights +3 -0
  49. weights/needle-pebble-ft-int4/encoder_layer_3_attn_q_norm.weights +0 -0
  50. weights/needle-pebble-ft-int4/encoder_layer_3_attn_v.weights +0 -0
.gitattributes CHANGED
@@ -33,3 +33,61 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ weights/needle-pebble-ft-int4/encoder_layer_0_attn_output.weights filter=lfs diff=lfs merge=lfs -text
37
+ weights/needle-pebble-ft-int4/encoder_layer_0_attn_q.weights filter=lfs diff=lfs merge=lfs -text
38
+ weights/needle-pebble-ft-int4/encoder_layer_10_attn_output.weights filter=lfs diff=lfs merge=lfs -text
39
+ weights/needle-pebble-ft-int4/encoder_layer_10_attn_q.weights filter=lfs diff=lfs merge=lfs -text
40
+ weights/needle-pebble-ft-int4/encoder_layer_11_attn_output.weights filter=lfs diff=lfs merge=lfs -text
41
+ weights/needle-pebble-ft-int4/encoder_layer_11_attn_q.weights filter=lfs diff=lfs merge=lfs -text
42
+ weights/needle-pebble-ft-int4/encoder_layer_1_attn_output.weights filter=lfs diff=lfs merge=lfs -text
43
+ weights/needle-pebble-ft-int4/encoder_layer_1_attn_q.weights filter=lfs diff=lfs merge=lfs -text
44
+ weights/needle-pebble-ft-int4/encoder_layer_2_attn_output.weights filter=lfs diff=lfs merge=lfs -text
45
+ weights/needle-pebble-ft-int4/encoder_layer_2_attn_q.weights filter=lfs diff=lfs merge=lfs -text
46
+ weights/needle-pebble-ft-int4/encoder_layer_3_attn_output.weights filter=lfs diff=lfs merge=lfs -text
47
+ weights/needle-pebble-ft-int4/encoder_layer_3_attn_q.weights filter=lfs diff=lfs merge=lfs -text
48
+ weights/needle-pebble-ft-int4/encoder_layer_4_attn_output.weights filter=lfs diff=lfs merge=lfs -text
49
+ weights/needle-pebble-ft-int4/encoder_layer_4_attn_q.weights filter=lfs diff=lfs merge=lfs -text
50
+ weights/needle-pebble-ft-int4/encoder_layer_5_attn_output.weights filter=lfs diff=lfs merge=lfs -text
51
+ weights/needle-pebble-ft-int4/encoder_layer_5_attn_q.weights filter=lfs diff=lfs merge=lfs -text
52
+ weights/needle-pebble-ft-int4/encoder_layer_6_attn_output.weights filter=lfs diff=lfs merge=lfs -text
53
+ weights/needle-pebble-ft-int4/encoder_layer_6_attn_q.weights filter=lfs diff=lfs merge=lfs -text
54
+ weights/needle-pebble-ft-int4/encoder_layer_7_attn_output.weights filter=lfs diff=lfs merge=lfs -text
55
+ weights/needle-pebble-ft-int4/encoder_layer_7_attn_q.weights filter=lfs diff=lfs merge=lfs -text
56
+ weights/needle-pebble-ft-int4/encoder_layer_8_attn_output.weights filter=lfs diff=lfs merge=lfs -text
57
+ weights/needle-pebble-ft-int4/encoder_layer_8_attn_q.weights filter=lfs diff=lfs merge=lfs -text
58
+ weights/needle-pebble-ft-int4/encoder_layer_9_attn_output.weights filter=lfs diff=lfs merge=lfs -text
59
+ weights/needle-pebble-ft-int4/encoder_layer_9_attn_q.weights filter=lfs diff=lfs merge=lfs -text
60
+ weights/needle-pebble-ft-int4/layer_0_attn_output.weights filter=lfs diff=lfs merge=lfs -text
61
+ weights/needle-pebble-ft-int4/layer_0_attn_q.weights filter=lfs diff=lfs merge=lfs -text
62
+ weights/needle-pebble-ft-int4/layer_0_encoder_attn_output.weights filter=lfs diff=lfs merge=lfs -text
63
+ weights/needle-pebble-ft-int4/layer_0_encoder_attn_q.weights filter=lfs diff=lfs merge=lfs -text
64
+ weights/needle-pebble-ft-int4/layer_1_attn_output.weights filter=lfs diff=lfs merge=lfs -text
65
+ weights/needle-pebble-ft-int4/layer_1_attn_q.weights filter=lfs diff=lfs merge=lfs -text
66
+ weights/needle-pebble-ft-int4/layer_1_encoder_attn_output.weights filter=lfs diff=lfs merge=lfs -text
67
+ weights/needle-pebble-ft-int4/layer_1_encoder_attn_q.weights filter=lfs diff=lfs merge=lfs -text
68
+ weights/needle-pebble-ft-int4/layer_2_attn_output.weights filter=lfs diff=lfs merge=lfs -text
69
+ weights/needle-pebble-ft-int4/layer_2_attn_q.weights filter=lfs diff=lfs merge=lfs -text
70
+ weights/needle-pebble-ft-int4/layer_2_encoder_attn_output.weights filter=lfs diff=lfs merge=lfs -text
71
+ weights/needle-pebble-ft-int4/layer_2_encoder_attn_q.weights filter=lfs diff=lfs merge=lfs -text
72
+ weights/needle-pebble-ft-int4/layer_3_attn_output.weights filter=lfs diff=lfs merge=lfs -text
73
+ weights/needle-pebble-ft-int4/layer_3_attn_q.weights filter=lfs diff=lfs merge=lfs -text
74
+ weights/needle-pebble-ft-int4/layer_3_encoder_attn_output.weights filter=lfs diff=lfs merge=lfs -text
75
+ weights/needle-pebble-ft-int4/layer_3_encoder_attn_q.weights filter=lfs diff=lfs merge=lfs -text
76
+ weights/needle-pebble-ft-int4/layer_4_attn_output.weights filter=lfs diff=lfs merge=lfs -text
77
+ weights/needle-pebble-ft-int4/layer_4_attn_q.weights filter=lfs diff=lfs merge=lfs -text
78
+ weights/needle-pebble-ft-int4/layer_4_encoder_attn_output.weights filter=lfs diff=lfs merge=lfs -text
79
+ weights/needle-pebble-ft-int4/layer_4_encoder_attn_q.weights filter=lfs diff=lfs merge=lfs -text
80
+ weights/needle-pebble-ft-int4/layer_5_attn_output.weights filter=lfs diff=lfs merge=lfs -text
81
+ weights/needle-pebble-ft-int4/layer_5_attn_q.weights filter=lfs diff=lfs merge=lfs -text
82
+ weights/needle-pebble-ft-int4/layer_5_encoder_attn_output.weights filter=lfs diff=lfs merge=lfs -text
83
+ weights/needle-pebble-ft-int4/layer_5_encoder_attn_q.weights filter=lfs diff=lfs merge=lfs -text
84
+ weights/needle-pebble-ft-int4/layer_6_attn_output.weights filter=lfs diff=lfs merge=lfs -text
85
+ weights/needle-pebble-ft-int4/layer_6_attn_q.weights filter=lfs diff=lfs merge=lfs -text
86
+ weights/needle-pebble-ft-int4/layer_6_encoder_attn_output.weights filter=lfs diff=lfs merge=lfs -text
87
+ weights/needle-pebble-ft-int4/layer_6_encoder_attn_q.weights filter=lfs diff=lfs merge=lfs -text
88
+ weights/needle-pebble-ft-int4/layer_7_attn_output.weights filter=lfs diff=lfs merge=lfs -text
89
+ weights/needle-pebble-ft-int4/layer_7_attn_q.weights filter=lfs diff=lfs merge=lfs -text
90
+ weights/needle-pebble-ft-int4/layer_7_encoder_attn_output.weights filter=lfs diff=lfs merge=lfs -text
91
+ weights/needle-pebble-ft-int4/layer_7_encoder_attn_q.weights filter=lfs diff=lfs merge=lfs -text
92
+ weights/needle-pebble-ft-int4/output_weight.weights filter=lfs diff=lfs merge=lfs -text
93
+ weights/needle-pebble-ft-int4/token_embeddings.weights filter=lfs diff=lfs merge=lfs -text
weights/needle-pebble-ft-int4/config.txt ADDED
@@ -0,0 +1,18 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ model_type=needle
2
+ precision=FP16
3
+ quantization=INT4
4
+ vocab_size=8192
5
+ hidden_dim=512
6
+ num_layers=8
7
+ num_encoder_layers=12
8
+ num_decoder_layers=8
9
+ attention_heads=8
10
+ attention_kv_heads=4
11
+ attention_head_dim=64
12
+ ffn_intermediate_dim=2048
13
+ context_length=1024
14
+ rope_theta=10000.0
15
+ layer_norm_eps=1e-06
16
+ pad_token_id=0
17
+ tie_word_embeddings=true
18
+ no_feedforward=true
weights/needle-pebble-ft-int4/contrastive_proj.weights ADDED
Binary file (9.31 kB). View file
 
weights/needle-pebble-ft-int4/encoder_layer_0_attn_gate.weights ADDED
Binary file (98 Bytes). View file
 
weights/needle-pebble-ft-int4/encoder_layer_0_attn_k.weights ADDED
Binary file (73.8 kB). View file
 
weights/needle-pebble-ft-int4/encoder_layer_0_attn_k_norm.weights ADDED
Binary file (224 Bytes). View file
 
weights/needle-pebble-ft-int4/encoder_layer_0_attn_output.weights ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2cf9b306f3d13c2c459941c5beae6160fbde98fbac1855575959e43c625abe49
3
+ size 147552
weights/needle-pebble-ft-int4/encoder_layer_0_attn_q.weights ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8868611edb2e4aa718ae17b08f286ddfe84d431a4fe0fb2c11d97ebf1ce36b75
3
+ size 147552
weights/needle-pebble-ft-int4/encoder_layer_0_attn_q_norm.weights ADDED
Binary file (224 Bytes). View file
 
weights/needle-pebble-ft-int4/encoder_layer_0_attn_v.weights ADDED
Binary file (73.8 kB). View file
 
weights/needle-pebble-ft-int4/encoder_layer_0_input_norm.weights ADDED
Binary file (1.12 kB). View file
 
weights/needle-pebble-ft-int4/encoder_layer_10_attn_gate.weights ADDED
Binary file (98 Bytes). View file
 
weights/needle-pebble-ft-int4/encoder_layer_10_attn_k.weights ADDED
Binary file (73.8 kB). View file
 
weights/needle-pebble-ft-int4/encoder_layer_10_attn_k_norm.weights ADDED
Binary file (224 Bytes). View file
 
weights/needle-pebble-ft-int4/encoder_layer_10_attn_output.weights ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0e95f61e8c4080fe9b533f37b2ffa8cd5f31c3b925b57a1d98f3c2c2e74fb4bf
3
+ size 147552
weights/needle-pebble-ft-int4/encoder_layer_10_attn_q.weights ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8e0df94f9a6c9bd6500686b8cd00b72068983cc70ef1a5e546372ba7818f4cfa
3
+ size 147552
weights/needle-pebble-ft-int4/encoder_layer_10_attn_q_norm.weights ADDED
Binary file (224 Bytes). View file
 
weights/needle-pebble-ft-int4/encoder_layer_10_attn_v.weights ADDED
Binary file (73.8 kB). View file
 
weights/needle-pebble-ft-int4/encoder_layer_10_input_norm.weights ADDED
Binary file (1.12 kB). View file
 
weights/needle-pebble-ft-int4/encoder_layer_11_attn_gate.weights ADDED
Binary file (98 Bytes). View file
 
weights/needle-pebble-ft-int4/encoder_layer_11_attn_k.weights ADDED
Binary file (73.8 kB). View file
 
weights/needle-pebble-ft-int4/encoder_layer_11_attn_k_norm.weights ADDED
Binary file (224 Bytes). View file
 
weights/needle-pebble-ft-int4/encoder_layer_11_attn_output.weights ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:951e4184e1d9e963d13494bc132a28859dc41f039866dd5244f78ea79c51ef66
3
+ size 147552
weights/needle-pebble-ft-int4/encoder_layer_11_attn_q.weights ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:97943b66fffbf50c9b673034e2b7a4072730bd7aaa3baf4d07af405ee780aa79
3
+ size 147552
weights/needle-pebble-ft-int4/encoder_layer_11_attn_q_norm.weights ADDED
Binary file (224 Bytes). View file
 
weights/needle-pebble-ft-int4/encoder_layer_11_attn_v.weights ADDED
Binary file (73.8 kB). View file
 
weights/needle-pebble-ft-int4/encoder_layer_11_input_norm.weights ADDED
Binary file (1.12 kB). View file
 
weights/needle-pebble-ft-int4/encoder_layer_1_attn_gate.weights ADDED
Binary file (98 Bytes). View file
 
weights/needle-pebble-ft-int4/encoder_layer_1_attn_k.weights ADDED
Binary file (73.8 kB). View file
 
weights/needle-pebble-ft-int4/encoder_layer_1_attn_k_norm.weights ADDED
Binary file (224 Bytes). View file
 
weights/needle-pebble-ft-int4/encoder_layer_1_attn_output.weights ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fad3153e1f7c19e3424128627e251f30a69398f579b3dc0464dd70b2f02c85c1
3
+ size 147552
weights/needle-pebble-ft-int4/encoder_layer_1_attn_q.weights ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fa76fe82a430e9ccc844874c80b13352b703ab2aeabc906a7dc4946574f3699d
3
+ size 147552
weights/needle-pebble-ft-int4/encoder_layer_1_attn_q_norm.weights ADDED
Binary file (224 Bytes). View file
 
weights/needle-pebble-ft-int4/encoder_layer_1_attn_v.weights ADDED
Binary file (73.8 kB). View file
 
weights/needle-pebble-ft-int4/encoder_layer_1_input_norm.weights ADDED
Binary file (1.12 kB). View file
 
weights/needle-pebble-ft-int4/encoder_layer_2_attn_gate.weights ADDED
Binary file (98 Bytes). View file
 
weights/needle-pebble-ft-int4/encoder_layer_2_attn_k.weights ADDED
Binary file (73.8 kB). View file
 
weights/needle-pebble-ft-int4/encoder_layer_2_attn_k_norm.weights ADDED
Binary file (224 Bytes). View file
 
weights/needle-pebble-ft-int4/encoder_layer_2_attn_output.weights ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6f5c63e4c8480c87686319fd64b786a50a7a71324b2ae0d096f3262ceba91654
3
+ size 147552
weights/needle-pebble-ft-int4/encoder_layer_2_attn_q.weights ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:034255d51861b8292fe6a25d0e7712feabaa95d534a83aa96a2c2058cb113d84
3
+ size 147552
weights/needle-pebble-ft-int4/encoder_layer_2_attn_q_norm.weights ADDED
Binary file (224 Bytes). View file
 
weights/needle-pebble-ft-int4/encoder_layer_2_attn_v.weights ADDED
Binary file (73.8 kB). View file
 
weights/needle-pebble-ft-int4/encoder_layer_2_input_norm.weights ADDED
Binary file (1.12 kB). View file
 
weights/needle-pebble-ft-int4/encoder_layer_3_attn_gate.weights ADDED
Binary file (98 Bytes). View file
 
weights/needle-pebble-ft-int4/encoder_layer_3_attn_k.weights ADDED
Binary file (73.8 kB). View file
 
weights/needle-pebble-ft-int4/encoder_layer_3_attn_k_norm.weights ADDED
Binary file (224 Bytes). View file
 
weights/needle-pebble-ft-int4/encoder_layer_3_attn_output.weights ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3bcb2ec53966382e168863e8b9541bea0ed5ff02fabd94608ad41a2e544a5d27
3
+ size 147552
weights/needle-pebble-ft-int4/encoder_layer_3_attn_q.weights ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4a4c36cd97c3edbc81d02bc83409c1dfc61ba6c842e261f881ddb152e9878075
3
+ size 147552
weights/needle-pebble-ft-int4/encoder_layer_3_attn_q_norm.weights ADDED
Binary file (224 Bytes). View file
 
weights/needle-pebble-ft-int4/encoder_layer_3_attn_v.weights ADDED
Binary file (73.8 kB). View file