Hugging Face's logo

silx-ai
/
Quasar-Preview

Text Generation
Transformers
Safetensors
English
Arabic
quasar_long
silx-ai
quasar-preview
quasar
foundation-model
Mixture of Experts
18b
2b-active
long-context
bittensor
sn24
decentralized-training
distillation
hybrid-transformer
loop-transformer
safe-nope
drope
conversational
custom_code
Model card Files
xet
 Community
4
eyad-silx commited on
Commit
dd59ce8
·
verified ·
1 Parent(s): df13683

Update Quasar-Preview weights from checkpoint-10900

Browse files
Files changed (44) hide show
  1. .gitattributes +1 -0
  2. __init__.py +0 -0
  3. __pycache__/__init__.cpython-312.pyc +0 -0
  4. __pycache__/configuration_quasar_long.cpython-312.pyc +0 -0
  5. __pycache__/engram.cpython-312.pyc +0 -0
  6. __pycache__/modeling_quasar_long.cpython-312.pyc +3 -0
  7. config.json +2 -2
  8. generation_config.json +1 -4
  9. model-00001-of-00020.safetensors +2 -2
  10. model-00002-of-00020.safetensors +2 -2
  11. model-00003-of-00020.safetensors +2 -2
  12. model-00004-of-00020.safetensors +2 -2
  13. model-00005-of-00020.safetensors +2 -2
  14. model-00006-of-00020.safetensors +2 -2
  15. model-00007-of-00020.safetensors +2 -2
  16. model-00008-of-00020.safetensors +2 -2
  17. model-00009-of-00020.safetensors +2 -2
  18. model-00010-of-00020.safetensors +2 -2
  19. model-00011-of-00020.safetensors +2 -2
  20. model-00012-of-00020.safetensors +2 -2
  21. model-00013-of-00020.safetensors +2 -2
  22. model-00014-of-00020.safetensors +2 -2
  23. model-00015-of-00020.safetensors +2 -2
  24. model-00016-of-00020.safetensors +2 -2
  25. model-00017-of-00020.safetensors +2 -2
  26. model-00018-of-00020.safetensors +2 -2
  27. model-00019-of-00020.safetensors +2 -2
  28. model-00020-of-00020.safetensors +2 -2
  29. model.safetensors.index.json +517 -517
  30. modeling_quasar_long.py +1 -0
  31. raven/__init__.py +7 -0
  32. raven/__pycache__/__init__.cpython-312.pyc +0 -0
  33. raven/layers/__init__.py +5 -0
  34. raven/layers/__pycache__/__init__.cpython-312.pyc +0 -0
  35. raven/layers/__pycache__/raven.cpython-312.pyc +0 -0
  36. raven/layers/raven.py +348 -0
  37. raven/models/__init__.py +1 -0
  38. raven/models/__pycache__/__init__.cpython-312.pyc +0 -0
  39. raven/models/raven/__init__.py +12 -0
  40. raven/models/raven/__pycache__/__init__.cpython-312.pyc +0 -0
  41. raven/models/raven/__pycache__/configuration_raven.cpython-312.pyc +0 -0
  42. raven/models/raven/__pycache__/modeling_raven.cpython-312.pyc +0 -0
  43. raven/models/raven/configuration_raven.py +111 -0
  44. raven/models/raven/modeling_raven.py +429 -0
.gitattributes CHANGED
@@ -35,3 +35,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
  tokenizer.json filter=lfs diff=lfs merge=lfs -text
37
  quasar_banner.png filter=lfs diff=lfs merge=lfs -text
 
 
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
  tokenizer.json filter=lfs diff=lfs merge=lfs -text
37
  quasar_banner.png filter=lfs diff=lfs merge=lfs -text
38
+ __pycache__/modeling_quasar_long.cpython-312.pyc filter=lfs diff=lfs merge=lfs -text
__init__.py ADDED
File without changes
__pycache__/__init__.cpython-312.pyc ADDED
Binary file (151 Bytes). View file
 
__pycache__/configuration_quasar_long.cpython-312.pyc ADDED
Binary file (4.34 kB). View file
 
__pycache__/engram.cpython-312.pyc ADDED
Binary file (36.6 kB). View file
 
__pycache__/modeling_quasar_long.cpython-312.pyc ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bcffd4f449e7c5ccd77a875680f3ab8301d90b87892ece52f0be580c0efdb33f
3
+ size 174383
config.json CHANGED
@@ -1,5 +1,4 @@
1
  {
2
- "_name_or_path": "silx-ai/Quasar-T",
3
  "architectures": [
4
  "QuasarLongForCausalLM"
5
  ],
@@ -104,7 +103,8 @@
104
  "score_function": "sigmoid",
105
  "tie_word_embeddings": false,
106
  "topk_group": 4,
107
- "transformers_version": "5.10.2",
 
108
  "use_bias": false,
109
  "use_cache": true,
110
  "use_looped_injection": false,
 
1
  {
 
2
  "architectures": [
3
  "QuasarLongForCausalLM"
4
  ],
 
103
  "score_function": "sigmoid",
104
  "tie_word_embeddings": false,
105
  "topk_group": 4,
106
+ "torch_dtype": "bfloat16",
107
+ "transformers_version": "4.52.3",
108
  "use_bias": false,
109
  "use_cache": true,
110
  "use_looped_injection": false,
generation_config.json CHANGED
@@ -1,9 +1,6 @@
1
  {
2
  "_from_model_config": true,
3
  "eos_token_id": 156892,
4
- "output_attentions": false,
5
- "output_hidden_states": false,
6
  "pad_token_id": 156892,
7
- "transformers_version": "5.10.2",
8
- "use_cache": false
9
  }
 
1
  {
2
  "_from_model_config": true,
3
  "eos_token_id": 156892,
 
 
4
  "pad_token_id": 156892,
5
+ "transformers_version": "4.52.3"
 
6
  }
model-00001-of-00020.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:ea35cabb6bcb9a3d98af7c2d23b74a6c1c16d7b2250d1b33f4adf1148e1527cc
3
- size 3644870344
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9890977d4bc2031b8481792fc02821470cf8525a8c11b377373b4e97b91882a2
3
+ size 1822436040
model-00002-of-00020.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:d46fc3c261789b35418e83d44ddb7e14b57d90f5357e9245285fdc586cc02735
3
- size 3277868552
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a0332bffe8dc85e769bb16b046e943e45e26b9f1e7cf7bb17e574a553094853b
3
+ size 2204128136
model-00003-of-00020.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:c591c0a0e142188c63be1c33dc718a3e9ee3c3ba64ce8756feaaa6559d35cad9
3
- size 3277868552
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:be88e65e22af5a913f2985b360cc45389a38cc976b0794dfbc37510e315c01f7
3
+ size 1682138422
model-00004-of-00020.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:c72452db4fa323c790cc732f731d0a5eecbbef21bcf333083e0c08fbb7059a35
3
- size 3364272708
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3c4db1de2750c20e28312240801a90d0026a8690c3fc495b564f56bfc8514cbf
3
+ size 1664308670
model-00005-of-00020.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:7483399d4af1b189702dfe1c54f6d93bc9eb0544acbf0018f5c631176ece89a4
3
- size 3328613356
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9967d52642ad8eef099f223d60a658cccaa9381bad569731fa2a1829efd98d9a
3
+ size 1682138422
model-00006-of-00020.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:6a5153b37b6f06b6e2fdceff1e115513e1dbe713f824c9bca73ddaf268971130
3
- size 3364272708
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7677132ba3b705f73af722c4af23af362e96f734cec372cb4744a9e5b3cca81f
3
+ size 1682138422
model-00007-of-00020.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:bc7f64f0ce413e15e6f1bab1b9507305a2080d754f0450017993b4924ddbeb3d
3
- size 3364272708
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:736d8aee814c1c253e21a25380d792ba2b8e83b9bbc1de47134393f8bba24b13
3
+ size 1668436894
model-00008-of-00020.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:2116d3232611e812a81c7247e6262e38fccc129ff4be7cfd7c4c753e359eca15
3
- size 3336870324
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f24b64d7d612d58b014d0a987b91b86febe0764187fc8625aa93e265dd2c88a6
3
+ size 1682138422
model-00009-of-00020.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:3d54aa7600d0880a26c8d887448515ab70952d0bb653c75191934467347aaa34
3
- size 3364272708
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:63c9bb0da5c950e7715c584b35fc78e255cf285e4f195e23b9f903deadb2634b
3
+ size 1664308542
model-00010-of-00020.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:7afc84fcfc4308a29fd9a9d4e702505c390a4bf34849ca8d10e64cba252e0736
3
- size 3328613244
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6cfe62e7d83d0f115024c9b382abd71da39b057aa52296671b82f191d1634ad5
3
+ size 1682138454
model-00011-of-00020.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:8232672b670795f66f6b874ce344a5ef67f42b4ee5da0507bf746dd3330e3c6f
3
- size 3364272740
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ef30e29cde8b5a2a5c1228e8b04ee64579be4f4d99c8f653745fe4c1bebbaa71
3
+ size 1682138454
model-00012-of-00020.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:a86044f975598c5b4eb60f5208c6224fdd024b7a4e740ddb15ed9160166d0471
3
- size 3364272740
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6276d1b5cdb0163911fb7f8f0f536c68f73dcdcf8278eb8c53fa97acea4eaa42
3
+ size 1668436926
model-00013-of-00020.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:3fc3f688bf2d9fc9ccca11642231d4053a8eaca1772107dd539beb41a5ea4d52
3
- size 3336870348
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bb77ee8382922f5ca5f0ead32a2fd6d941d9752fb1c3776a0904db62709aaff9
3
+ size 1682138454
model-00014-of-00020.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:0754df10e787162c5a54d7245f63a82df6d7a26738cdb4eb7ee375693d1603a2
3
- size 3364272740
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:03940ce137ecb618bdd0b92d69bf48c33db756516520f50e40fb26310dcb029b
3
+ size 1664308702
model-00015-of-00020.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:0bbb28aadeab35e80ad577864c680a90574132b1f2a76803cce54887457c0123
3
- size 3328613388
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e1690c93e4e3e5c457cfa6a44a3fad954b30c4488cc755cd722872fdb1e8ff97
3
+ size 1682138454
model-00016-of-00020.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:edd656d1e7c6daea6e1dcbc1ef36de0a8dbb0e54fa39211db639ca02ce5e7220
3
- size 3364272740
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:206da5cef429ce642b01e53d566c1edf753c002b2bdd53f915c4dad20618539a
3
+ size 1682138454
model-00017-of-00020.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:239998760c692b641107311e31b70ca42f89777973c390f71c5ed20f96bd01df
3
- size 3364272740
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f45fa6d9bcdad4f2c01e3fd1a5ba3860c5c5d03bdd3115850f172d43e7972708
3
+ size 1668436926
model-00018-of-00020.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:d948b2ec33dc3166c5af404c67d5aee89a1ad4b3242463121a91cb92083d5f48
3
- size 3336870348
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4b8f33c0a808df6d38961012360c593eb6015511ce4ed1d7a4e3d9cd434b6f59
3
+ size 1682138454
model-00019-of-00020.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:9e8c98b8722eafa5ac5f957ee1f06163c745aff1cd3963f0c2b91a31c1658827
3
- size 3364272740
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a705e402649491b1730f382bded63f4c61e0f41dcebb77f48773b898f6ea3323
3
+ size 1617966904
model-00020-of-00020.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:909c38d3a8b436f83a388ef94089330645ce82b2eb8b9f84d915b6831d4a4401
3
- size 2376100200
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:69f535ac19eb93716ab84568efcf221907a610bd4d25add512157c1ffbddeaf8
3
+ size 643825792
model.safetensors.index.json CHANGED
@@ -1,6 +1,6 @@
1
  {
2
  "metadata": {
3
- "total_size": 66215816960
4
  },
5
  "weight_map": {
6
  "lm_head.weight": "model-00020-of-00020.safetensors",
@@ -26,544 +26,544 @@
26
  "model.layers.1.mlp.shared_experts.gate_proj.weight": "model-00002-of-00020.safetensors",
27
  "model.layers.1.mlp.shared_experts.up_proj.weight": "model-00002-of-00020.safetensors",
28
  "model.layers.1.post_attention_layernorm.weight": "model-00002-of-00020.safetensors",
29
- "model.layers.10.attention.branch_global_output_gain": "model-00010-of-00020.safetensors",
30
- "model.layers.10.attention.branch_local_window_mix_logit": "model-00010-of-00020.safetensors",
31
- "model.layers.10.attention.branch_mix_logits": "model-00010-of-00020.safetensors",
32
- "model.layers.10.attention.branch_output_adapter_down.weight": "model-00010-of-00020.safetensors",
33
- "model.layers.10.attention.branch_output_adapter_up.weight": "model-00010-of-00020.safetensors",
34
- "model.layers.10.attention.branch_output_channel_gain": "model-00010-of-00020.safetensors",
35
- "model.layers.10.attention.branch_output_gain": "model-00010-of-00020.safetensors",
36
- "model.layers.10.attention.dense.weight": "model-00010-of-00020.safetensors",
37
- "model.layers.10.attention.key_layernorm.weight": "model-00010-of-00020.safetensors",
38
- "model.layers.10.attention.query_key_value.weight": "model-00010-of-00020.safetensors",
39
- "model.layers.10.attention.query_layernorm.weight": "model-00010-of-00020.safetensors",
40
- "model.layers.10.attention.raven_attention.A_log": "model-00010-of-00020.safetensors",
41
- "model.layers.10.attention.raven_attention.a_proj.weight": "model-00010-of-00020.safetensors",
42
- "model.layers.10.attention.raven_attention.dt_bias": "model-00010-of-00020.safetensors",
43
- "model.layers.10.attention.raven_attention.g_norm.weight": "model-00010-of-00020.safetensors",
44
- "model.layers.10.attention.raven_attention.k_norm.weight": "model-00010-of-00020.safetensors",
45
- "model.layers.10.attention.raven_attention.k_proj.weight": "model-00010-of-00020.safetensors",
46
- "model.layers.10.attention.raven_attention.o_proj.weight": "model-00010-of-00020.safetensors",
47
- "model.layers.10.attention.raven_attention.q_norm.weight": "model-00010-of-00020.safetensors",
48
- "model.layers.10.attention.raven_attention.q_proj.weight": "model-00010-of-00020.safetensors",
49
- "model.layers.10.attention.raven_attention.r_proj.weight": "model-00010-of-00020.safetensors",
50
- "model.layers.10.attention.raven_attention.v_proj.weight": "model-00010-of-00020.safetensors",
51
- "model.layers.10.attention.replace_alpha_raw": "model-00010-of-00020.safetensors",
52
- "model.layers.10.input_layernorm.weight": "model-00011-of-00020.safetensors",
53
  "model.layers.10.mlp.experts_w12": "model-00010-of-00020.safetensors",
54
- "model.layers.10.mlp.experts_w3": "model-00011-of-00020.safetensors",
55
- "model.layers.10.mlp.gate.expert_bias": "model-00011-of-00020.safetensors",
56
- "model.layers.10.mlp.gate.weight": "model-00011-of-00020.safetensors",
57
- "model.layers.10.mlp.shared_experts.down_proj.weight": "model-00011-of-00020.safetensors",
58
- "model.layers.10.mlp.shared_experts.gate_proj.weight": "model-00011-of-00020.safetensors",
59
- "model.layers.10.mlp.shared_experts.up_proj.weight": "model-00011-of-00020.safetensors",
60
- "model.layers.10.post_attention_layernorm.weight": "model-00011-of-00020.safetensors",
61
- "model.layers.11.attention.branch_global_output_gain": "model-00011-of-00020.safetensors",
62
- "model.layers.11.attention.branch_local_window_mix_logit": "model-00011-of-00020.safetensors",
63
- "model.layers.11.attention.branch_mix_logits": "model-00011-of-00020.safetensors",
64
- "model.layers.11.attention.branch_output_adapter_down.weight": "model-00011-of-00020.safetensors",
65
- "model.layers.11.attention.branch_output_adapter_up.weight": "model-00011-of-00020.safetensors",
66
- "model.layers.11.attention.branch_output_channel_gain": "model-00011-of-00020.safetensors",
67
- "model.layers.11.attention.branch_output_gain": "model-00011-of-00020.safetensors",
68
- "model.layers.11.attention.dense.weight": "model-00011-of-00020.safetensors",
69
- "model.layers.11.attention.key_layernorm.weight": "model-00011-of-00020.safetensors",
70
- "model.layers.11.attention.quasar_attention.A_log": "model-00011-of-00020.safetensors",
71
- "model.layers.11.attention.quasar_attention.b_proj.weight": "model-00011-of-00020.safetensors",
72
- "model.layers.11.attention.quasar_attention.dt_bias": "model-00011-of-00020.safetensors",
73
- "model.layers.11.attention.quasar_attention.f_proj.weight": "model-00011-of-00020.safetensors",
74
- "model.layers.11.attention.quasar_attention.g_proj.0.weight": "model-00011-of-00020.safetensors",
75
- "model.layers.11.attention.quasar_attention.g_proj.1.bias": "model-00011-of-00020.safetensors",
76
- "model.layers.11.attention.quasar_attention.g_proj.1.weight": "model-00011-of-00020.safetensors",
77
- "model.layers.11.attention.quasar_attention.k_proj.weight": "model-00011-of-00020.safetensors",
78
- "model.layers.11.attention.quasar_attention.o_norm.weight": "model-00011-of-00020.safetensors",
79
- "model.layers.11.attention.quasar_attention.o_proj.weight": "model-00011-of-00020.safetensors",
80
- "model.layers.11.attention.quasar_attention.q_proj.weight": "model-00011-of-00020.safetensors",
81
- "model.layers.11.attention.quasar_attention.v_proj.weight": "model-00011-of-00020.safetensors",
82
- "model.layers.11.attention.query_key_value.weight": "model-00011-of-00020.safetensors",
83
- "model.layers.11.attention.query_layernorm.weight": "model-00011-of-00020.safetensors",
84
- "model.layers.11.attention.replace_alpha_raw": "model-00011-of-00020.safetensors",
85
- "model.layers.11.input_layernorm.weight": "model-00012-of-00020.safetensors",
86
  "model.layers.11.mlp.experts_w12": "model-00011-of-00020.safetensors",
87
- "model.layers.11.mlp.experts_w3": "model-00012-of-00020.safetensors",
88
- "model.layers.11.mlp.gate.expert_bias": "model-00012-of-00020.safetensors",
89
- "model.layers.11.mlp.gate.weight": "model-00012-of-00020.safetensors",
90
- "model.layers.11.mlp.shared_experts.down_proj.weight": "model-00012-of-00020.safetensors",
91
- "model.layers.11.mlp.shared_experts.gate_proj.weight": "model-00012-of-00020.safetensors",
92
- "model.layers.11.mlp.shared_experts.up_proj.weight": "model-00012-of-00020.safetensors",
93
- "model.layers.11.post_attention_layernorm.weight": "model-00012-of-00020.safetensors",
94
- "model.layers.12.attention.branch_global_output_gain": "model-00012-of-00020.safetensors",
95
- "model.layers.12.attention.branch_local_window_mix_logit": "model-00012-of-00020.safetensors",
96
- "model.layers.12.attention.branch_mix_logits": "model-00012-of-00020.safetensors",
97
- "model.layers.12.attention.branch_output_adapter_down.weight": "model-00012-of-00020.safetensors",
98
- "model.layers.12.attention.branch_output_adapter_up.weight": "model-00012-of-00020.safetensors",
99
- "model.layers.12.attention.branch_output_channel_gain": "model-00012-of-00020.safetensors",
100
- "model.layers.12.attention.branch_output_gain": "model-00012-of-00020.safetensors",
101
- "model.layers.12.attention.dense.weight": "model-00012-of-00020.safetensors",
102
- "model.layers.12.attention.key_layernorm.weight": "model-00012-of-00020.safetensors",
103
- "model.layers.12.attention.quasar_attention.A_log": "model-00012-of-00020.safetensors",
104
- "model.layers.12.attention.quasar_attention.b_proj.weight": "model-00012-of-00020.safetensors",
105
- "model.layers.12.attention.quasar_attention.dt_bias": "model-00012-of-00020.safetensors",
106
- "model.layers.12.attention.quasar_attention.f_proj.weight": "model-00012-of-00020.safetensors",
107
- "model.layers.12.attention.quasar_attention.g_proj.0.weight": "model-00012-of-00020.safetensors",
108
- "model.layers.12.attention.quasar_attention.g_proj.1.bias": "model-00012-of-00020.safetensors",
109
- "model.layers.12.attention.quasar_attention.g_proj.1.weight": "model-00012-of-00020.safetensors",
110
- "model.layers.12.attention.quasar_attention.k_proj.weight": "model-00012-of-00020.safetensors",
111
- "model.layers.12.attention.quasar_attention.o_norm.weight": "model-00012-of-00020.safetensors",
112
- "model.layers.12.attention.quasar_attention.o_proj.weight": "model-00012-of-00020.safetensors",
113
- "model.layers.12.attention.quasar_attention.q_proj.weight": "model-00012-of-00020.safetensors",
114
- "model.layers.12.attention.quasar_attention.v_proj.weight": "model-00012-of-00020.safetensors",
115
- "model.layers.12.attention.query_key_value.weight": "model-00012-of-00020.safetensors",
116
- "model.layers.12.attention.query_layernorm.weight": "model-00012-of-00020.safetensors",
117
- "model.layers.12.attention.replace_alpha_raw": "model-00012-of-00020.safetensors",
118
- "model.layers.12.input_layernorm.weight": "model-00013-of-00020.safetensors",
119
  "model.layers.12.mlp.experts_w12": "model-00012-of-00020.safetensors",
120
- "model.layers.12.mlp.experts_w3": "model-00013-of-00020.safetensors",
121
- "model.layers.12.mlp.gate.expert_bias": "model-00013-of-00020.safetensors",
122
- "model.layers.12.mlp.gate.weight": "model-00013-of-00020.safetensors",
123
- "model.layers.12.mlp.shared_experts.down_proj.weight": "model-00013-of-00020.safetensors",
124
- "model.layers.12.mlp.shared_experts.gate_proj.weight": "model-00013-of-00020.safetensors",
125
- "model.layers.12.mlp.shared_experts.up_proj.weight": "model-00013-of-00020.safetensors",
126
- "model.layers.12.post_attention_layernorm.weight": "model-00013-of-00020.safetensors",
127
- "model.layers.13.attention.branch_global_output_gain": "model-00013-of-00020.safetensors",
128
- "model.layers.13.attention.branch_local_window_mix_logit": "model-00013-of-00020.safetensors",
129
- "model.layers.13.attention.branch_mix_logits": "model-00013-of-00020.safetensors",
130
- "model.layers.13.attention.branch_output_adapter_down.weight": "model-00013-of-00020.safetensors",
131
- "model.layers.13.attention.branch_output_adapter_up.weight": "model-00013-of-00020.safetensors",
132
- "model.layers.13.attention.branch_output_channel_gain": "model-00013-of-00020.safetensors",
133
- "model.layers.13.attention.branch_output_gain": "model-00013-of-00020.safetensors",
134
- "model.layers.13.attention.dense.weight": "model-00013-of-00020.safetensors",
135
- "model.layers.13.attention.gla_attention.dense.weight": "model-00013-of-00020.safetensors",
136
- "model.layers.13.attention.gla_attention.g_norm.weight": "model-00013-of-00020.safetensors",
137
- "model.layers.13.attention.gla_attention.g_proj.weight": "model-00013-of-00020.safetensors",
138
- "model.layers.13.attention.gla_attention.key_layernorm.weight": "model-00013-of-00020.safetensors",
139
- "model.layers.13.attention.gla_attention.query_key_value.weight": "model-00013-of-00020.safetensors",
140
- "model.layers.13.attention.gla_attention.query_layernorm.weight": "model-00013-of-00020.safetensors",
141
- "model.layers.13.attention.gla_attention.slope": "model-00013-of-00020.safetensors",
142
- "model.layers.13.attention.key_layernorm.weight": "model-00013-of-00020.safetensors",
143
- "model.layers.13.attention.query_key_value.weight": "model-00013-of-00020.safetensors",
144
- "model.layers.13.attention.query_layernorm.weight": "model-00013-of-00020.safetensors",
145
- "model.layers.13.attention.replace_alpha_raw": "model-00013-of-00020.safetensors",
146
- "model.layers.13.input_layernorm.weight": "model-00014-of-00020.safetensors",
147
  "model.layers.13.mlp.experts_w12": "model-00013-of-00020.safetensors",
148
- "model.layers.13.mlp.experts_w3": "model-00014-of-00020.safetensors",
149
- "model.layers.13.mlp.gate.expert_bias": "model-00014-of-00020.safetensors",
150
- "model.layers.13.mlp.gate.weight": "model-00014-of-00020.safetensors",
151
- "model.layers.13.mlp.shared_experts.down_proj.weight": "model-00014-of-00020.safetensors",
152
- "model.layers.13.mlp.shared_experts.gate_proj.weight": "model-00014-of-00020.safetensors",
153
- "model.layers.13.mlp.shared_experts.up_proj.weight": "model-00014-of-00020.safetensors",
154
- "model.layers.13.post_attention_layernorm.weight": "model-00014-of-00020.safetensors",
155
- "model.layers.14.attention.branch_global_output_gain": "model-00014-of-00020.safetensors",
156
- "model.layers.14.attention.branch_local_window_mix_logit": "model-00014-of-00020.safetensors",
157
- "model.layers.14.attention.branch_mix_logits": "model-00014-of-00020.safetensors",
158
- "model.layers.14.attention.branch_output_adapter_down.weight": "model-00014-of-00020.safetensors",
159
- "model.layers.14.attention.branch_output_adapter_up.weight": "model-00014-of-00020.safetensors",
160
- "model.layers.14.attention.branch_output_channel_gain": "model-00014-of-00020.safetensors",
161
- "model.layers.14.attention.branch_output_gain": "model-00014-of-00020.safetensors",
162
- "model.layers.14.attention.dense.weight": "model-00014-of-00020.safetensors",
163
- "model.layers.14.attention.key_layernorm.weight": "model-00014-of-00020.safetensors",
164
- "model.layers.14.attention.quasar_attention.A_log": "model-00014-of-00020.safetensors",
165
- "model.layers.14.attention.quasar_attention.b_proj.weight": "model-00014-of-00020.safetensors",
166
- "model.layers.14.attention.quasar_attention.dt_bias": "model-00014-of-00020.safetensors",
167
- "model.layers.14.attention.quasar_attention.f_proj.weight": "model-00014-of-00020.safetensors",
168
- "model.layers.14.attention.quasar_attention.g_proj.0.weight": "model-00014-of-00020.safetensors",
169
- "model.layers.14.attention.quasar_attention.g_proj.1.bias": "model-00014-of-00020.safetensors",
170
- "model.layers.14.attention.quasar_attention.g_proj.1.weight": "model-00014-of-00020.safetensors",
171
- "model.layers.14.attention.quasar_attention.k_proj.weight": "model-00014-of-00020.safetensors",
172
- "model.layers.14.attention.quasar_attention.o_norm.weight": "model-00014-of-00020.safetensors",
173
- "model.layers.14.attention.quasar_attention.o_proj.weight": "model-00014-of-00020.safetensors",
174
- "model.layers.14.attention.quasar_attention.q_proj.weight": "model-00014-of-00020.safetensors",
175
- "model.layers.14.attention.quasar_attention.v_proj.weight": "model-00014-of-00020.safetensors",
176
- "model.layers.14.attention.query_key_value.weight": "model-00014-of-00020.safetensors",
177
- "model.layers.14.attention.query_layernorm.weight": "model-00014-of-00020.safetensors",
178
- "model.layers.14.attention.replace_alpha_raw": "model-00014-of-00020.safetensors",
179
- "model.layers.14.input_layernorm.weight": "model-00015-of-00020.safetensors",
180
  "model.layers.14.mlp.experts_w12": "model-00014-of-00020.safetensors",
181
- "model.layers.14.mlp.experts_w3": "model-00015-of-00020.safetensors",
182
- "model.layers.14.mlp.gate.expert_bias": "model-00015-of-00020.safetensors",
183
- "model.layers.14.mlp.gate.weight": "model-00015-of-00020.safetensors",
184
- "model.layers.14.mlp.shared_experts.down_proj.weight": "model-00015-of-00020.safetensors",
185
- "model.layers.14.mlp.shared_experts.gate_proj.weight": "model-00015-of-00020.safetensors",
186
- "model.layers.14.mlp.shared_experts.up_proj.weight": "model-00015-of-00020.safetensors",
187
- "model.layers.14.post_attention_layernorm.weight": "model-00015-of-00020.safetensors",
188
- "model.layers.15.attention.branch_global_output_gain": "model-00015-of-00020.safetensors",
189
- "model.layers.15.attention.branch_local_window_mix_logit": "model-00015-of-00020.safetensors",
190
- "model.layers.15.attention.branch_mix_logits": "model-00015-of-00020.safetensors",
191
- "model.layers.15.attention.branch_output_adapter_down.weight": "model-00015-of-00020.safetensors",
192
- "model.layers.15.attention.branch_output_adapter_up.weight": "model-00015-of-00020.safetensors",
193
- "model.layers.15.attention.branch_output_channel_gain": "model-00015-of-00020.safetensors",
194
- "model.layers.15.attention.branch_output_gain": "model-00015-of-00020.safetensors",
195
- "model.layers.15.attention.dense.weight": "model-00015-of-00020.safetensors",
196
- "model.layers.15.attention.key_layernorm.weight": "model-00015-of-00020.safetensors",
197
- "model.layers.15.attention.query_key_value.weight": "model-00015-of-00020.safetensors",
198
- "model.layers.15.attention.query_layernorm.weight": "model-00015-of-00020.safetensors",
199
- "model.layers.15.attention.raven_attention.A_log": "model-00015-of-00020.safetensors",
200
- "model.layers.15.attention.raven_attention.a_proj.weight": "model-00015-of-00020.safetensors",
201
- "model.layers.15.attention.raven_attention.dt_bias": "model-00015-of-00020.safetensors",
202
- "model.layers.15.attention.raven_attention.g_norm.weight": "model-00015-of-00020.safetensors",
203
- "model.layers.15.attention.raven_attention.k_norm.weight": "model-00015-of-00020.safetensors",
204
- "model.layers.15.attention.raven_attention.k_proj.weight": "model-00015-of-00020.safetensors",
205
- "model.layers.15.attention.raven_attention.o_proj.weight": "model-00015-of-00020.safetensors",
206
- "model.layers.15.attention.raven_attention.q_norm.weight": "model-00015-of-00020.safetensors",
207
- "model.layers.15.attention.raven_attention.q_proj.weight": "model-00015-of-00020.safetensors",
208
- "model.layers.15.attention.raven_attention.r_proj.weight": "model-00015-of-00020.safetensors",
209
- "model.layers.15.attention.raven_attention.v_proj.weight": "model-00015-of-00020.safetensors",
210
- "model.layers.15.attention.replace_alpha_raw": "model-00015-of-00020.safetensors",
211
- "model.layers.15.input_layernorm.weight": "model-00016-of-00020.safetensors",
212
  "model.layers.15.mlp.experts_w12": "model-00015-of-00020.safetensors",
213
- "model.layers.15.mlp.experts_w3": "model-00016-of-00020.safetensors",
214
- "model.layers.15.mlp.gate.expert_bias": "model-00016-of-00020.safetensors",
215
- "model.layers.15.mlp.gate.weight": "model-00016-of-00020.safetensors",
216
- "model.layers.15.mlp.shared_experts.down_proj.weight": "model-00016-of-00020.safetensors",
217
- "model.layers.15.mlp.shared_experts.gate_proj.weight": "model-00016-of-00020.safetensors",
218
- "model.layers.15.mlp.shared_experts.up_proj.weight": "model-00016-of-00020.safetensors",
219
- "model.layers.15.post_attention_layernorm.weight": "model-00016-of-00020.safetensors",
220
- "model.layers.16.attention.branch_global_output_gain": "model-00016-of-00020.safetensors",
221
- "model.layers.16.attention.branch_local_window_mix_logit": "model-00016-of-00020.safetensors",
222
- "model.layers.16.attention.branch_mix_logits": "model-00016-of-00020.safetensors",
223
- "model.layers.16.attention.branch_output_adapter_down.weight": "model-00016-of-00020.safetensors",
224
- "model.layers.16.attention.branch_output_adapter_up.weight": "model-00016-of-00020.safetensors",
225
- "model.layers.16.attention.branch_output_channel_gain": "model-00016-of-00020.safetensors",
226
- "model.layers.16.attention.branch_output_gain": "model-00016-of-00020.safetensors",
227
- "model.layers.16.attention.dense.weight": "model-00016-of-00020.safetensors",
228
- "model.layers.16.attention.key_layernorm.weight": "model-00016-of-00020.safetensors",
229
- "model.layers.16.attention.quasar_attention.A_log": "model-00016-of-00020.safetensors",
230
- "model.layers.16.attention.quasar_attention.b_proj.weight": "model-00016-of-00020.safetensors",
231
- "model.layers.16.attention.quasar_attention.dt_bias": "model-00016-of-00020.safetensors",
232
- "model.layers.16.attention.quasar_attention.f_proj.weight": "model-00016-of-00020.safetensors",
233
- "model.layers.16.attention.quasar_attention.g_proj.0.weight": "model-00016-of-00020.safetensors",
234
- "model.layers.16.attention.quasar_attention.g_proj.1.bias": "model-00016-of-00020.safetensors",
235
- "model.layers.16.attention.quasar_attention.g_proj.1.weight": "model-00016-of-00020.safetensors",
236
- "model.layers.16.attention.quasar_attention.k_proj.weight": "model-00016-of-00020.safetensors",
237
- "model.layers.16.attention.quasar_attention.o_norm.weight": "model-00016-of-00020.safetensors",
238
- "model.layers.16.attention.quasar_attention.o_proj.weight": "model-00016-of-00020.safetensors",
239
- "model.layers.16.attention.quasar_attention.q_proj.weight": "model-00016-of-00020.safetensors",
240
- "model.layers.16.attention.quasar_attention.v_proj.weight": "model-00016-of-00020.safetensors",
241
- "model.layers.16.attention.query_key_value.weight": "model-00016-of-00020.safetensors",
242
- "model.layers.16.attention.query_layernorm.weight": "model-00016-of-00020.safetensors",
243
- "model.layers.16.attention.replace_alpha_raw": "model-00016-of-00020.safetensors",
244
- "model.layers.16.input_layernorm.weight": "model-00017-of-00020.safetensors",
245
  "model.layers.16.mlp.experts_w12": "model-00016-of-00020.safetensors",
246
- "model.layers.16.mlp.experts_w3": "model-00017-of-00020.safetensors",
247
- "model.layers.16.mlp.gate.expert_bias": "model-00017-of-00020.safetensors",
248
- "model.layers.16.mlp.gate.weight": "model-00017-of-00020.safetensors",
249
- "model.layers.16.mlp.shared_experts.down_proj.weight": "model-00017-of-00020.safetensors",
250
- "model.layers.16.mlp.shared_experts.gate_proj.weight": "model-00017-of-00020.safetensors",
251
- "model.layers.16.mlp.shared_experts.up_proj.weight": "model-00017-of-00020.safetensors",
252
- "model.layers.16.post_attention_layernorm.weight": "model-00017-of-00020.safetensors",
253
- "model.layers.17.attention.branch_global_output_gain": "model-00017-of-00020.safetensors",
254
- "model.layers.17.attention.branch_local_window_mix_logit": "model-00017-of-00020.safetensors",
255
- "model.layers.17.attention.branch_mix_logits": "model-00017-of-00020.safetensors",
256
- "model.layers.17.attention.branch_output_adapter_down.weight": "model-00017-of-00020.safetensors",
257
- "model.layers.17.attention.branch_output_adapter_up.weight": "model-00017-of-00020.safetensors",
258
- "model.layers.17.attention.branch_output_channel_gain": "model-00017-of-00020.safetensors",
259
- "model.layers.17.attention.branch_output_gain": "model-00017-of-00020.safetensors",
260
- "model.layers.17.attention.dense.weight": "model-00017-of-00020.safetensors",
261
- "model.layers.17.attention.key_layernorm.weight": "model-00017-of-00020.safetensors",
262
- "model.layers.17.attention.quasar_attention.A_log": "model-00017-of-00020.safetensors",
263
- "model.layers.17.attention.quasar_attention.b_proj.weight": "model-00017-of-00020.safetensors",
264
- "model.layers.17.attention.quasar_attention.dt_bias": "model-00017-of-00020.safetensors",
265
- "model.layers.17.attention.quasar_attention.f_proj.weight": "model-00017-of-00020.safetensors",
266
- "model.layers.17.attention.quasar_attention.g_proj.0.weight": "model-00017-of-00020.safetensors",
267
- "model.layers.17.attention.quasar_attention.g_proj.1.bias": "model-00017-of-00020.safetensors",
268
- "model.layers.17.attention.quasar_attention.g_proj.1.weight": "model-00017-of-00020.safetensors",
269
- "model.layers.17.attention.quasar_attention.k_proj.weight": "model-00017-of-00020.safetensors",
270
- "model.layers.17.attention.quasar_attention.o_norm.weight": "model-00017-of-00020.safetensors",
271
- "model.layers.17.attention.quasar_attention.o_proj.weight": "model-00017-of-00020.safetensors",
272
- "model.layers.17.attention.quasar_attention.q_proj.weight": "model-00017-of-00020.safetensors",
273
- "model.layers.17.attention.quasar_attention.v_proj.weight": "model-00017-of-00020.safetensors",
274
- "model.layers.17.attention.query_key_value.weight": "model-00017-of-00020.safetensors",
275
- "model.layers.17.attention.query_layernorm.weight": "model-00017-of-00020.safetensors",
276
- "model.layers.17.attention.replace_alpha_raw": "model-00017-of-00020.safetensors",
277
- "model.layers.17.input_layernorm.weight": "model-00018-of-00020.safetensors",
278
  "model.layers.17.mlp.experts_w12": "model-00017-of-00020.safetensors",
279
- "model.layers.17.mlp.experts_w3": "model-00018-of-00020.safetensors",
280
- "model.layers.17.mlp.gate.expert_bias": "model-00018-of-00020.safetensors",
281
- "model.layers.17.mlp.gate.weight": "model-00018-of-00020.safetensors",
282
- "model.layers.17.mlp.shared_experts.down_proj.weight": "model-00018-of-00020.safetensors",
283
- "model.layers.17.mlp.shared_experts.gate_proj.weight": "model-00018-of-00020.safetensors",
284
- "model.layers.17.mlp.shared_experts.up_proj.weight": "model-00018-of-00020.safetensors",
285
- "model.layers.17.post_attention_layernorm.weight": "model-00018-of-00020.safetensors",
286
- "model.layers.18.attention.branch_global_output_gain": "model-00018-of-00020.safetensors",
287
- "model.layers.18.attention.branch_local_window_mix_logit": "model-00018-of-00020.safetensors",
288
- "model.layers.18.attention.branch_mix_logits": "model-00018-of-00020.safetensors",
289
- "model.layers.18.attention.branch_output_adapter_down.weight": "model-00018-of-00020.safetensors",
290
- "model.layers.18.attention.branch_output_adapter_up.weight": "model-00018-of-00020.safetensors",
291
- "model.layers.18.attention.branch_output_channel_gain": "model-00018-of-00020.safetensors",
292
- "model.layers.18.attention.branch_output_gain": "model-00018-of-00020.safetensors",
293
- "model.layers.18.attention.dense.weight": "model-00018-of-00020.safetensors",
294
- "model.layers.18.attention.gla_attention.dense.weight": "model-00018-of-00020.safetensors",
295
- "model.layers.18.attention.gla_attention.g_norm.weight": "model-00018-of-00020.safetensors",
296
- "model.layers.18.attention.gla_attention.g_proj.weight": "model-00018-of-00020.safetensors",
297
- "model.layers.18.attention.gla_attention.key_layernorm.weight": "model-00018-of-00020.safetensors",
298
- "model.layers.18.attention.gla_attention.query_key_value.weight": "model-00018-of-00020.safetensors",
299
- "model.layers.18.attention.gla_attention.query_layernorm.weight": "model-00018-of-00020.safetensors",
300
- "model.layers.18.attention.gla_attention.slope": "model-00018-of-00020.safetensors",
301
- "model.layers.18.attention.key_layernorm.weight": "model-00018-of-00020.safetensors",
302
- "model.layers.18.attention.query_key_value.weight": "model-00018-of-00020.safetensors",
303
- "model.layers.18.attention.query_layernorm.weight": "model-00018-of-00020.safetensors",
304
- "model.layers.18.attention.replace_alpha_raw": "model-00018-of-00020.safetensors",
305
- "model.layers.18.input_layernorm.weight": "model-00019-of-00020.safetensors",
306
  "model.layers.18.mlp.experts_w12": "model-00018-of-00020.safetensors",
307
- "model.layers.18.mlp.experts_w3": "model-00019-of-00020.safetensors",
308
- "model.layers.18.mlp.gate.expert_bias": "model-00019-of-00020.safetensors",
309
- "model.layers.18.mlp.gate.weight": "model-00019-of-00020.safetensors",
310
- "model.layers.18.mlp.shared_experts.down_proj.weight": "model-00019-of-00020.safetensors",
311
- "model.layers.18.mlp.shared_experts.gate_proj.weight": "model-00019-of-00020.safetensors",
312
- "model.layers.18.mlp.shared_experts.up_proj.weight": "model-00019-of-00020.safetensors",
313
- "model.layers.18.post_attention_layernorm.weight": "model-00019-of-00020.safetensors",
314
- "model.layers.19.attention.branch_global_output_gain": "model-00019-of-00020.safetensors",
315
- "model.layers.19.attention.branch_local_window_mix_logit": "model-00019-of-00020.safetensors",
316
- "model.layers.19.attention.branch_mix_logits": "model-00019-of-00020.safetensors",
317
- "model.layers.19.attention.branch_output_adapter_down.weight": "model-00019-of-00020.safetensors",
318
- "model.layers.19.attention.branch_output_adapter_up.weight": "model-00019-of-00020.safetensors",
319
- "model.layers.19.attention.branch_output_channel_gain": "model-00019-of-00020.safetensors",
320
- "model.layers.19.attention.branch_output_gain": "model-00019-of-00020.safetensors",
321
- "model.layers.19.attention.dense.weight": "model-00019-of-00020.safetensors",
322
- "model.layers.19.attention.key_layernorm.weight": "model-00019-of-00020.safetensors",
323
- "model.layers.19.attention.quasar_attention.A_log": "model-00019-of-00020.safetensors",
324
- "model.layers.19.attention.quasar_attention.b_proj.weight": "model-00019-of-00020.safetensors",
325
- "model.layers.19.attention.quasar_attention.dt_bias": "model-00019-of-00020.safetensors",
326
- "model.layers.19.attention.quasar_attention.f_proj.weight": "model-00019-of-00020.safetensors",
327
- "model.layers.19.attention.quasar_attention.g_proj.0.weight": "model-00019-of-00020.safetensors",
328
- "model.layers.19.attention.quasar_attention.g_proj.1.bias": "model-00019-of-00020.safetensors",
329
- "model.layers.19.attention.quasar_attention.g_proj.1.weight": "model-00019-of-00020.safetensors",
330
- "model.layers.19.attention.quasar_attention.k_proj.weight": "model-00019-of-00020.safetensors",
331
- "model.layers.19.attention.quasar_attention.o_norm.weight": "model-00019-of-00020.safetensors",
332
- "model.layers.19.attention.quasar_attention.o_proj.weight": "model-00019-of-00020.safetensors",
333
- "model.layers.19.attention.quasar_attention.q_proj.weight": "model-00019-of-00020.safetensors",
334
- "model.layers.19.attention.quasar_attention.v_proj.weight": "model-00019-of-00020.safetensors",
335
- "model.layers.19.attention.query_key_value.weight": "model-00019-of-00020.safetensors",
336
- "model.layers.19.attention.query_layernorm.weight": "model-00019-of-00020.safetensors",
337
- "model.layers.19.attention.replace_alpha_raw": "model-00019-of-00020.safetensors",
338
- "model.layers.19.input_layernorm.weight": "model-00020-of-00020.safetensors",
339
  "model.layers.19.mlp.experts_w12": "model-00019-of-00020.safetensors",
340
- "model.layers.19.mlp.experts_w3": "model-00020-of-00020.safetensors",
341
- "model.layers.19.mlp.gate.expert_bias": "model-00020-of-00020.safetensors",
342
- "model.layers.19.mlp.gate.weight": "model-00020-of-00020.safetensors",
343
- "model.layers.19.mlp.shared_experts.down_proj.weight": "model-00020-of-00020.safetensors",
344
- "model.layers.19.mlp.shared_experts.gate_proj.weight": "model-00020-of-00020.safetensors",
345
- "model.layers.19.mlp.shared_experts.up_proj.weight": "model-00020-of-00020.safetensors",
346
- "model.layers.19.post_attention_layernorm.weight": "model-00020-of-00020.safetensors",
347
  "model.layers.2.attention.dense.weight": "model-00002-of-00020.safetensors",
348
  "model.layers.2.attention.key_layernorm.weight": "model-00002-of-00020.safetensors",
349
  "model.layers.2.attention.query_key_value.weight": "model-00002-of-00020.safetensors",
350
  "model.layers.2.attention.query_layernorm.weight": "model-00002-of-00020.safetensors",
351
- "model.layers.2.input_layernorm.weight": "model-00003-of-00020.safetensors",
352
  "model.layers.2.mlp.experts_w12": "model-00002-of-00020.safetensors",
353
- "model.layers.2.mlp.experts_w3": "model-00003-of-00020.safetensors",
354
- "model.layers.2.mlp.gate.expert_bias": "model-00003-of-00020.safetensors",
355
- "model.layers.2.mlp.gate.weight": "model-00003-of-00020.safetensors",
356
- "model.layers.2.mlp.shared_experts.down_proj.weight": "model-00003-of-00020.safetensors",
357
- "model.layers.2.mlp.shared_experts.gate_proj.weight": "model-00003-of-00020.safetensors",
358
- "model.layers.2.mlp.shared_experts.up_proj.weight": "model-00003-of-00020.safetensors",
359
- "model.layers.2.post_attention_layernorm.weight": "model-00003-of-00020.safetensors",
360
- "model.layers.3.attention.dense.weight": "model-00003-of-00020.safetensors",
361
- "model.layers.3.attention.key_layernorm.weight": "model-00003-of-00020.safetensors",
362
- "model.layers.3.attention.query_key_value.weight": "model-00003-of-00020.safetensors",
363
- "model.layers.3.attention.query_layernorm.weight": "model-00003-of-00020.safetensors",
364
- "model.layers.3.input_layernorm.weight": "model-00004-of-00020.safetensors",
365
  "model.layers.3.mlp.experts_w12": "model-00003-of-00020.safetensors",
366
- "model.layers.3.mlp.experts_w3": "model-00004-of-00020.safetensors",
367
- "model.layers.3.mlp.gate.expert_bias": "model-00004-of-00020.safetensors",
368
- "model.layers.3.mlp.gate.weight": "model-00004-of-00020.safetensors",
369
- "model.layers.3.mlp.shared_experts.down_proj.weight": "model-00004-of-00020.safetensors",
370
- "model.layers.3.mlp.shared_experts.gate_proj.weight": "model-00004-of-00020.safetensors",
371
- "model.layers.3.mlp.shared_experts.up_proj.weight": "model-00004-of-00020.safetensors",
372
- "model.layers.3.post_attention_layernorm.weight": "model-00004-of-00020.safetensors",
373
- "model.layers.4.attention.branch_global_output_gain": "model-00004-of-00020.safetensors",
374
- "model.layers.4.attention.branch_local_window_mix_logit": "model-00004-of-00020.safetensors",
375
- "model.layers.4.attention.branch_mix_logits": "model-00004-of-00020.safetensors",
376
- "model.layers.4.attention.branch_output_adapter_down.weight": "model-00004-of-00020.safetensors",
377
- "model.layers.4.attention.branch_output_adapter_up.weight": "model-00004-of-00020.safetensors",
378
- "model.layers.4.attention.branch_output_channel_gain": "model-00004-of-00020.safetensors",
379
- "model.layers.4.attention.branch_output_gain": "model-00004-of-00020.safetensors",
380
- "model.layers.4.attention.dense.weight": "model-00004-of-00020.safetensors",
381
- "model.layers.4.attention.key_layernorm.weight": "model-00004-of-00020.safetensors",
382
- "model.layers.4.attention.quasar_attention.A_log": "model-00004-of-00020.safetensors",
383
- "model.layers.4.attention.quasar_attention.b_proj.weight": "model-00004-of-00020.safetensors",
384
- "model.layers.4.attention.quasar_attention.dt_bias": "model-00004-of-00020.safetensors",
385
- "model.layers.4.attention.quasar_attention.f_proj.weight": "model-00004-of-00020.safetensors",
386
- "model.layers.4.attention.quasar_attention.g_proj.0.weight": "model-00004-of-00020.safetensors",
387
- "model.layers.4.attention.quasar_attention.g_proj.1.bias": "model-00004-of-00020.safetensors",
388
- "model.layers.4.attention.quasar_attention.g_proj.1.weight": "model-00004-of-00020.safetensors",
389
- "model.layers.4.attention.quasar_attention.k_proj.weight": "model-00004-of-00020.safetensors",
390
- "model.layers.4.attention.quasar_attention.o_norm.weight": "model-00004-of-00020.safetensors",
391
- "model.layers.4.attention.quasar_attention.o_proj.weight": "model-00004-of-00020.safetensors",
392
- "model.layers.4.attention.quasar_attention.q_proj.weight": "model-00004-of-00020.safetensors",
393
- "model.layers.4.attention.quasar_attention.v_proj.weight": "model-00004-of-00020.safetensors",
394
- "model.layers.4.attention.query_key_value.weight": "model-00004-of-00020.safetensors",
395
- "model.layers.4.attention.query_layernorm.weight": "model-00004-of-00020.safetensors",
396
- "model.layers.4.attention.replace_alpha_raw": "model-00004-of-00020.safetensors",
397
- "model.layers.4.input_layernorm.weight": "model-00005-of-00020.safetensors",
398
  "model.layers.4.mlp.experts_w12": "model-00004-of-00020.safetensors",
399
- "model.layers.4.mlp.experts_w3": "model-00005-of-00020.safetensors",
400
- "model.layers.4.mlp.gate.expert_bias": "model-00005-of-00020.safetensors",
401
- "model.layers.4.mlp.gate.weight": "model-00005-of-00020.safetensors",
402
- "model.layers.4.mlp.shared_experts.down_proj.weight": "model-00005-of-00020.safetensors",
403
- "model.layers.4.mlp.shared_experts.gate_proj.weight": "model-00005-of-00020.safetensors",
404
- "model.layers.4.mlp.shared_experts.up_proj.weight": "model-00005-of-00020.safetensors",
405
- "model.layers.4.post_attention_layernorm.weight": "model-00005-of-00020.safetensors",
406
- "model.layers.5.attention.branch_global_output_gain": "model-00005-of-00020.safetensors",
407
- "model.layers.5.attention.branch_local_window_mix_logit": "model-00005-of-00020.safetensors",
408
- "model.layers.5.attention.branch_mix_logits": "model-00005-of-00020.safetensors",
409
- "model.layers.5.attention.branch_output_adapter_down.weight": "model-00005-of-00020.safetensors",
410
- "model.layers.5.attention.branch_output_adapter_up.weight": "model-00005-of-00020.safetensors",
411
- "model.layers.5.attention.branch_output_channel_gain": "model-00005-of-00020.safetensors",
412
- "model.layers.5.attention.branch_output_gain": "model-00005-of-00020.safetensors",
413
- "model.layers.5.attention.dense.weight": "model-00005-of-00020.safetensors",
414
- "model.layers.5.attention.key_layernorm.weight": "model-00005-of-00020.safetensors",
415
- "model.layers.5.attention.query_key_value.weight": "model-00005-of-00020.safetensors",
416
- "model.layers.5.attention.query_layernorm.weight": "model-00005-of-00020.safetensors",
417
- "model.layers.5.attention.raven_attention.A_log": "model-00005-of-00020.safetensors",
418
- "model.layers.5.attention.raven_attention.a_proj.weight": "model-00005-of-00020.safetensors",
419
- "model.layers.5.attention.raven_attention.dt_bias": "model-00005-of-00020.safetensors",
420
- "model.layers.5.attention.raven_attention.g_norm.weight": "model-00005-of-00020.safetensors",
421
- "model.layers.5.attention.raven_attention.k_norm.weight": "model-00005-of-00020.safetensors",
422
- "model.layers.5.attention.raven_attention.k_proj.weight": "model-00005-of-00020.safetensors",
423
- "model.layers.5.attention.raven_attention.o_proj.weight": "model-00005-of-00020.safetensors",
424
- "model.layers.5.attention.raven_attention.q_norm.weight": "model-00005-of-00020.safetensors",
425
- "model.layers.5.attention.raven_attention.q_proj.weight": "model-00005-of-00020.safetensors",
426
- "model.layers.5.attention.raven_attention.r_proj.weight": "model-00005-of-00020.safetensors",
427
- "model.layers.5.attention.raven_attention.v_proj.weight": "model-00005-of-00020.safetensors",
428
- "model.layers.5.attention.replace_alpha_raw": "model-00005-of-00020.safetensors",
429
- "model.layers.5.input_layernorm.weight": "model-00006-of-00020.safetensors",
430
  "model.layers.5.mlp.experts_w12": "model-00005-of-00020.safetensors",
431
- "model.layers.5.mlp.experts_w3": "model-00006-of-00020.safetensors",
432
- "model.layers.5.mlp.gate.expert_bias": "model-00006-of-00020.safetensors",
433
- "model.layers.5.mlp.gate.weight": "model-00006-of-00020.safetensors",
434
- "model.layers.5.mlp.shared_experts.down_proj.weight": "model-00006-of-00020.safetensors",
435
- "model.layers.5.mlp.shared_experts.gate_proj.weight": "model-00006-of-00020.safetensors",
436
- "model.layers.5.mlp.shared_experts.up_proj.weight": "model-00006-of-00020.safetensors",
437
- "model.layers.5.post_attention_layernorm.weight": "model-00006-of-00020.safetensors",
438
- "model.layers.6.attention.branch_global_output_gain": "model-00006-of-00020.safetensors",
439
- "model.layers.6.attention.branch_local_window_mix_logit": "model-00006-of-00020.safetensors",
440
- "model.layers.6.attention.branch_mix_logits": "model-00006-of-00020.safetensors",
441
- "model.layers.6.attention.branch_output_adapter_down.weight": "model-00006-of-00020.safetensors",
442
- "model.layers.6.attention.branch_output_adapter_up.weight": "model-00006-of-00020.safetensors",
443
- "model.layers.6.attention.branch_output_channel_gain": "model-00006-of-00020.safetensors",
444
- "model.layers.6.attention.branch_output_gain": "model-00006-of-00020.safetensors",
445
- "model.layers.6.attention.dense.weight": "model-00006-of-00020.safetensors",
446
- "model.layers.6.attention.key_layernorm.weight": "model-00006-of-00020.safetensors",
447
- "model.layers.6.attention.quasar_attention.A_log": "model-00006-of-00020.safetensors",
448
- "model.layers.6.attention.quasar_attention.b_proj.weight": "model-00006-of-00020.safetensors",
449
- "model.layers.6.attention.quasar_attention.dt_bias": "model-00006-of-00020.safetensors",
450
- "model.layers.6.attention.quasar_attention.f_proj.weight": "model-00006-of-00020.safetensors",
451
- "model.layers.6.attention.quasar_attention.g_proj.0.weight": "model-00006-of-00020.safetensors",
452
- "model.layers.6.attention.quasar_attention.g_proj.1.bias": "model-00006-of-00020.safetensors",
453
- "model.layers.6.attention.quasar_attention.g_proj.1.weight": "model-00006-of-00020.safetensors",
454
- "model.layers.6.attention.quasar_attention.k_proj.weight": "model-00006-of-00020.safetensors",
455
- "model.layers.6.attention.quasar_attention.o_norm.weight": "model-00006-of-00020.safetensors",
456
- "model.layers.6.attention.quasar_attention.o_proj.weight": "model-00006-of-00020.safetensors",
457
- "model.layers.6.attention.quasar_attention.q_proj.weight": "model-00006-of-00020.safetensors",
458
- "model.layers.6.attention.quasar_attention.v_proj.weight": "model-00006-of-00020.safetensors",
459
- "model.layers.6.attention.query_key_value.weight": "model-00006-of-00020.safetensors",
460
- "model.layers.6.attention.query_layernorm.weight": "model-00006-of-00020.safetensors",
461
- "model.layers.6.attention.replace_alpha_raw": "model-00006-of-00020.safetensors",
462
- "model.layers.6.input_layernorm.weight": "model-00007-of-00020.safetensors",
463
  "model.layers.6.mlp.experts_w12": "model-00006-of-00020.safetensors",
464
- "model.layers.6.mlp.experts_w3": "model-00007-of-00020.safetensors",
465
- "model.layers.6.mlp.gate.expert_bias": "model-00007-of-00020.safetensors",
466
- "model.layers.6.mlp.gate.weight": "model-00007-of-00020.safetensors",
467
- "model.layers.6.mlp.shared_experts.down_proj.weight": "model-00007-of-00020.safetensors",
468
- "model.layers.6.mlp.shared_experts.gate_proj.weight": "model-00007-of-00020.safetensors",
469
- "model.layers.6.mlp.shared_experts.up_proj.weight": "model-00007-of-00020.safetensors",
470
- "model.layers.6.post_attention_layernorm.weight": "model-00007-of-00020.safetensors",
471
- "model.layers.7.attention.branch_global_output_gain": "model-00007-of-00020.safetensors",
472
- "model.layers.7.attention.branch_local_window_mix_logit": "model-00007-of-00020.safetensors",
473
- "model.layers.7.attention.branch_mix_logits": "model-00007-of-00020.safetensors",
474
- "model.layers.7.attention.branch_output_adapter_down.weight": "model-00007-of-00020.safetensors",
475
- "model.layers.7.attention.branch_output_adapter_up.weight": "model-00007-of-00020.safetensors",
476
- "model.layers.7.attention.branch_output_channel_gain": "model-00007-of-00020.safetensors",
477
- "model.layers.7.attention.branch_output_gain": "model-00007-of-00020.safetensors",
478
- "model.layers.7.attention.dense.weight": "model-00007-of-00020.safetensors",
479
- "model.layers.7.attention.key_layernorm.weight": "model-00007-of-00020.safetensors",
480
- "model.layers.7.attention.quasar_attention.A_log": "model-00007-of-00020.safetensors",
481
- "model.layers.7.attention.quasar_attention.b_proj.weight": "model-00007-of-00020.safetensors",
482
- "model.layers.7.attention.quasar_attention.dt_bias": "model-00007-of-00020.safetensors",
483
- "model.layers.7.attention.quasar_attention.f_proj.weight": "model-00007-of-00020.safetensors",
484
- "model.layers.7.attention.quasar_attention.g_proj.0.weight": "model-00007-of-00020.safetensors",
485
- "model.layers.7.attention.quasar_attention.g_proj.1.bias": "model-00007-of-00020.safetensors",
486
- "model.layers.7.attention.quasar_attention.g_proj.1.weight": "model-00007-of-00020.safetensors",
487
- "model.layers.7.attention.quasar_attention.k_proj.weight": "model-00007-of-00020.safetensors",
488
- "model.layers.7.attention.quasar_attention.o_norm.weight": "model-00007-of-00020.safetensors",
489
- "model.layers.7.attention.quasar_attention.o_proj.weight": "model-00007-of-00020.safetensors",
490
- "model.layers.7.attention.quasar_attention.q_proj.weight": "model-00007-of-00020.safetensors",
491
- "model.layers.7.attention.quasar_attention.v_proj.weight": "model-00007-of-00020.safetensors",
492
- "model.layers.7.attention.query_key_value.weight": "model-00007-of-00020.safetensors",
493
- "model.layers.7.attention.query_layernorm.weight": "model-00007-of-00020.safetensors",
494
- "model.layers.7.attention.replace_alpha_raw": "model-00007-of-00020.safetensors",
495
- "model.layers.7.input_layernorm.weight": "model-00008-of-00020.safetensors",
496
  "model.layers.7.mlp.experts_w12": "model-00007-of-00020.safetensors",
497
- "model.layers.7.mlp.experts_w3": "model-00008-of-00020.safetensors",
498
- "model.layers.7.mlp.gate.expert_bias": "model-00008-of-00020.safetensors",
499
- "model.layers.7.mlp.gate.weight": "model-00008-of-00020.safetensors",
500
- "model.layers.7.mlp.shared_experts.down_proj.weight": "model-00008-of-00020.safetensors",
501
- "model.layers.7.mlp.shared_experts.gate_proj.weight": "model-00008-of-00020.safetensors",
502
- "model.layers.7.mlp.shared_experts.up_proj.weight": "model-00008-of-00020.safetensors",
503
- "model.layers.7.post_attention_layernorm.weight": "model-00008-of-00020.safetensors",
504
- "model.layers.8.attention.branch_global_output_gain": "model-00008-of-00020.safetensors",
505
- "model.layers.8.attention.branch_local_window_mix_logit": "model-00008-of-00020.safetensors",
506
- "model.layers.8.attention.branch_mix_logits": "model-00008-of-00020.safetensors",
507
- "model.layers.8.attention.branch_output_adapter_down.weight": "model-00008-of-00020.safetensors",
508
- "model.layers.8.attention.branch_output_adapter_up.weight": "model-00008-of-00020.safetensors",
509
- "model.layers.8.attention.branch_output_channel_gain": "model-00008-of-00020.safetensors",
510
- "model.layers.8.attention.branch_output_gain": "model-00008-of-00020.safetensors",
511
- "model.layers.8.attention.dense.weight": "model-00008-of-00020.safetensors",
512
- "model.layers.8.attention.gla_attention.dense.weight": "model-00008-of-00020.safetensors",
513
- "model.layers.8.attention.gla_attention.g_norm.weight": "model-00008-of-00020.safetensors",
514
- "model.layers.8.attention.gla_attention.g_proj.weight": "model-00008-of-00020.safetensors",
515
- "model.layers.8.attention.gla_attention.key_layernorm.weight": "model-00008-of-00020.safetensors",
516
- "model.layers.8.attention.gla_attention.query_key_value.weight": "model-00008-of-00020.safetensors",
517
- "model.layers.8.attention.gla_attention.query_layernorm.weight": "model-00008-of-00020.safetensors",
518
- "model.layers.8.attention.gla_attention.slope": "model-00008-of-00020.safetensors",
519
- "model.layers.8.attention.key_layernorm.weight": "model-00008-of-00020.safetensors",
520
- "model.layers.8.attention.query_key_value.weight": "model-00008-of-00020.safetensors",
521
- "model.layers.8.attention.query_layernorm.weight": "model-00008-of-00020.safetensors",
522
- "model.layers.8.attention.replace_alpha_raw": "model-00008-of-00020.safetensors",
523
- "model.layers.8.input_layernorm.weight": "model-00009-of-00020.safetensors",
524
  "model.layers.8.mlp.experts_w12": "model-00008-of-00020.safetensors",
525
- "model.layers.8.mlp.experts_w3": "model-00009-of-00020.safetensors",
526
- "model.layers.8.mlp.gate.expert_bias": "model-00009-of-00020.safetensors",
527
- "model.layers.8.mlp.gate.weight": "model-00009-of-00020.safetensors",
528
- "model.layers.8.mlp.shared_experts.down_proj.weight": "model-00009-of-00020.safetensors",
529
- "model.layers.8.mlp.shared_experts.gate_proj.weight": "model-00009-of-00020.safetensors",
530
- "model.layers.8.mlp.shared_experts.up_proj.weight": "model-00009-of-00020.safetensors",
531
- "model.layers.8.post_attention_layernorm.weight": "model-00009-of-00020.safetensors",
532
- "model.layers.9.attention.branch_global_output_gain": "model-00009-of-00020.safetensors",
533
- "model.layers.9.attention.branch_local_window_mix_logit": "model-00009-of-00020.safetensors",
534
- "model.layers.9.attention.branch_mix_logits": "model-00009-of-00020.safetensors",
535
- "model.layers.9.attention.branch_output_adapter_down.weight": "model-00009-of-00020.safetensors",
536
- "model.layers.9.attention.branch_output_adapter_up.weight": "model-00009-of-00020.safetensors",
537
- "model.layers.9.attention.branch_output_channel_gain": "model-00009-of-00020.safetensors",
538
- "model.layers.9.attention.branch_output_gain": "model-00009-of-00020.safetensors",
539
- "model.layers.9.attention.dense.weight": "model-00009-of-00020.safetensors",
540
- "model.layers.9.attention.key_layernorm.weight": "model-00009-of-00020.safetensors",
541
- "model.layers.9.attention.quasar_attention.A_log": "model-00009-of-00020.safetensors",
542
- "model.layers.9.attention.quasar_attention.b_proj.weight": "model-00009-of-00020.safetensors",
543
- "model.layers.9.attention.quasar_attention.dt_bias": "model-00009-of-00020.safetensors",
544
- "model.layers.9.attention.quasar_attention.f_proj.weight": "model-00009-of-00020.safetensors",
545
- "model.layers.9.attention.quasar_attention.g_proj.0.weight": "model-00009-of-00020.safetensors",
546
- "model.layers.9.attention.quasar_attention.g_proj.1.bias": "model-00009-of-00020.safetensors",
547
- "model.layers.9.attention.quasar_attention.g_proj.1.weight": "model-00009-of-00020.safetensors",
548
- "model.layers.9.attention.quasar_attention.k_proj.weight": "model-00009-of-00020.safetensors",
549
- "model.layers.9.attention.quasar_attention.o_norm.weight": "model-00009-of-00020.safetensors",
550
- "model.layers.9.attention.quasar_attention.o_proj.weight": "model-00009-of-00020.safetensors",
551
- "model.layers.9.attention.quasar_attention.q_proj.weight": "model-00009-of-00020.safetensors",
552
- "model.layers.9.attention.quasar_attention.v_proj.weight": "model-00009-of-00020.safetensors",
553
- "model.layers.9.attention.query_key_value.weight": "model-00009-of-00020.safetensors",
554
- "model.layers.9.attention.query_layernorm.weight": "model-00009-of-00020.safetensors",
555
- "model.layers.9.attention.replace_alpha_raw": "model-00009-of-00020.safetensors",
556
- "model.layers.9.input_layernorm.weight": "model-00010-of-00020.safetensors",
557
  "model.layers.9.mlp.experts_w12": "model-00009-of-00020.safetensors",
558
- "model.layers.9.mlp.experts_w3": "model-00010-of-00020.safetensors",
559
- "model.layers.9.mlp.gate.expert_bias": "model-00010-of-00020.safetensors",
560
- "model.layers.9.mlp.gate.weight": "model-00010-of-00020.safetensors",
561
- "model.layers.9.mlp.shared_experts.down_proj.weight": "model-00010-of-00020.safetensors",
562
- "model.layers.9.mlp.shared_experts.gate_proj.weight": "model-00010-of-00020.safetensors",
563
- "model.layers.9.mlp.shared_experts.up_proj.weight": "model-00010-of-00020.safetensors",
564
- "model.layers.9.post_attention_layernorm.weight": "model-00010-of-00020.safetensors",
565
- "model.norm.weight": "model-00020-of-00020.safetensors",
566
- "model.rotary_emb.inv_freq": "model-00020-of-00020.safetensors",
567
  "model.word_embeddings.weight": "model-00001-of-00020.safetensors"
568
  }
569
  }
 
1
  {
2
  "metadata": {
3
+ "total_size": 33107908480
4
  },
5
  "weight_map": {
6
  "lm_head.weight": "model-00020-of-00020.safetensors",
 
26
  "model.layers.1.mlp.shared_experts.gate_proj.weight": "model-00002-of-00020.safetensors",
27
  "model.layers.1.mlp.shared_experts.up_proj.weight": "model-00002-of-00020.safetensors",
28
  "model.layers.1.post_attention_layernorm.weight": "model-00002-of-00020.safetensors",
29
+ "model.layers.10.attention.branch_global_output_gain": "model-00009-of-00020.safetensors",
30
+ "model.layers.10.attention.branch_local_window_mix_logit": "model-00009-of-00020.safetensors",
31
+ "model.layers.10.attention.branch_mix_logits": "model-00009-of-00020.safetensors",
32
+ "model.layers.10.attention.branch_output_adapter_down.weight": "model-00009-of-00020.safetensors",
33
+ "model.layers.10.attention.branch_output_adapter_up.weight": "model-00009-of-00020.safetensors",
34
+ "model.layers.10.attention.branch_output_channel_gain": "model-00009-of-00020.safetensors",
35
+ "model.layers.10.attention.branch_output_gain": "model-00009-of-00020.safetensors",
36
+ "model.layers.10.attention.dense.weight": "model-00009-of-00020.safetensors",
37
+ "model.layers.10.attention.key_layernorm.weight": "model-00009-of-00020.safetensors",
38
+ "model.layers.10.attention.query_key_value.weight": "model-00009-of-00020.safetensors",
39
+ "model.layers.10.attention.query_layernorm.weight": "model-00009-of-00020.safetensors",
40
+ "model.layers.10.attention.raven_attention.A_log": "model-00009-of-00020.safetensors",
41
+ "model.layers.10.attention.raven_attention.a_proj.weight": "model-00009-of-00020.safetensors",
42
+ "model.layers.10.attention.raven_attention.dt_bias": "model-00009-of-00020.safetensors",
43
+ "model.layers.10.attention.raven_attention.g_norm.weight": "model-00009-of-00020.safetensors",
44
+ "model.layers.10.attention.raven_attention.k_norm.weight": "model-00009-of-00020.safetensors",
45
+ "model.layers.10.attention.raven_attention.k_proj.weight": "model-00009-of-00020.safetensors",
46
+ "model.layers.10.attention.raven_attention.o_proj.weight": "model-00009-of-00020.safetensors",
47
+ "model.layers.10.attention.raven_attention.q_norm.weight": "model-00009-of-00020.safetensors",
48
+ "model.layers.10.attention.raven_attention.q_proj.weight": "model-00009-of-00020.safetensors",
49
+ "model.layers.10.attention.raven_attention.r_proj.weight": "model-00009-of-00020.safetensors",
50
+ "model.layers.10.attention.raven_attention.v_proj.weight": "model-00009-of-00020.safetensors",
51
+ "model.layers.10.attention.replace_alpha_raw": "model-00009-of-00020.safetensors",
52
+ "model.layers.10.input_layernorm.weight": "model-00010-of-00020.safetensors",
53
  "model.layers.10.mlp.experts_w12": "model-00010-of-00020.safetensors",
54
+ "model.layers.10.mlp.experts_w3": "model-00010-of-00020.safetensors",
55
+ "model.layers.10.mlp.gate.expert_bias": "model-00010-of-00020.safetensors",
56
+ "model.layers.10.mlp.gate.weight": "model-00010-of-00020.safetensors",
57
+ "model.layers.10.mlp.shared_experts.down_proj.weight": "model-00010-of-00020.safetensors",
58
+ "model.layers.10.mlp.shared_experts.gate_proj.weight": "model-00010-of-00020.safetensors",
59
+ "model.layers.10.mlp.shared_experts.up_proj.weight": "model-00010-of-00020.safetensors",
60
+ "model.layers.10.post_attention_layernorm.weight": "model-00010-of-00020.safetensors",
61
+ "model.layers.11.attention.branch_global_output_gain": "model-00010-of-00020.safetensors",
62
+ "model.layers.11.attention.branch_local_window_mix_logit": "model-00010-of-00020.safetensors",
63
+ "model.layers.11.attention.branch_mix_logits": "model-00010-of-00020.safetensors",
64
+ "model.layers.11.attention.branch_output_adapter_down.weight": "model-00010-of-00020.safetensors",
65
+ "model.layers.11.attention.branch_output_adapter_up.weight": "model-00010-of-00020.safetensors",
66
+ "model.layers.11.attention.branch_output_channel_gain": "model-00010-of-00020.safetensors",
67
+ "model.layers.11.attention.branch_output_gain": "model-00010-of-00020.safetensors",
68
+ "model.layers.11.attention.dense.weight": "model-00010-of-00020.safetensors",
69
+ "model.layers.11.attention.key_layernorm.weight": "model-00010-of-00020.safetensors",
70
+ "model.layers.11.attention.quasar_attention.A_log": "model-00010-of-00020.safetensors",
71
+ "model.layers.11.attention.quasar_attention.b_proj.weight": "model-00010-of-00020.safetensors",
72
+ "model.layers.11.attention.quasar_attention.dt_bias": "model-00010-of-00020.safetensors",
73
+ "model.layers.11.attention.quasar_attention.f_proj.weight": "model-00010-of-00020.safetensors",
74
+ "model.layers.11.attention.quasar_attention.g_proj.0.weight": "model-00010-of-00020.safetensors",
75
+ "model.layers.11.attention.quasar_attention.g_proj.1.bias": "model-00010-of-00020.safetensors",
76
+ "model.layers.11.attention.quasar_attention.g_proj.1.weight": "model-00010-of-00020.safetensors",
77
+ "model.layers.11.attention.quasar_attention.k_proj.weight": "model-00010-of-00020.safetensors",
78
+ "model.layers.11.attention.quasar_attention.o_norm.weight": "model-00010-of-00020.safetensors",
79
+ "model.layers.11.attention.quasar_attention.o_proj.weight": "model-00010-of-00020.safetensors",
80
+ "model.layers.11.attention.quasar_attention.q_proj.weight": "model-00010-of-00020.safetensors",
81
+ "model.layers.11.attention.quasar_attention.v_proj.weight": "model-00010-of-00020.safetensors",
82
+ "model.layers.11.attention.query_key_value.weight": "model-00010-of-00020.safetensors",
83
+ "model.layers.11.attention.query_layernorm.weight": "model-00010-of-00020.safetensors",
84
+ "model.layers.11.attention.replace_alpha_raw": "model-00010-of-00020.safetensors",
85
+ "model.layers.11.input_layernorm.weight": "model-00011-of-00020.safetensors",
86
  "model.layers.11.mlp.experts_w12": "model-00011-of-00020.safetensors",
87
+ "model.layers.11.mlp.experts_w3": "model-00011-of-00020.safetensors",
88
+ "model.layers.11.mlp.gate.expert_bias": "model-00011-of-00020.safetensors",
89
+ "model.layers.11.mlp.gate.weight": "model-00011-of-00020.safetensors",
90
+ "model.layers.11.mlp.shared_experts.down_proj.weight": "model-00011-of-00020.safetensors",
91
+ "model.layers.11.mlp.shared_experts.gate_proj.weight": "model-00011-of-00020.safetensors",
92
+ "model.layers.11.mlp.shared_experts.up_proj.weight": "model-00011-of-00020.safetensors",
93
+ "model.layers.11.post_attention_layernorm.weight": "model-00011-of-00020.safetensors",
94
+ "model.layers.12.attention.branch_global_output_gain": "model-00011-of-00020.safetensors",
95
+ "model.layers.12.attention.branch_local_window_mix_logit": "model-00011-of-00020.safetensors",
96
+ "model.layers.12.attention.branch_mix_logits": "model-00011-of-00020.safetensors",
97
+ "model.layers.12.attention.branch_output_adapter_down.weight": "model-00011-of-00020.safetensors",
98
+ "model.layers.12.attention.branch_output_adapter_up.weight": "model-00011-of-00020.safetensors",
99
+ "model.layers.12.attention.branch_output_channel_gain": "model-00011-of-00020.safetensors",
100
+ "model.layers.12.attention.branch_output_gain": "model-00011-of-00020.safetensors",
101
+ "model.layers.12.attention.dense.weight": "model-00011-of-00020.safetensors",
102
+ "model.layers.12.attention.key_layernorm.weight": "model-00011-of-00020.safetensors",
103
+ "model.layers.12.attention.quasar_attention.A_log": "model-00011-of-00020.safetensors",
104
+ "model.layers.12.attention.quasar_attention.b_proj.weight": "model-00011-of-00020.safetensors",
105
+ "model.layers.12.attention.quasar_attention.dt_bias": "model-00011-of-00020.safetensors",
106
+ "model.layers.12.attention.quasar_attention.f_proj.weight": "model-00011-of-00020.safetensors",
107
+ "model.layers.12.attention.quasar_attention.g_proj.0.weight": "model-00011-of-00020.safetensors",
108
+ "model.layers.12.attention.quasar_attention.g_proj.1.bias": "model-00011-of-00020.safetensors",
109
+ "model.layers.12.attention.quasar_attention.g_proj.1.weight": "model-00011-of-00020.safetensors",
110
+ "model.layers.12.attention.quasar_attention.k_proj.weight": "model-00011-of-00020.safetensors",
111
+ "model.layers.12.attention.quasar_attention.o_norm.weight": "model-00011-of-00020.safetensors",
112
+ "model.layers.12.attention.quasar_attention.o_proj.weight": "model-00011-of-00020.safetensors",
113
+ "model.layers.12.attention.quasar_attention.q_proj.weight": "model-00011-of-00020.safetensors",
114
+ "model.layers.12.attention.quasar_attention.v_proj.weight": "model-00011-of-00020.safetensors",
115
+ "model.layers.12.attention.query_key_value.weight": "model-00011-of-00020.safetensors",
116
+ "model.layers.12.attention.query_layernorm.weight": "model-00011-of-00020.safetensors",
117
+ "model.layers.12.attention.replace_alpha_raw": "model-00011-of-00020.safetensors",
118
+ "model.layers.12.input_layernorm.weight": "model-00012-of-00020.safetensors",
119
  "model.layers.12.mlp.experts_w12": "model-00012-of-00020.safetensors",
120
+ "model.layers.12.mlp.experts_w3": "model-00012-of-00020.safetensors",
121
+ "model.layers.12.mlp.gate.expert_bias": "model-00012-of-00020.safetensors",
122
+ "model.layers.12.mlp.gate.weight": "model-00012-of-00020.safetensors",
123
+ "model.layers.12.mlp.shared_experts.down_proj.weight": "model-00012-of-00020.safetensors",
124
+ "model.layers.12.mlp.shared_experts.gate_proj.weight": "model-00012-of-00020.safetensors",
125
+ "model.layers.12.mlp.shared_experts.up_proj.weight": "model-00012-of-00020.safetensors",
126
+ "model.layers.12.post_attention_layernorm.weight": "model-00012-of-00020.safetensors",
127
+ "model.layers.13.attention.branch_global_output_gain": "model-00012-of-00020.safetensors",
128
+ "model.layers.13.attention.branch_local_window_mix_logit": "model-00012-of-00020.safetensors",
129
+ "model.layers.13.attention.branch_mix_logits": "model-00012-of-00020.safetensors",
130
+ "model.layers.13.attention.branch_output_adapter_down.weight": "model-00012-of-00020.safetensors",
131
+ "model.layers.13.attention.branch_output_adapter_up.weight": "model-00012-of-00020.safetensors",
132
+ "model.layers.13.attention.branch_output_channel_gain": "model-00012-of-00020.safetensors",
133
+ "model.layers.13.attention.branch_output_gain": "model-00012-of-00020.safetensors",
134
+ "model.layers.13.attention.dense.weight": "model-00012-of-00020.safetensors",
135
+ "model.layers.13.attention.gla_attention.dense.weight": "model-00012-of-00020.safetensors",
136
+ "model.layers.13.attention.gla_attention.g_norm.weight": "model-00012-of-00020.safetensors",
137
+ "model.layers.13.attention.gla_attention.g_proj.weight": "model-00012-of-00020.safetensors",
138
+ "model.layers.13.attention.gla_attention.key_layernorm.weight": "model-00012-of-00020.safetensors",
139
+ "model.layers.13.attention.gla_attention.query_key_value.weight": "model-00012-of-00020.safetensors",
140
+ "model.layers.13.attention.gla_attention.query_layernorm.weight": "model-00012-of-00020.safetensors",
141
+ "model.layers.13.attention.gla_attention.slope": "model-00012-of-00020.safetensors",
142
+ "model.layers.13.attention.key_layernorm.weight": "model-00012-of-00020.safetensors",
143
+ "model.layers.13.attention.query_key_value.weight": "model-00012-of-00020.safetensors",
144
+ "model.layers.13.attention.query_layernorm.weight": "model-00012-of-00020.safetensors",
145
+ "model.layers.13.attention.replace_alpha_raw": "model-00012-of-00020.safetensors",
146
+ "model.layers.13.input_layernorm.weight": "model-00013-of-00020.safetensors",
147
  "model.layers.13.mlp.experts_w12": "model-00013-of-00020.safetensors",
148
+ "model.layers.13.mlp.experts_w3": "model-00013-of-00020.safetensors",
149
+ "model.layers.13.mlp.gate.expert_bias": "model-00013-of-00020.safetensors",
150
+ "model.layers.13.mlp.gate.weight": "model-00013-of-00020.safetensors",
151
+ "model.layers.13.mlp.shared_experts.down_proj.weight": "model-00013-of-00020.safetensors",
152
+ "model.layers.13.mlp.shared_experts.gate_proj.weight": "model-00013-of-00020.safetensors",
153
+ "model.layers.13.mlp.shared_experts.up_proj.weight": "model-00013-of-00020.safetensors",
154
+ "model.layers.13.post_attention_layernorm.weight": "model-00013-of-00020.safetensors",
155
+ "model.layers.14.attention.branch_global_output_gain": "model-00013-of-00020.safetensors",
156
+ "model.layers.14.attention.branch_local_window_mix_logit": "model-00013-of-00020.safetensors",
157
+ "model.layers.14.attention.branch_mix_logits": "model-00013-of-00020.safetensors",
158
+ "model.layers.14.attention.branch_output_adapter_down.weight": "model-00013-of-00020.safetensors",
159
+ "model.layers.14.attention.branch_output_adapter_up.weight": "model-00013-of-00020.safetensors",
160
+ "model.layers.14.attention.branch_output_channel_gain": "model-00013-of-00020.safetensors",
161
+ "model.layers.14.attention.branch_output_gain": "model-00013-of-00020.safetensors",
162
+ "model.layers.14.attention.dense.weight": "model-00013-of-00020.safetensors",
163
+ "model.layers.14.attention.key_layernorm.weight": "model-00013-of-00020.safetensors",
164
+ "model.layers.14.attention.quasar_attention.A_log": "model-00013-of-00020.safetensors",
165
+ "model.layers.14.attention.quasar_attention.b_proj.weight": "model-00013-of-00020.safetensors",
166
+ "model.layers.14.attention.quasar_attention.dt_bias": "model-00013-of-00020.safetensors",
167
+ "model.layers.14.attention.quasar_attention.f_proj.weight": "model-00013-of-00020.safetensors",
168
+ "model.layers.14.attention.quasar_attention.g_proj.0.weight": "model-00013-of-00020.safetensors",
169
+ "model.layers.14.attention.quasar_attention.g_proj.1.bias": "model-00013-of-00020.safetensors",
170
+ "model.layers.14.attention.quasar_attention.g_proj.1.weight": "model-00013-of-00020.safetensors",
171
+ "model.layers.14.attention.quasar_attention.k_proj.weight": "model-00013-of-00020.safetensors",
172
+ "model.layers.14.attention.quasar_attention.o_norm.weight": "model-00013-of-00020.safetensors",
173
+ "model.layers.14.attention.quasar_attention.o_proj.weight": "model-00013-of-00020.safetensors",
174
+ "model.layers.14.attention.quasar_attention.q_proj.weight": "model-00013-of-00020.safetensors",
175
+ "model.layers.14.attention.quasar_attention.v_proj.weight": "model-00013-of-00020.safetensors",
176
+ "model.layers.14.attention.query_key_value.weight": "model-00013-of-00020.safetensors",
177
+ "model.layers.14.attention.query_layernorm.weight": "model-00013-of-00020.safetensors",
178
+ "model.layers.14.attention.replace_alpha_raw": "model-00013-of-00020.safetensors",
179
+ "model.layers.14.input_layernorm.weight": "model-00014-of-00020.safetensors",
180
  "model.layers.14.mlp.experts_w12": "model-00014-of-00020.safetensors",
181
+ "model.layers.14.mlp.experts_w3": "model-00014-of-00020.safetensors",
182
+ "model.layers.14.mlp.gate.expert_bias": "model-00014-of-00020.safetensors",
183
+ "model.layers.14.mlp.gate.weight": "model-00014-of-00020.safetensors",
184
+ "model.layers.14.mlp.shared_experts.down_proj.weight": "model-00014-of-00020.safetensors",
185
+ "model.layers.14.mlp.shared_experts.gate_proj.weight": "model-00014-of-00020.safetensors",
186
+ "model.layers.14.mlp.shared_experts.up_proj.weight": "model-00014-of-00020.safetensors",
187
+ "model.layers.14.post_attention_layernorm.weight": "model-00014-of-00020.safetensors",
188
+ "model.layers.15.attention.branch_global_output_gain": "model-00014-of-00020.safetensors",
189
+ "model.layers.15.attention.branch_local_window_mix_logit": "model-00014-of-00020.safetensors",
190
+ "model.layers.15.attention.branch_mix_logits": "model-00014-of-00020.safetensors",
191
+ "model.layers.15.attention.branch_output_adapter_down.weight": "model-00014-of-00020.safetensors",
192
+ "model.layers.15.attention.branch_output_adapter_up.weight": "model-00014-of-00020.safetensors",
193
+ "model.layers.15.attention.branch_output_channel_gain": "model-00014-of-00020.safetensors",
194
+ "model.layers.15.attention.branch_output_gain": "model-00014-of-00020.safetensors",
195
+ "model.layers.15.attention.dense.weight": "model-00014-of-00020.safetensors",
196
+ "model.layers.15.attention.key_layernorm.weight": "model-00014-of-00020.safetensors",
197
+ "model.layers.15.attention.query_key_value.weight": "model-00014-of-00020.safetensors",
198
+ "model.layers.15.attention.query_layernorm.weight": "model-00014-of-00020.safetensors",
199
+ "model.layers.15.attention.raven_attention.A_log": "model-00014-of-00020.safetensors",
200
+ "model.layers.15.attention.raven_attention.a_proj.weight": "model-00014-of-00020.safetensors",
201
+ "model.layers.15.attention.raven_attention.dt_bias": "model-00014-of-00020.safetensors",
202
+ "model.layers.15.attention.raven_attention.g_norm.weight": "model-00014-of-00020.safetensors",
203
+ "model.layers.15.attention.raven_attention.k_norm.weight": "model-00014-of-00020.safetensors",
204
+ "model.layers.15.attention.raven_attention.k_proj.weight": "model-00014-of-00020.safetensors",
205
+ "model.layers.15.attention.raven_attention.o_proj.weight": "model-00014-of-00020.safetensors",
206
+ "model.layers.15.attention.raven_attention.q_norm.weight": "model-00014-of-00020.safetensors",
207
+ "model.layers.15.attention.raven_attention.q_proj.weight": "model-00014-of-00020.safetensors",
208
+ "model.layers.15.attention.raven_attention.r_proj.weight": "model-00014-of-00020.safetensors",
209
+ "model.layers.15.attention.raven_attention.v_proj.weight": "model-00014-of-00020.safetensors",
210
+ "model.layers.15.attention.replace_alpha_raw": "model-00014-of-00020.safetensors",
211
+ "model.layers.15.input_layernorm.weight": "model-00015-of-00020.safetensors",
212
  "model.layers.15.mlp.experts_w12": "model-00015-of-00020.safetensors",
213
+ "model.layers.15.mlp.experts_w3": "model-00015-of-00020.safetensors",
214
+ "model.layers.15.mlp.gate.expert_bias": "model-00015-of-00020.safetensors",
215
+ "model.layers.15.mlp.gate.weight": "model-00015-of-00020.safetensors",
216
+ "model.layers.15.mlp.shared_experts.down_proj.weight": "model-00015-of-00020.safetensors",
217
+ "model.layers.15.mlp.shared_experts.gate_proj.weight": "model-00015-of-00020.safetensors",
218
+ "model.layers.15.mlp.shared_experts.up_proj.weight": "model-00015-of-00020.safetensors",
219
+ "model.layers.15.post_attention_layernorm.weight": "model-00015-of-00020.safetensors",
220
+ "model.layers.16.attention.branch_global_output_gain": "model-00015-of-00020.safetensors",
221
+ "model.layers.16.attention.branch_local_window_mix_logit": "model-00015-of-00020.safetensors",
222
+ "model.layers.16.attention.branch_mix_logits": "model-00015-of-00020.safetensors",
223
+ "model.layers.16.attention.branch_output_adapter_down.weight": "model-00015-of-00020.safetensors",
224
+ "model.layers.16.attention.branch_output_adapter_up.weight": "model-00015-of-00020.safetensors",
225
+ "model.layers.16.attention.branch_output_channel_gain": "model-00015-of-00020.safetensors",
226
+ "model.layers.16.attention.branch_output_gain": "model-00015-of-00020.safetensors",
227
+ "model.layers.16.attention.dense.weight": "model-00015-of-00020.safetensors",
228
+ "model.layers.16.attention.key_layernorm.weight": "model-00015-of-00020.safetensors",
229
+ "model.layers.16.attention.quasar_attention.A_log": "model-00015-of-00020.safetensors",
230
+ "model.layers.16.attention.quasar_attention.b_proj.weight": "model-00015-of-00020.safetensors",
231
+ "model.layers.16.attention.quasar_attention.dt_bias": "model-00015-of-00020.safetensors",
232
+ "model.layers.16.attention.quasar_attention.f_proj.weight": "model-00015-of-00020.safetensors",
233
+ "model.layers.16.attention.quasar_attention.g_proj.0.weight": "model-00015-of-00020.safetensors",
234
+ "model.layers.16.attention.quasar_attention.g_proj.1.bias": "model-00015-of-00020.safetensors",
235
+ "model.layers.16.attention.quasar_attention.g_proj.1.weight": "model-00015-of-00020.safetensors",
236
+ "model.layers.16.attention.quasar_attention.k_proj.weight": "model-00015-of-00020.safetensors",
237
+ "model.layers.16.attention.quasar_attention.o_norm.weight": "model-00015-of-00020.safetensors",
238
+ "model.layers.16.attention.quasar_attention.o_proj.weight": "model-00015-of-00020.safetensors",
239
+ "model.layers.16.attention.quasar_attention.q_proj.weight": "model-00015-of-00020.safetensors",
240
+ "model.layers.16.attention.quasar_attention.v_proj.weight": "model-00015-of-00020.safetensors",
241
+ "model.layers.16.attention.query_key_value.weight": "model-00015-of-00020.safetensors",
242
+ "model.layers.16.attention.query_layernorm.weight": "model-00015-of-00020.safetensors",
243
+ "model.layers.16.attention.replace_alpha_raw": "model-00015-of-00020.safetensors",
244
+ "model.layers.16.input_layernorm.weight": "model-00016-of-00020.safetensors",
245
  "model.layers.16.mlp.experts_w12": "model-00016-of-00020.safetensors",
246
+ "model.layers.16.mlp.experts_w3": "model-00016-of-00020.safetensors",
247
+ "model.layers.16.mlp.gate.expert_bias": "model-00016-of-00020.safetensors",
248
+ "model.layers.16.mlp.gate.weight": "model-00016-of-00020.safetensors",
249
+ "model.layers.16.mlp.shared_experts.down_proj.weight": "model-00016-of-00020.safetensors",
250
+ "model.layers.16.mlp.shared_experts.gate_proj.weight": "model-00016-of-00020.safetensors",
251
+ "model.layers.16.mlp.shared_experts.up_proj.weight": "model-00016-of-00020.safetensors",
252
+ "model.layers.16.post_attention_layernorm.weight": "model-00016-of-00020.safetensors",
253
+ "model.layers.17.attention.branch_global_output_gain": "model-00016-of-00020.safetensors",
254
+ "model.layers.17.attention.branch_local_window_mix_logit": "model-00016-of-00020.safetensors",
255
+ "model.layers.17.attention.branch_mix_logits": "model-00016-of-00020.safetensors",
256
+ "model.layers.17.attention.branch_output_adapter_down.weight": "model-00016-of-00020.safetensors",
257
+ "model.layers.17.attention.branch_output_adapter_up.weight": "model-00016-of-00020.safetensors",
258
+ "model.layers.17.attention.branch_output_channel_gain": "model-00016-of-00020.safetensors",
259
+ "model.layers.17.attention.branch_output_gain": "model-00016-of-00020.safetensors",
260
+ "model.layers.17.attention.dense.weight": "model-00016-of-00020.safetensors",
261
+ "model.layers.17.attention.key_layernorm.weight": "model-00016-of-00020.safetensors",
262
+ "model.layers.17.attention.quasar_attention.A_log": "model-00016-of-00020.safetensors",
263
+ "model.layers.17.attention.quasar_attention.b_proj.weight": "model-00016-of-00020.safetensors",
264
+ "model.layers.17.attention.quasar_attention.dt_bias": "model-00016-of-00020.safetensors",
265
+ "model.layers.17.attention.quasar_attention.f_proj.weight": "model-00016-of-00020.safetensors",
266
+ "model.layers.17.attention.quasar_attention.g_proj.0.weight": "model-00016-of-00020.safetensors",
267
+ "model.layers.17.attention.quasar_attention.g_proj.1.bias": "model-00016-of-00020.safetensors",
268
+ "model.layers.17.attention.quasar_attention.g_proj.1.weight": "model-00016-of-00020.safetensors",
269
+ "model.layers.17.attention.quasar_attention.k_proj.weight": "model-00016-of-00020.safetensors",
270
+ "model.layers.17.attention.quasar_attention.o_norm.weight": "model-00016-of-00020.safetensors",
271
+ "model.layers.17.attention.quasar_attention.o_proj.weight": "model-00016-of-00020.safetensors",
272
+ "model.layers.17.attention.quasar_attention.q_proj.weight": "model-00016-of-00020.safetensors",
273
+ "model.layers.17.attention.quasar_attention.v_proj.weight": "model-00016-of-00020.safetensors",
274
+ "model.layers.17.attention.query_key_value.weight": "model-00016-of-00020.safetensors",
275
+ "model.layers.17.attention.query_layernorm.weight": "model-00016-of-00020.safetensors",
276
+ "model.layers.17.attention.replace_alpha_raw": "model-00016-of-00020.safetensors",
277
+ "model.layers.17.input_layernorm.weight": "model-00017-of-00020.safetensors",
278
  "model.layers.17.mlp.experts_w12": "model-00017-of-00020.safetensors",
279
+ "model.layers.17.mlp.experts_w3": "model-00017-of-00020.safetensors",
280
+ "model.layers.17.mlp.gate.expert_bias": "model-00017-of-00020.safetensors",
281
+ "model.layers.17.mlp.gate.weight": "model-00017-of-00020.safetensors",
282
+ "model.layers.17.mlp.shared_experts.down_proj.weight": "model-00017-of-00020.safetensors",
283
+ "model.layers.17.mlp.shared_experts.gate_proj.weight": "model-00017-of-00020.safetensors",
284
+ "model.layers.17.mlp.shared_experts.up_proj.weight": "model-00017-of-00020.safetensors",
285
+ "model.layers.17.post_attention_layernorm.weight": "model-00017-of-00020.safetensors",
286
+ "model.layers.18.attention.branch_global_output_gain": "model-00017-of-00020.safetensors",
287
+ "model.layers.18.attention.branch_local_window_mix_logit": "model-00017-of-00020.safetensors",
288
+ "model.layers.18.attention.branch_mix_logits": "model-00017-of-00020.safetensors",
289
+ "model.layers.18.attention.branch_output_adapter_down.weight": "model-00017-of-00020.safetensors",
290
+ "model.layers.18.attention.branch_output_adapter_up.weight": "model-00017-of-00020.safetensors",
291
+ "model.layers.18.attention.branch_output_channel_gain": "model-00017-of-00020.safetensors",
292
+ "model.layers.18.attention.branch_output_gain": "model-00017-of-00020.safetensors",
293
+ "model.layers.18.attention.dense.weight": "model-00017-of-00020.safetensors",
294
+ "model.layers.18.attention.gla_attention.dense.weight": "model-00017-of-00020.safetensors",
295
+ "model.layers.18.attention.gla_attention.g_norm.weight": "model-00017-of-00020.safetensors",
296
+ "model.layers.18.attention.gla_attention.g_proj.weight": "model-00017-of-00020.safetensors",
297
+ "model.layers.18.attention.gla_attention.key_layernorm.weight": "model-00017-of-00020.safetensors",
298
+ "model.layers.18.attention.gla_attention.query_key_value.weight": "model-00017-of-00020.safetensors",
299
+ "model.layers.18.attention.gla_attention.query_layernorm.weight": "model-00017-of-00020.safetensors",
300
+ "model.layers.18.attention.gla_attention.slope": "model-00017-of-00020.safetensors",
301
+ "model.layers.18.attention.key_layernorm.weight": "model-00017-of-00020.safetensors",
302
+ "model.layers.18.attention.query_key_value.weight": "model-00017-of-00020.safetensors",
303
+ "model.layers.18.attention.query_layernorm.weight": "model-00017-of-00020.safetensors",
304
+ "model.layers.18.attention.replace_alpha_raw": "model-00017-of-00020.safetensors",
305
+ "model.layers.18.input_layernorm.weight": "model-00018-of-00020.safetensors",
306
  "model.layers.18.mlp.experts_w12": "model-00018-of-00020.safetensors",
307
+ "model.layers.18.mlp.experts_w3": "model-00018-of-00020.safetensors",
308
+ "model.layers.18.mlp.gate.expert_bias": "model-00018-of-00020.safetensors",
309
+ "model.layers.18.mlp.gate.weight": "model-00018-of-00020.safetensors",
310
+ "model.layers.18.mlp.shared_experts.down_proj.weight": "model-00018-of-00020.safetensors",
311
+ "model.layers.18.mlp.shared_experts.gate_proj.weight": "model-00018-of-00020.safetensors",
312
+ "model.layers.18.mlp.shared_experts.up_proj.weight": "model-00018-of-00020.safetensors",
313
+ "model.layers.18.post_attention_layernorm.weight": "model-00018-of-00020.safetensors",
314
+ "model.layers.19.attention.branch_global_output_gain": "model-00018-of-00020.safetensors",
315
+ "model.layers.19.attention.branch_local_window_mix_logit": "model-00018-of-00020.safetensors",
316
+ "model.layers.19.attention.branch_mix_logits": "model-00018-of-00020.safetensors",
317
+ "model.layers.19.attention.branch_output_adapter_down.weight": "model-00018-of-00020.safetensors",
318
+ "model.layers.19.attention.branch_output_adapter_up.weight": "model-00018-of-00020.safetensors",
319
+ "model.layers.19.attention.branch_output_channel_gain": "model-00018-of-00020.safetensors",
320
+ "model.layers.19.attention.branch_output_gain": "model-00018-of-00020.safetensors",
321
+ "model.layers.19.attention.dense.weight": "model-00018-of-00020.safetensors",
322
+ "model.layers.19.attention.key_layernorm.weight": "model-00018-of-00020.safetensors",
323
+ "model.layers.19.attention.quasar_attention.A_log": "model-00018-of-00020.safetensors",
324
+ "model.layers.19.attention.quasar_attention.b_proj.weight": "model-00018-of-00020.safetensors",
325
+ "model.layers.19.attention.quasar_attention.dt_bias": "model-00018-of-00020.safetensors",
326
+ "model.layers.19.attention.quasar_attention.f_proj.weight": "model-00018-of-00020.safetensors",
327
+ "model.layers.19.attention.quasar_attention.g_proj.0.weight": "model-00018-of-00020.safetensors",
328
+ "model.layers.19.attention.quasar_attention.g_proj.1.bias": "model-00018-of-00020.safetensors",
329
+ "model.layers.19.attention.quasar_attention.g_proj.1.weight": "model-00018-of-00020.safetensors",
330
+ "model.layers.19.attention.quasar_attention.k_proj.weight": "model-00018-of-00020.safetensors",
331
+ "model.layers.19.attention.quasar_attention.o_norm.weight": "model-00018-of-00020.safetensors",
332
+ "model.layers.19.attention.quasar_attention.o_proj.weight": "model-00018-of-00020.safetensors",
333
+ "model.layers.19.attention.quasar_attention.q_proj.weight": "model-00018-of-00020.safetensors",
334
+ "model.layers.19.attention.quasar_attention.v_proj.weight": "model-00018-of-00020.safetensors",
335
+ "model.layers.19.attention.query_key_value.weight": "model-00018-of-00020.safetensors",
336
+ "model.layers.19.attention.query_layernorm.weight": "model-00018-of-00020.safetensors",
337
+ "model.layers.19.attention.replace_alpha_raw": "model-00018-of-00020.safetensors",
338
+ "model.layers.19.input_layernorm.weight": "model-00019-of-00020.safetensors",
339
  "model.layers.19.mlp.experts_w12": "model-00019-of-00020.safetensors",
340
+ "model.layers.19.mlp.experts_w3": "model-00019-of-00020.safetensors",
341
+ "model.layers.19.mlp.gate.expert_bias": "model-00019-of-00020.safetensors",
342
+ "model.layers.19.mlp.gate.weight": "model-00019-of-00020.safetensors",
343
+ "model.layers.19.mlp.shared_experts.down_proj.weight": "model-00019-of-00020.safetensors",
344
+ "model.layers.19.mlp.shared_experts.gate_proj.weight": "model-00019-of-00020.safetensors",
345
+ "model.layers.19.mlp.shared_experts.up_proj.weight": "model-00019-of-00020.safetensors",
346
+ "model.layers.19.post_attention_layernorm.weight": "model-00019-of-00020.safetensors",
347
  "model.layers.2.attention.dense.weight": "model-00002-of-00020.safetensors",
348
  "model.layers.2.attention.key_layernorm.weight": "model-00002-of-00020.safetensors",
349
  "model.layers.2.attention.query_key_value.weight": "model-00002-of-00020.safetensors",
350
  "model.layers.2.attention.query_layernorm.weight": "model-00002-of-00020.safetensors",
351
+ "model.layers.2.input_layernorm.weight": "model-00002-of-00020.safetensors",
352
  "model.layers.2.mlp.experts_w12": "model-00002-of-00020.safetensors",
353
+ "model.layers.2.mlp.experts_w3": "model-00002-of-00020.safetensors",
354
+ "model.layers.2.mlp.gate.expert_bias": "model-00002-of-00020.safetensors",
355
+ "model.layers.2.mlp.gate.weight": "model-00002-of-00020.safetensors",
356
+ "model.layers.2.mlp.shared_experts.down_proj.weight": "model-00002-of-00020.safetensors",
357
+ "model.layers.2.mlp.shared_experts.gate_proj.weight": "model-00002-of-00020.safetensors",
358
+ "model.layers.2.mlp.shared_experts.up_proj.weight": "model-00002-of-00020.safetensors",
359
+ "model.layers.2.post_attention_layernorm.weight": "model-00002-of-00020.safetensors",
360
+ "model.layers.3.attention.dense.weight": "model-00002-of-00020.safetensors",
361
+ "model.layers.3.attention.key_layernorm.weight": "model-00002-of-00020.safetensors",
362
+ "model.layers.3.attention.query_key_value.weight": "model-00002-of-00020.safetensors",
363
+ "model.layers.3.attention.query_layernorm.weight": "model-00002-of-00020.safetensors",
364
+ "model.layers.3.input_layernorm.weight": "model-00003-of-00020.safetensors",
365
  "model.layers.3.mlp.experts_w12": "model-00003-of-00020.safetensors",
366
+ "model.layers.3.mlp.experts_w3": "model-00003-of-00020.safetensors",
367
+ "model.layers.3.mlp.gate.expert_bias": "model-00003-of-00020.safetensors",
368
+ "model.layers.3.mlp.gate.weight": "model-00003-of-00020.safetensors",
369
+ "model.layers.3.mlp.shared_experts.down_proj.weight": "model-00003-of-00020.safetensors",
370
+ "model.layers.3.mlp.shared_experts.gate_proj.weight": "model-00003-of-00020.safetensors",
371
+ "model.layers.3.mlp.shared_experts.up_proj.weight": "model-00003-of-00020.safetensors",
372
+ "model.layers.3.post_attention_layernorm.weight": "model-00003-of-00020.safetensors",
373
+ "model.layers.4.attention.branch_global_output_gain": "model-00003-of-00020.safetensors",
374
+ "model.layers.4.attention.branch_local_window_mix_logit": "model-00003-of-00020.safetensors",
375
+ "model.layers.4.attention.branch_mix_logits": "model-00003-of-00020.safetensors",
376
+ "model.layers.4.attention.branch_output_adapter_down.weight": "model-00003-of-00020.safetensors",
377
+ "model.layers.4.attention.branch_output_adapter_up.weight": "model-00003-of-00020.safetensors",
378
+ "model.layers.4.attention.branch_output_channel_gain": "model-00003-of-00020.safetensors",
379
+ "model.layers.4.attention.branch_output_gain": "model-00003-of-00020.safetensors",
380
+ "model.layers.4.attention.dense.weight": "model-00003-of-00020.safetensors",
381
+ "model.layers.4.attention.key_layernorm.weight": "model-00003-of-00020.safetensors",
382
+ "model.layers.4.attention.quasar_attention.A_log": "model-00003-of-00020.safetensors",
383
+ "model.layers.4.attention.quasar_attention.b_proj.weight": "model-00003-of-00020.safetensors",
384
+ "model.layers.4.attention.quasar_attention.dt_bias": "model-00003-of-00020.safetensors",
385
+ "model.layers.4.attention.quasar_attention.f_proj.weight": "model-00003-of-00020.safetensors",
386
+ "model.layers.4.attention.quasar_attention.g_proj.0.weight": "model-00003-of-00020.safetensors",
387
+ "model.layers.4.attention.quasar_attention.g_proj.1.bias": "model-00003-of-00020.safetensors",
388
+ "model.layers.4.attention.quasar_attention.g_proj.1.weight": "model-00003-of-00020.safetensors",
389
+ "model.layers.4.attention.quasar_attention.k_proj.weight": "model-00003-of-00020.safetensors",
390
+ "model.layers.4.attention.quasar_attention.o_norm.weight": "model-00003-of-00020.safetensors",
391
+ "model.layers.4.attention.quasar_attention.o_proj.weight": "model-00003-of-00020.safetensors",
392
+ "model.layers.4.attention.quasar_attention.q_proj.weight": "model-00003-of-00020.safetensors",
393
+ "model.layers.4.attention.quasar_attention.v_proj.weight": "model-00003-of-00020.safetensors",
394
+ "model.layers.4.attention.query_key_value.weight": "model-00003-of-00020.safetensors",
395
+ "model.layers.4.attention.query_layernorm.weight": "model-00003-of-00020.safetensors",
396
+ "model.layers.4.attention.replace_alpha_raw": "model-00003-of-00020.safetensors",
397
+ "model.layers.4.input_layernorm.weight": "model-00004-of-00020.safetensors",
398
  "model.layers.4.mlp.experts_w12": "model-00004-of-00020.safetensors",
399
+ "model.layers.4.mlp.experts_w3": "model-00004-of-00020.safetensors",
400
+ "model.layers.4.mlp.gate.expert_bias": "model-00004-of-00020.safetensors",
401
+ "model.layers.4.mlp.gate.weight": "model-00004-of-00020.safetensors",
402
+ "model.layers.4.mlp.shared_experts.down_proj.weight": "model-00004-of-00020.safetensors",
403
+ "model.layers.4.mlp.shared_experts.gate_proj.weight": "model-00004-of-00020.safetensors",
404
+ "model.layers.4.mlp.shared_experts.up_proj.weight": "model-00004-of-00020.safetensors",
405
+ "model.layers.4.post_attention_layernorm.weight": "model-00004-of-00020.safetensors",
406
+ "model.layers.5.attention.branch_global_output_gain": "model-00004-of-00020.safetensors",
407
+ "model.layers.5.attention.branch_local_window_mix_logit": "model-00004-of-00020.safetensors",
408
+ "model.layers.5.attention.branch_mix_logits": "model-00004-of-00020.safetensors",
409
+ "model.layers.5.attention.branch_output_adapter_down.weight": "model-00004-of-00020.safetensors",
410
+ "model.layers.5.attention.branch_output_adapter_up.weight": "model-00004-of-00020.safetensors",
411
+ "model.layers.5.attention.branch_output_channel_gain": "model-00004-of-00020.safetensors",
412
+ "model.layers.5.attention.branch_output_gain": "model-00004-of-00020.safetensors",
413
+ "model.layers.5.attention.dense.weight": "model-00004-of-00020.safetensors",
414
+ "model.layers.5.attention.key_layernorm.weight": "model-00004-of-00020.safetensors",
415
+ "model.layers.5.attention.query_key_value.weight": "model-00004-of-00020.safetensors",
416
+ "model.layers.5.attention.query_layernorm.weight": "model-00004-of-00020.safetensors",
417
+ "model.layers.5.attention.raven_attention.A_log": "model-00004-of-00020.safetensors",
418
+ "model.layers.5.attention.raven_attention.a_proj.weight": "model-00004-of-00020.safetensors",
419
+ "model.layers.5.attention.raven_attention.dt_bias": "model-00004-of-00020.safetensors",
420
+ "model.layers.5.attention.raven_attention.g_norm.weight": "model-00004-of-00020.safetensors",
421
+ "model.layers.5.attention.raven_attention.k_norm.weight": "model-00004-of-00020.safetensors",
422
+ "model.layers.5.attention.raven_attention.k_proj.weight": "model-00004-of-00020.safetensors",
423
+ "model.layers.5.attention.raven_attention.o_proj.weight": "model-00004-of-00020.safetensors",
424
+ "model.layers.5.attention.raven_attention.q_norm.weight": "model-00004-of-00020.safetensors",
425
+ "model.layers.5.attention.raven_attention.q_proj.weight": "model-00004-of-00020.safetensors",
426
+ "model.layers.5.attention.raven_attention.r_proj.weight": "model-00004-of-00020.safetensors",
427
+ "model.layers.5.attention.raven_attention.v_proj.weight": "model-00004-of-00020.safetensors",
428
+ "model.layers.5.attention.replace_alpha_raw": "model-00004-of-00020.safetensors",
429
+ "model.layers.5.input_layernorm.weight": "model-00005-of-00020.safetensors",
430
  "model.layers.5.mlp.experts_w12": "model-00005-of-00020.safetensors",
431
+ "model.layers.5.mlp.experts_w3": "model-00005-of-00020.safetensors",
432
+ "model.layers.5.mlp.gate.expert_bias": "model-00005-of-00020.safetensors",
433
+ "model.layers.5.mlp.gate.weight": "model-00005-of-00020.safetensors",
434
+ "model.layers.5.mlp.shared_experts.down_proj.weight": "model-00005-of-00020.safetensors",
435
+ "model.layers.5.mlp.shared_experts.gate_proj.weight": "model-00005-of-00020.safetensors",
436
+ "model.layers.5.mlp.shared_experts.up_proj.weight": "model-00005-of-00020.safetensors",
437
+ "model.layers.5.post_attention_layernorm.weight": "model-00005-of-00020.safetensors",
438
+ "model.layers.6.attention.branch_global_output_gain": "model-00005-of-00020.safetensors",
439
+ "model.layers.6.attention.branch_local_window_mix_logit": "model-00005-of-00020.safetensors",
440
+ "model.layers.6.attention.branch_mix_logits": "model-00005-of-00020.safetensors",
441
+ "model.layers.6.attention.branch_output_adapter_down.weight": "model-00005-of-00020.safetensors",
442
+ "model.layers.6.attention.branch_output_adapter_up.weight": "model-00005-of-00020.safetensors",
443
+ "model.layers.6.attention.branch_output_channel_gain": "model-00005-of-00020.safetensors",
444
+ "model.layers.6.attention.branch_output_gain": "model-00005-of-00020.safetensors",
445
+ "model.layers.6.attention.dense.weight": "model-00005-of-00020.safetensors",
446
+ "model.layers.6.attention.key_layernorm.weight": "model-00005-of-00020.safetensors",
447
+ "model.layers.6.attention.quasar_attention.A_log": "model-00005-of-00020.safetensors",
448
+ "model.layers.6.attention.quasar_attention.b_proj.weight": "model-00005-of-00020.safetensors",
449
+ "model.layers.6.attention.quasar_attention.dt_bias": "model-00005-of-00020.safetensors",
450
+ "model.layers.6.attention.quasar_attention.f_proj.weight": "model-00005-of-00020.safetensors",
451
+ "model.layers.6.attention.quasar_attention.g_proj.0.weight": "model-00005-of-00020.safetensors",
452
+ "model.layers.6.attention.quasar_attention.g_proj.1.bias": "model-00005-of-00020.safetensors",
453
+ "model.layers.6.attention.quasar_attention.g_proj.1.weight": "model-00005-of-00020.safetensors",
454
+ "model.layers.6.attention.quasar_attention.k_proj.weight": "model-00005-of-00020.safetensors",
455
+ "model.layers.6.attention.quasar_attention.o_norm.weight": "model-00005-of-00020.safetensors",
456
+ "model.layers.6.attention.quasar_attention.o_proj.weight": "model-00005-of-00020.safetensors",
457
+ "model.layers.6.attention.quasar_attention.q_proj.weight": "model-00005-of-00020.safetensors",
458
+ "model.layers.6.attention.quasar_attention.v_proj.weight": "model-00005-of-00020.safetensors",
459
+ "model.layers.6.attention.query_key_value.weight": "model-00005-of-00020.safetensors",
460
+ "model.layers.6.attention.query_layernorm.weight": "model-00005-of-00020.safetensors",
461
+ "model.layers.6.attention.replace_alpha_raw": "model-00005-of-00020.safetensors",
462
+ "model.layers.6.input_layernorm.weight": "model-00006-of-00020.safetensors",
463
  "model.layers.6.mlp.experts_w12": "model-00006-of-00020.safetensors",
464
+ "model.layers.6.mlp.experts_w3": "model-00006-of-00020.safetensors",
465
+ "model.layers.6.mlp.gate.expert_bias": "model-00006-of-00020.safetensors",
466
+ "model.layers.6.mlp.gate.weight": "model-00006-of-00020.safetensors",
467
+ "model.layers.6.mlp.shared_experts.down_proj.weight": "model-00006-of-00020.safetensors",
468
+ "model.layers.6.mlp.shared_experts.gate_proj.weight": "model-00006-of-00020.safetensors",
469
+ "model.layers.6.mlp.shared_experts.up_proj.weight": "model-00006-of-00020.safetensors",
470
+ "model.layers.6.post_attention_layernorm.weight": "model-00006-of-00020.safetensors",
471
+ "model.layers.7.attention.branch_global_output_gain": "model-00006-of-00020.safetensors",
472
+ "model.layers.7.attention.branch_local_window_mix_logit": "model-00006-of-00020.safetensors",
473
+ "model.layers.7.attention.branch_mix_logits": "model-00006-of-00020.safetensors",
474
+ "model.layers.7.attention.branch_output_adapter_down.weight": "model-00006-of-00020.safetensors",
475
+ "model.layers.7.attention.branch_output_adapter_up.weight": "model-00006-of-00020.safetensors",
476
+ "model.layers.7.attention.branch_output_channel_gain": "model-00006-of-00020.safetensors",
477
+ "model.layers.7.attention.branch_output_gain": "model-00006-of-00020.safetensors",
478
+ "model.layers.7.attention.dense.weight": "model-00006-of-00020.safetensors",
479
+ "model.layers.7.attention.key_layernorm.weight": "model-00006-of-00020.safetensors",
480
+ "model.layers.7.attention.quasar_attention.A_log": "model-00006-of-00020.safetensors",
481
+ "model.layers.7.attention.quasar_attention.b_proj.weight": "model-00006-of-00020.safetensors",
482
+ "model.layers.7.attention.quasar_attention.dt_bias": "model-00006-of-00020.safetensors",
483
+ "model.layers.7.attention.quasar_attention.f_proj.weight": "model-00006-of-00020.safetensors",
484
+ "model.layers.7.attention.quasar_attention.g_proj.0.weight": "model-00006-of-00020.safetensors",
485
+ "model.layers.7.attention.quasar_attention.g_proj.1.bias": "model-00006-of-00020.safetensors",
486
+ "model.layers.7.attention.quasar_attention.g_proj.1.weight": "model-00006-of-00020.safetensors",
487
+ "model.layers.7.attention.quasar_attention.k_proj.weight": "model-00006-of-00020.safetensors",
488
+ "model.layers.7.attention.quasar_attention.o_norm.weight": "model-00006-of-00020.safetensors",
489
+ "model.layers.7.attention.quasar_attention.o_proj.weight": "model-00006-of-00020.safetensors",
490
+ "model.layers.7.attention.quasar_attention.q_proj.weight": "model-00006-of-00020.safetensors",
491
+ "model.layers.7.attention.quasar_attention.v_proj.weight": "model-00006-of-00020.safetensors",
492
+ "model.layers.7.attention.query_key_value.weight": "model-00006-of-00020.safetensors",
493
+ "model.layers.7.attention.query_layernorm.weight": "model-00006-of-00020.safetensors",
494
+ "model.layers.7.attention.replace_alpha_raw": "model-00006-of-00020.safetensors",
495
+ "model.layers.7.input_layernorm.weight": "model-00007-of-00020.safetensors",
496
  "model.layers.7.mlp.experts_w12": "model-00007-of-00020.safetensors",
497
+ "model.layers.7.mlp.experts_w3": "model-00007-of-00020.safetensors",
498
+ "model.layers.7.mlp.gate.expert_bias": "model-00007-of-00020.safetensors",
499
+ "model.layers.7.mlp.gate.weight": "model-00007-of-00020.safetensors",
500
+ "model.layers.7.mlp.shared_experts.down_proj.weight": "model-00007-of-00020.safetensors",
501
+ "model.layers.7.mlp.shared_experts.gate_proj.weight": "model-00007-of-00020.safetensors",
502
+ "model.layers.7.mlp.shared_experts.up_proj.weight": "model-00007-of-00020.safetensors",
503
+ "model.layers.7.post_attention_layernorm.weight": "model-00007-of-00020.safetensors",
504
+ "model.layers.8.attention.branch_global_output_gain": "model-00007-of-00020.safetensors",
505
+ "model.layers.8.attention.branch_local_window_mix_logit": "model-00007-of-00020.safetensors",
506
+ "model.layers.8.attention.branch_mix_logits": "model-00007-of-00020.safetensors",
507
+ "model.layers.8.attention.branch_output_adapter_down.weight": "model-00007-of-00020.safetensors",
508
+ "model.layers.8.attention.branch_output_adapter_up.weight": "model-00007-of-00020.safetensors",
509
+ "model.layers.8.attention.branch_output_channel_gain": "model-00007-of-00020.safetensors",
510
+ "model.layers.8.attention.branch_output_gain": "model-00007-of-00020.safetensors",
511
+ "model.layers.8.attention.dense.weight": "model-00007-of-00020.safetensors",
512
+ "model.layers.8.attention.gla_attention.dense.weight": "model-00007-of-00020.safetensors",
513
+ "model.layers.8.attention.gla_attention.g_norm.weight": "model-00007-of-00020.safetensors",
514
+ "model.layers.8.attention.gla_attention.g_proj.weight": "model-00007-of-00020.safetensors",
515
+ "model.layers.8.attention.gla_attention.key_layernorm.weight": "model-00007-of-00020.safetensors",
516
+ "model.layers.8.attention.gla_attention.query_key_value.weight": "model-00007-of-00020.safetensors",
517
+ "model.layers.8.attention.gla_attention.query_layernorm.weight": "model-00007-of-00020.safetensors",
518
+ "model.layers.8.attention.gla_attention.slope": "model-00007-of-00020.safetensors",
519
+ "model.layers.8.attention.key_layernorm.weight": "model-00007-of-00020.safetensors",
520
+ "model.layers.8.attention.query_key_value.weight": "model-00007-of-00020.safetensors",
521
+ "model.layers.8.attention.query_layernorm.weight": "model-00007-of-00020.safetensors",
522
+ "model.layers.8.attention.replace_alpha_raw": "model-00007-of-00020.safetensors",
523
+ "model.layers.8.input_layernorm.weight": "model-00008-of-00020.safetensors",
524
  "model.layers.8.mlp.experts_w12": "model-00008-of-00020.safetensors",
525
+ "model.layers.8.mlp.experts_w3": "model-00008-of-00020.safetensors",
526
+ "model.layers.8.mlp.gate.expert_bias": "model-00008-of-00020.safetensors",
527
+ "model.layers.8.mlp.gate.weight": "model-00008-of-00020.safetensors",
528
+ "model.layers.8.mlp.shared_experts.down_proj.weight": "model-00008-of-00020.safetensors",
529
+ "model.layers.8.mlp.shared_experts.gate_proj.weight": "model-00008-of-00020.safetensors",
530
+ "model.layers.8.mlp.shared_experts.up_proj.weight": "model-00008-of-00020.safetensors",
531
+ "model.layers.8.post_attention_layernorm.weight": "model-00008-of-00020.safetensors",
532
+ "model.layers.9.attention.branch_global_output_gain": "model-00008-of-00020.safetensors",
533
+ "model.layers.9.attention.branch_local_window_mix_logit": "model-00008-of-00020.safetensors",
534
+ "model.layers.9.attention.branch_mix_logits": "model-00008-of-00020.safetensors",
535
+ "model.layers.9.attention.branch_output_adapter_down.weight": "model-00008-of-00020.safetensors",
536
+ "model.layers.9.attention.branch_output_adapter_up.weight": "model-00008-of-00020.safetensors",
537
+ "model.layers.9.attention.branch_output_channel_gain": "model-00008-of-00020.safetensors",
538
+ "model.layers.9.attention.branch_output_gain": "model-00008-of-00020.safetensors",
539
+ "model.layers.9.attention.dense.weight": "model-00008-of-00020.safetensors",
540
+ "model.layers.9.attention.key_layernorm.weight": "model-00008-of-00020.safetensors",
541
+ "model.layers.9.attention.quasar_attention.A_log": "model-00008-of-00020.safetensors",
542
+ "model.layers.9.attention.quasar_attention.b_proj.weight": "model-00008-of-00020.safetensors",
543
+ "model.layers.9.attention.quasar_attention.dt_bias": "model-00008-of-00020.safetensors",
544
+ "model.layers.9.attention.quasar_attention.f_proj.weight": "model-00008-of-00020.safetensors",
545
+ "model.layers.9.attention.quasar_attention.g_proj.0.weight": "model-00008-of-00020.safetensors",
546
+ "model.layers.9.attention.quasar_attention.g_proj.1.bias": "model-00008-of-00020.safetensors",
547
+ "model.layers.9.attention.quasar_attention.g_proj.1.weight": "model-00008-of-00020.safetensors",
548
+ "model.layers.9.attention.quasar_attention.k_proj.weight": "model-00008-of-00020.safetensors",
549
+ "model.layers.9.attention.quasar_attention.o_norm.weight": "model-00008-of-00020.safetensors",
550
+ "model.layers.9.attention.quasar_attention.o_proj.weight": "model-00008-of-00020.safetensors",
551
+ "model.layers.9.attention.quasar_attention.q_proj.weight": "model-00008-of-00020.safetensors",
552
+ "model.layers.9.attention.quasar_attention.v_proj.weight": "model-00008-of-00020.safetensors",
553
+ "model.layers.9.attention.query_key_value.weight": "model-00008-of-00020.safetensors",
554
+ "model.layers.9.attention.query_layernorm.weight": "model-00008-of-00020.safetensors",
555
+ "model.layers.9.attention.replace_alpha_raw": "model-00008-of-00020.safetensors",
556
+ "model.layers.9.input_layernorm.weight": "model-00009-of-00020.safetensors",
557
  "model.layers.9.mlp.experts_w12": "model-00009-of-00020.safetensors",
558
+ "model.layers.9.mlp.experts_w3": "model-00009-of-00020.safetensors",
559
+ "model.layers.9.mlp.gate.expert_bias": "model-00009-of-00020.safetensors",
560
+ "model.layers.9.mlp.gate.weight": "model-00009-of-00020.safetensors",
561
+ "model.layers.9.mlp.shared_experts.down_proj.weight": "model-00009-of-00020.safetensors",
562
+ "model.layers.9.mlp.shared_experts.gate_proj.weight": "model-00009-of-00020.safetensors",
563
+ "model.layers.9.mlp.shared_experts.up_proj.weight": "model-00009-of-00020.safetensors",
564
+ "model.layers.9.post_attention_layernorm.weight": "model-00009-of-00020.safetensors",
565
+ "model.norm.weight": "model-00019-of-00020.safetensors",
566
+ "model.rotary_emb.inv_freq": "model-00019-of-00020.safetensors",
567
  "model.word_embeddings.weight": "model-00001-of-00020.safetensors"
568
  }
569
  }
modeling_quasar_long.py CHANGED
@@ -75,6 +75,7 @@ except ImportError:
75
  return False
76
  from .configuration_quasar_long import QuasarLongConfig
77
  from transformers.generation.utils import GenerationMixin
 
78
  from dataclasses import dataclass
79
  from transformers.utils import ModelOutput
80
  try:
 
75
  return False
76
  from .configuration_quasar_long import QuasarLongConfig
77
  from transformers.generation.utils import GenerationMixin
78
+ _CONFIG_FOR_DOC = "QuasarLongConfig"
79
  from dataclasses import dataclass
80
  from transformers.utils import ModelOutput
81
  try:
raven/__init__.py ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ # -*- coding: utf-8 -*-
2
+
3
+ from raven.models.raven import RavenConfig, RavenForCausalLM, RavenModel
4
+
5
+ __all__ = [
6
+ 'RavenConfig', 'RavenForCausalLM', 'RavenModel',
7
+ ]
raven/__pycache__/__init__.cpython-312.pyc ADDED
Binary file (292 Bytes). View file
 
raven/layers/__init__.py ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ # -*- coding: utf-8 -*-
2
+
3
+ from raven.layers.raven import RavenAttention
4
+
5
+ __all__ = ['RavenAttention']
raven/layers/__pycache__/__init__.cpython-312.pyc ADDED
Binary file (254 Bytes). View file
 
raven/layers/__pycache__/raven.cpython-312.pyc ADDED
Binary file (18 kB). View file
 
raven/layers/raven.py ADDED
@@ -0,0 +1,348 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+
3
+ from __future__ import annotations
4
+
5
+ import warnings
6
+ from typing import TYPE_CHECKING, Dict, Optional, Tuple
7
+
8
+ import torch
9
+ import torch.nn as nn
10
+ import math
11
+ import torch.nn.functional as F
12
+ from einops import rearrange, repeat
13
+
14
+ from fla.layers.utils import get_unpad_data, index_first_axis, pad_input
15
+ from fla.modules.feature_map import ReLUFeatureMap, SwishFeatureMap, T2RFeatureMap
16
+ from fla.modules.layernorm import rms_norm_linear
17
+ from fla.ops.gsa import chunk_gsa as chunk_raven, fused_recurrent_gsa as fused_recurrent_raven
18
+
19
+
20
+ from fla.ops.utils.index import prepare_lens_from_mask
21
+
22
+
23
+ from fla.modules import FusedRMSNormGated , RMSNorm, RotaryEmbedding #Added for RoPE
24
+
25
+ if TYPE_CHECKING:
26
+ from transformers.processing_utils import Unpack
27
+
28
+ from fla.models.utils import Cache
29
+
30
+ def _max_offset(seqlen_offset):
31
+ if seqlen_offset is None:
32
+ return 0
33
+ if isinstance(seqlen_offset, int):
34
+ # scalar offset, nothing fancy
35
+ return seqlen_offset
36
+ if isinstance(seqlen_offset, torch.Tensor):
37
+ # tensor of offsets -> take max
38
+ return int(seqlen_offset.max().item())
39
+ # list/tuple/other iterables
40
+ return max(seqlen_offset)
41
+
42
+
43
+ class RavenAttention(nn.Module):
44
+
45
+ def __init__(
46
+ self,
47
+ mode: str = 'chunk',
48
+ hidden_size: int = 1024,
49
+ expand_k: float = 1.,
50
+ expand_v: float = 1.,
51
+ num_heads: int = 4,
52
+ num_kv_heads: Optional[int] = None,
53
+ num_slots: Optional[int] = None,
54
+ elementwise_affine: Optional[bool] = True,
55
+ norm_eps: float = 1e-5,
56
+ gate_logit_normalizer: int = 8,
57
+ feature_map: str = 'swish',
58
+ use_output_gate: bool = False,
59
+ use_norm: bool = True,
60
+ layer_idx: Optional[int] = None,
61
+ scale: Optional[float] = 1.,
62
+ decay_type: str = 'Mamba2',
63
+ topk: int = 32,
64
+ bias_rmm: bool = False,
65
+ add_gumbel_noise: bool = True,
66
+ router_score: str = 'sigmoid',
67
+ router_type: str = 'lin',
68
+ use_rope: bool = False,
69
+ **kwargs
70
+ ) -> RavenAttention:
71
+ super().__init__()
72
+
73
+ self.mode = mode
74
+ self.decay_type = decay_type
75
+ self.hidden_size = hidden_size
76
+ self.expand_k = expand_k
77
+ self.expand_v = expand_v
78
+ self.num_heads = num_heads
79
+ self.num_kv_heads = num_heads if num_kv_heads is None else num_kv_heads
80
+ self.num_kv_groups = self.num_heads // self.num_kv_heads
81
+ self.key_dim = int(hidden_size * expand_k)
82
+ self.value_dim = int(hidden_size * expand_v)
83
+ self.key_dim_per_group = self.key_dim // self.num_kv_groups
84
+ self.value_dim_per_group = self.value_dim // self.num_kv_groups
85
+ self.head_k_dim = self.key_dim // self.num_heads
86
+ self.head_v_dim = self.value_dim // self.num_heads
87
+ self.topk = topk
88
+ self.use_output_gate = use_output_gate
89
+ self.use_rope = use_rope
90
+ self.gate_logit_normalizer = gate_logit_normalizer
91
+ self.use_norm = use_norm
92
+ self.scale = scale
93
+ self.rope_theta = 10000.
94
+
95
+ ## For Router Design
96
+ self.bias_rmm = bias_rmm # For no gumbel router with bias
97
+ self.add_gumbel_noise = add_gumbel_noise # For no gumbel router with bias
98
+ self.router_score = router_score
99
+ self.router_type = router_type
100
+
101
+ if num_slots is None:
102
+ num_slots = self.head_k_dim
103
+ self.num_slots = num_slots
104
+
105
+ self.layer_idx = layer_idx
106
+
107
+ if layer_idx is None:
108
+ warnings.warn(
109
+ f"Instantiating {self.__class__.__name__} without passing `layer_idx` is not recommended and will "
110
+ "to errors during the forward call, if caching is used. Please make sure to provide a `layer_idx` "
111
+ "when creating this class."
112
+ )
113
+
114
+ self.register_module('feature_map', None)
115
+ if feature_map == 'swish':
116
+ self.feature_map = SwishFeatureMap()
117
+ elif feature_map == 'relu':
118
+ self.feature_map = ReLUFeatureMap()
119
+ elif feature_map == 't2r':
120
+ self.feature_map = T2RFeatureMap(self.head_k_dim, self.head_k_dim)
121
+ else:
122
+ raise NotImplementedError(f"Feature map `{feature_map}` is not supported now.")
123
+
124
+ ## ===== QKV Proj =====
125
+ self.q_proj = nn.Linear(self.hidden_size, self.key_dim, bias=False)
126
+ self.k_proj = nn.Linear(self.hidden_size, self.key_dim_per_group, bias=False)
127
+ self.v_proj = nn.Linear(self.hidden_size, self.value_dim_per_group, bias=False)
128
+
129
+ ## ===== Forget Gate/ Decay =====
130
+ if self.decay_type == 'Mamba2':
131
+ # Decay of Mamba2
132
+ self.a_proj = nn.Linear(self.hidden_size, self.num_heads, bias=False)
133
+ A = torch.empty(self.num_heads, dtype=torch.float32).uniform_(0, 16)
134
+ self.A_log = nn.Parameter(torch.log(A))
135
+ self.A_log._no_weight_decay = True
136
+ # hard coded for now
137
+ dt_min = 0.001
138
+ dt_max = 0.1
139
+ dt_init_floor = 1e-4
140
+ dt = torch.exp(
141
+ torch.rand(self.num_heads) * (math.log(dt_max) - math.log(dt_min))
142
+ + math.log(dt_min)
143
+ )
144
+ dt = torch.clamp(dt, min=dt_init_floor)
145
+ # Inverse of softplus: https://github.com/pytorch/pytorch/issues/72759
146
+ inv_dt = dt + torch.log(-torch.expm1(-dt))
147
+ self.dt_bias = nn.Parameter(inv_dt)
148
+ # Just to be explicit. Without this we already don't put wd on dt_bias because of the check
149
+ # name.endswith("bias") in param_grouping.py
150
+ self.dt_bias._no_weight_decay = True
151
+
152
+ elif self.decay_type == 'GLA':
153
+ self.f_proj = nn.Linear(self.hidden_size, self.num_kv_heads * self.num_slots, bias=False)
154
+
155
+ ## ===== Router =====
156
+ if self.bias_rmm:
157
+ self.r_bias = nn.Parameter(torch.empty( ( self.num_heads , self.num_slots) , dtype=torch.float32))
158
+
159
+ if self.router_type == 'lin':
160
+ self.r_proj = nn.Linear(self.hidden_size, self.num_heads * self.num_slots , bias=False)
161
+
162
+ elif self.router_type == 'mlp':
163
+ self.r_proj = nn.Sequential(
164
+ nn.Linear(self.hidden_size, self.hidden_size, bias=True),
165
+ nn.GELU(),
166
+ nn.Linear(self.hidden_size, self.num_heads * self.num_slots, bias=False) )
167
+
168
+ self.score_fn = (
169
+ (lambda x: torch.sigmoid(x))
170
+ if self.router_score == "sigmoid"
171
+ else (lambda x: torch.softmax(x, dim=-1))
172
+ )
173
+
174
+
175
+ ## ===== RoPE =====
176
+ if self.use_rope:
177
+ self.rotary = RotaryEmbedding(dim=self.head_k_dim, base=self.rope_theta) # Added for RoPE
178
+
179
+ ## ===== QK Norm =====
180
+ self.q_norm = RMSNorm(self.head_k_dim, elementwise_affine, eps=norm_eps)
181
+ self.k_norm = RMSNorm(self.head_k_dim, elementwise_affine, eps=norm_eps)
182
+
183
+ ## ===== Output Layer =====
184
+ if self.use_output_gate:
185
+ self.o_gate_proj = nn.Linear(hidden_size, self.value_dim, bias=False)
186
+ self.o_norm = FusedRMSNormGated(self.head_v_dim, eps=norm_eps)
187
+ else:
188
+ self.g_norm = RMSNorm(self.hidden_size, elementwise_affine, eps=norm_eps)
189
+
190
+ self.o_proj = nn.Linear(self.value_dim, self.hidden_size, bias=False)
191
+
192
+ def reset_parameters(self) -> None:
193
+ for module in self.children():
194
+ reset = getattr(module, "reset_parameters", None)
195
+ if callable(reset):
196
+ reset()
197
+
198
+ def forward(
199
+ self,
200
+ hidden_states: torch.Tensor,
201
+ attention_mask: Optional[torch.Tensor] = None,
202
+ past_key_values: Optional[Cache] = None,
203
+ use_cache: Optional[bool] = False,
204
+ output_attentions: Optional[bool] = False,
205
+ **kwargs: Unpack[Dict]
206
+ ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Cache]]:
207
+ if attention_mask is not None:
208
+ assert len(attention_mask.shape) == 2, (
209
+ "Expected attention_mask as a 0-1 matrix with shape [batch_size, seq_len] "
210
+ "for padding purposes (0 indicating padding). "
211
+ "Arbitrary attention masks of shape [batch_size, seq_len, seq_len] are not allowed."
212
+ )
213
+
214
+ batch_size, q_len, _ = hidden_states.shape
215
+ mode = self.mode
216
+ if (not self.training) and q_len == 1 and use_cache and past_key_values is not None and mode == 'chunk':
217
+ mode = 'fused_recurrent'
218
+ seqlen_offset, max_seqlen = 0, q_len # Added for RoPE
219
+
220
+ last_state = None
221
+ if past_key_values is not None and len(past_key_values) > self.layer_idx:
222
+ seqlen_offset = past_key_values.get_seq_length(self.layer_idx) # Added for RoPE
223
+ last_state = past_key_values[self.layer_idx]
224
+ max_seqlen = q_len + seqlen_offset # Added for RoPE
225
+
226
+ cu_seqlens = kwargs.get('cu_seqlens', None)
227
+ if attention_mask is not None:
228
+ indices, cu_seqlens, _ = get_unpad_data(attention_mask[:, -q_len:])
229
+ hidden_states = index_first_axis(rearrange(hidden_states, "b s ... -> (b s) ..."), indices).unsqueeze(0) # to deliminate the offsets of padding tokens
230
+ seqlen_offset = seqlen_offset + prepare_lens_from_mask(attention_mask) - attention_mask.shape[-1] # Added for RoPE
231
+ max_seqlen = max(q_len, q_len + _max_offset(seqlen_offset)) # Added for RoPE
232
+
233
+ q = rearrange(self.q_proj(hidden_states), '... (h d) -> ... h d', d=self.head_k_dim)
234
+ k = rearrange(self.k_proj(hidden_states), '... (h d) -> ... h d', d=self.head_k_dim)
235
+ v = rearrange(self.v_proj(hidden_states), '... (h d) -> ... h d', d=self.head_v_dim)
236
+ router = rearrange(self.r_proj(hidden_states) , '... (h m) -> ... h m', m=self.num_slots)
237
+
238
+ if self.decay_type == 'Mamba2':
239
+ # Build Mamba2 Decay
240
+ f = (- self.A_log.float().exp() * F.softplus(self.a_proj(hidden_states).float() + self.dt_bias)).unsqueeze(-1)
241
+ elif self.decay_type == 'GLA':
242
+ f = rearrange(self.f_proj(hidden_states) , '... (h m) -> ... h m', m=self.num_slots)
243
+ f = F.logsigmoid(f) / self.gate_logit_normalizer
244
+
245
+ if self.feature_map is not None:
246
+ q, k = map(lambda x: self.feature_map(x), (q, k))
247
+
248
+ # QK Norm
249
+ q, k = self.q_norm(q), self.k_norm(k)
250
+
251
+ if self.use_rope:
252
+ assert batch_size == 1, "RoPE is not supported for batch size > 1"
253
+ max_seqlen = max(max_seqlen, 8192) # Added for RoPE
254
+ q, k = self.rotary(q, k, seqlen_offset=seqlen_offset, max_seqlen=max_seqlen, cu_seqlens=cu_seqlens) #Added for RoPE
255
+
256
+ # V Feature map
257
+ v = F.silu(v)
258
+
259
+ # Build RMM Router
260
+ if self.add_gumbel_noise:
261
+ if self.training:
262
+ # bf16 exponential/log can underflow to zero and produce infinities,
263
+ # which then poison the sparse router and the GSA Triton kernels.
264
+ noise = torch.empty(
265
+ router.shape,
266
+ device=router.device,
267
+ dtype=torch.float32,
268
+ ).exponential_().clamp_min_(1e-6).log_()
269
+ router = router.float() - noise
270
+
271
+ router = torch.nan_to_num(router.float(), nan=0.0, posinf=20.0, neginf=-20.0).clamp_(-20.0, 20.0)
272
+
273
+ orig_scores = self.score_fn(router)
274
+ if self.bias_rmm:
275
+ scores = orig_scores + self.r_bias.float()
276
+ else:
277
+ scores = orig_scores
278
+
279
+ route_idx = scores.topk(self.topk, dim=-1).indices
280
+ topk_weights = torch.gather(orig_scores, dim=-1, index=route_idx)
281
+
282
+ if self.router_score == 'sigmoid':
283
+ topk_weights /= (topk_weights.sum(dim=-1, keepdim=True) + 1e-9)
284
+
285
+ s_multihot = torch.zeros_like(router).scatter_(-1, route_idx, topk_weights.to(router.dtype))
286
+
287
+ f = torch.nan_to_num(f.float(), nan=0.0, posinf=0.0, neginf=-20.0).clamp_(-20.0, 0.0)
288
+ s_multihot = torch.nan_to_num(s_multihot.float(), nan=0.0, posinf=1.0, neginf=0.0).clamp_(0.0, 1.0)
289
+ f = (f * s_multihot).to(q.dtype)
290
+ s = torch.nan_to_num((1 - f.float().exp()), nan=0.0, posinf=1.0, neginf=0.0).clamp_(0.0, 1.0).to(q.dtype)
291
+
292
+ q = torch.nan_to_num(q, nan=0.0, posinf=0.0, neginf=0.0)
293
+ k = torch.nan_to_num(k, nan=0.0, posinf=0.0, neginf=0.0)
294
+ v = torch.nan_to_num(v, nan=0.0, posinf=0.0, neginf=0.0)
295
+
296
+ recurrent_state = last_state['recurrent_state'] if last_state is not None else None
297
+ if self.num_kv_groups > 1:
298
+ k, v = map(lambda x: repeat(x, '... h d -> ... (h g) d', g=self.num_kv_groups), (k, v))
299
+
300
+ if mode == 'fused_recurrent':
301
+ o, recurrent_state = fused_recurrent_raven(
302
+ q=q,
303
+ k=k,
304
+ v=v,
305
+ s=s,
306
+ g=f,
307
+ initial_state=recurrent_state,
308
+ output_final_state=use_cache,
309
+ scale=self.scale,
310
+ cu_seqlens=cu_seqlens,
311
+ )
312
+ elif mode == 'chunk':
313
+ o, recurrent_state = chunk_raven(
314
+ q=q,
315
+ k=k,
316
+ v=v,
317
+ s=s,
318
+ g=f,
319
+ initial_state=recurrent_state,
320
+ output_final_state=use_cache,
321
+ scale=self.scale,
322
+ cu_seqlens=cu_seqlens,
323
+ )
324
+ else:
325
+ raise NotImplementedError(f"Not supported mode `{mode}`.")
326
+
327
+ if past_key_values is not None:
328
+ past_key_values.update(
329
+ recurrent_state=recurrent_state,
330
+ conv_state=None,
331
+ layer_idx=self.layer_idx,
332
+ offset=q_len
333
+ )
334
+
335
+
336
+ if self.use_output_gate:
337
+ gate_out = rearrange(self.o_gate_proj(hidden_states), '... (h d) -> ... h d', d=self.head_v_dim)
338
+ o = self.o_norm(F.silu(o), gate_out)
339
+ o = rearrange(o, '... h d -> ... (h d)')
340
+ o = self.o_proj(o)
341
+ else:
342
+ o = rearrange(o, '... h d -> ... (h d)')
343
+ o = rms_norm_linear(F.silu(o), self.g_norm.weight, self.g_norm.bias, self.o_proj.weight, self.o_proj.bias)
344
+
345
+ if attention_mask is not None:
346
+ o = pad_input(o.squeeze(0), indices, batch_size, q_len)
347
+
348
+ return o, None, past_key_values
raven/models/__init__.py ADDED
@@ -0,0 +1 @@
 
 
1
+ # -*- coding: utf-8 -*-
raven/models/__pycache__/__init__.cpython-312.pyc ADDED
Binary file (164 Bytes). View file
 
raven/models/raven/__init__.py ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # -*- coding: utf-8 -*-
2
+
3
+ from transformers import AutoConfig, AutoModel, AutoModelForCausalLM
4
+
5
+ from raven.models.raven.configuration_raven import RavenConfig
6
+ from raven.models.raven.modeling_raven import RavenForCausalLM, RavenModel
7
+
8
+ AutoConfig.register(RavenConfig.model_type, RavenConfig, exist_ok=True)
9
+ AutoModel.register(RavenConfig, RavenModel, exist_ok=True)
10
+ AutoModelForCausalLM.register(RavenConfig, RavenForCausalLM, exist_ok=True)
11
+
12
+ __all__ = ['RavenConfig', 'RavenForCausalLM', 'RavenModel']
raven/models/raven/__pycache__/__init__.cpython-312.pyc ADDED
Binary file (736 Bytes). View file
 
raven/models/raven/__pycache__/configuration_raven.cpython-312.pyc ADDED
Binary file (4.44 kB). View file
 
raven/models/raven/__pycache__/modeling_raven.cpython-312.pyc ADDED
Binary file (19.4 kB). View file
 
raven/models/raven/configuration_raven.py ADDED
@@ -0,0 +1,111 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # -*- coding: utf-8 -*-
2
+
3
+ from typing import Dict, Optional
4
+
5
+ from transformers.configuration_utils import PretrainedConfig
6
+
7
+
8
+ class RavenConfig(PretrainedConfig):
9
+
10
+ model_type = 'raven'
11
+ keys_to_ignore_at_inference = ['past_key_values']
12
+
13
+ def __init__(
14
+ self,
15
+ hidden_size: int = 2048,
16
+ gate_logit_normalizer: Optional[int] = 8,
17
+ clamp_min: Optional[float] = None,
18
+ clamp_max: Optional[float] = None,
19
+ hidden_ratio: Optional[int] = 4,
20
+ intermediate_size: Optional[int] = None,
21
+ num_hidden_layers: int = 24,
22
+ num_heads: int = 4,
23
+ num_kv_heads: Optional[int] = None,
24
+ num_slots: Optional[int] = 64,
25
+ use_short_conv: bool = False,
26
+ conv_size: int = 4,
27
+ exapnd_k: float = 1,
28
+ exapnd_v: float = 1,
29
+ feature_map: str = 'swish',
30
+ use_output_gate: bool = False,
31
+ use_norm: bool = True,
32
+ max_position_embeddings: int = 2048,
33
+ hidden_act: str = "swish",
34
+ decay_type: str = 'Mamba2',
35
+ bias_rmm: bool = False,
36
+ add_gumbel_noise: bool = True,
37
+ router_score: str = 'sigmoid',
38
+ router_type: str = 'lin',
39
+ topk = 32,
40
+ elementwise_affine: Optional[bool] = True,
41
+ norm_eps: float = 1e-6,
42
+ attn: Optional[Dict] = None,
43
+ use_cache: bool = True,
44
+ pad_token_id: Optional[int] = None,
45
+ bos_token_id: int = 1,
46
+ eos_token_id: int = 2,
47
+ initializer_range: float = 0.02,
48
+ tie_word_embeddings: bool = False,
49
+ fuse_norm: bool = True,
50
+ fuse_swiglu: bool = True,
51
+ fuse_cross_entropy: bool = True,
52
+ use_l2warp: bool = False,
53
+ vocab_size: int = 32000,
54
+ **kwargs
55
+ ):
56
+ self.hidden_size = hidden_size
57
+ self.gate_logit_normalizer = gate_logit_normalizer
58
+ self.clamp_min = clamp_min
59
+ self.clamp_max = clamp_max
60
+ self.hidden_ratio = hidden_ratio
61
+ self.intermediate_size = intermediate_size
62
+ self.num_hidden_layers = num_hidden_layers
63
+ self.num_heads = num_heads
64
+ self.num_kv_heads = num_kv_heads
65
+ self.num_slots = num_slots
66
+ self.use_short_conv = use_short_conv
67
+ self.conv_size = conv_size
68
+ self.expand_k = exapnd_k
69
+ self.expand_v = exapnd_v
70
+ self.feature_map = feature_map
71
+ self.use_output_gate = use_output_gate
72
+ self.use_norm = use_norm
73
+ self.max_position_embeddings = max_position_embeddings
74
+ self.hidden_act = hidden_act
75
+ self.elementwise_affine = elementwise_affine
76
+ self.norm_eps = norm_eps
77
+ self.attn = attn
78
+ self.use_cache = use_cache
79
+ self.initializer_range = initializer_range
80
+ self.decay_type = decay_type
81
+ self.topk = topk
82
+ self.bias_rmm = bias_rmm
83
+ self.add_gumbel_noise = add_gumbel_noise
84
+ self.router_score = router_score
85
+ self.router_type = router_type
86
+ self.fuse_norm = fuse_norm
87
+ self.fuse_swiglu = fuse_swiglu
88
+ self.fuse_cross_entropy = fuse_cross_entropy
89
+ self.use_l2warp = use_l2warp
90
+ self.vocab_size = vocab_size
91
+
92
+ if attn is not None:
93
+ if not isinstance(attn, Dict):
94
+ raise ValueError("attn must be a dictionary")
95
+ if 'layers' not in attn:
96
+ raise ValueError("Layer indices must be provided to initialize hybrid attention layers")
97
+ if 'num_heads' not in attn:
98
+ raise ValueError("Number of heads must be provided to initialize hybrid attention layers")
99
+ attn['num_kv_heads'] = attn.get('num_kv_heads', attn['num_heads'])
100
+ attn['qkv_bias'] = attn.get('qkv_bias', False)
101
+ attn['window_size'] = attn.get('window_size', None)
102
+ attn['rope_theta'] = attn.get('rope_theta', 10000.)
103
+ attn['use_rope'] = attn.get('use_rope', True)
104
+
105
+ super().__init__(
106
+ pad_token_id=pad_token_id,
107
+ bos_token_id=bos_token_id,
108
+ eos_token_id=eos_token_id,
109
+ tie_word_embeddings=tie_word_embeddings,
110
+ **kwargs,
111
+ )
raven/models/raven/modeling_raven.py ADDED
@@ -0,0 +1,429 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # -*- coding: utf-8 -*-
2
+
3
+ from __future__ import annotations
4
+
5
+ import math
6
+ import warnings
7
+ from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union
8
+
9
+ import torch
10
+ import torch.nn as nn
11
+ import torch.utils.checkpoint
12
+ from transformers.generation import GenerationMixin
13
+ from transformers.modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast
14
+ from transformers.modeling_utils import PreTrainedModel
15
+ from transformers.utils import logging
16
+ from transformers.utils.deprecation import deprecate_kwarg
17
+
18
+ from fla.layers.attn import Attention
19
+ from raven.layers.raven import RavenAttention
20
+ from raven.models.raven.configuration_raven import RavenConfig
21
+ from fla.models.utils import Cache
22
+ from fla.modules import FusedCrossEntropyLoss, FusedLinearCrossEntropyLoss
23
+ from fla.modules import GatedMLP as GSAMLP
24
+ from fla.modules import RMSNorm
25
+ from fla.modules.l2warp import l2_warp
26
+
27
+ if TYPE_CHECKING:
28
+ from transformers.processing_utils import Unpack
29
+
30
+ logger = logging.get_logger(__name__)
31
+
32
+
33
+ class RavenBlock(nn.Module):
34
+ def __init__(self, config: RavenConfig, layer_idx: int):
35
+ super().__init__()
36
+
37
+ self.config = config
38
+ self.layer_idx = layer_idx
39
+
40
+ self.attn_norm = (RMSNorm if config.fuse_norm else nn.RMSNorm)(config.hidden_size, eps=config.norm_eps)
41
+ if config.attn is not None and layer_idx in config.attn['layers']:
42
+ self.attn = Attention(
43
+ hidden_size=config.hidden_size,
44
+ num_heads=config.attn['num_heads'],
45
+ num_kv_heads=config.attn['num_kv_heads'],
46
+ qkv_bias=config.attn['qkv_bias'],
47
+ window_size=config.attn['window_size'],
48
+ use_rope=config.attn['use_rope'],
49
+ rope_theta=config.attn['rope_theta'],
50
+ max_position_embeddings=config.max_position_embeddings,
51
+ layer_idx=layer_idx
52
+ )
53
+ else:
54
+ self.attn = RavenAttention(
55
+ hidden_size=config.hidden_size,
56
+ expand_k=config.expand_k,
57
+ expand_v=config.expand_v,
58
+ num_heads=config.num_heads,
59
+ num_kv_heads=config.num_kv_heads,
60
+ num_slots=config.num_slots,
61
+ use_short_conv=config.use_short_conv,
62
+ conv_size=config.conv_size,
63
+ feature_map=config.feature_map,
64
+ use_output_gate=config.use_output_gate,
65
+ use_norm=config.use_norm,
66
+ gate_fn=config.hidden_act,
67
+ topk=config.topk,
68
+ bias_rmm=config.bias_rmm,
69
+ add_gumbel_noise=config.add_gumbel_noise,
70
+ router_score=config.router_score,
71
+ router_type=config.router_type,
72
+ decay_type = config.decay_type,
73
+ gate_logit_normalizer=config.gate_logit_normalizer,
74
+ elementwise_affine=config.elementwise_affine,
75
+ norm_eps=config.norm_eps,
76
+ fuse_norm=config.fuse_norm,
77
+ layer_idx=layer_idx
78
+ )
79
+ self.mlp_norm = (RMSNorm if config.fuse_norm else nn.RMSNorm)(config.hidden_size, eps=config.norm_eps)
80
+ self.mlp = GSAMLP(
81
+ hidden_size=config.hidden_size,
82
+ hidden_ratio=config.hidden_ratio,
83
+ intermediate_size=config.intermediate_size,
84
+ hidden_act=config.hidden_act,
85
+ fuse_swiglu=config.fuse_swiglu
86
+ )
87
+
88
+ def forward(
89
+ self,
90
+ hidden_states: torch.Tensor,
91
+ attention_mask: Optional[torch.Tensor] = None,
92
+ past_key_values: Optional[Union[Cache, List[torch.FloatTensor]]] = None,
93
+ use_cache: Optional[bool] = False,
94
+ output_attentions: Optional[bool] = False,
95
+ **kwargs: Unpack[Dict]
96
+ ) -> Tuple[torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]]:
97
+ residual = hidden_states
98
+ hidden_states = self.attn_norm(hidden_states)
99
+ hidden_states, attentions, past_key_values = self.attn(
100
+ hidden_states=hidden_states,
101
+ attention_mask=attention_mask,
102
+ past_key_values=past_key_values,
103
+ use_cache=use_cache,
104
+ output_attentions=output_attentions,
105
+ **kwargs
106
+ )
107
+ if self.config.fuse_norm:
108
+ hidden_states, residual = self.mlp_norm(hidden_states, residual, True)
109
+ else:
110
+ hidden_states = residual + hidden_states
111
+ residual = hidden_states
112
+ hidden_states = self.mlp_norm(hidden_states)
113
+ hidden_states = self.mlp(hidden_states, **kwargs)
114
+ hidden_states = residual + hidden_states
115
+
116
+ outputs = (hidden_states, attentions, past_key_values)
117
+
118
+ return outputs
119
+
120
+
121
+ class RavenPreTrainedModel(PreTrainedModel):
122
+
123
+ config_class = RavenConfig
124
+ base_model_prefix = 'model'
125
+ supports_gradient_checkpointing = True
126
+ _no_split_modules = ['RavenBlock']
127
+ _supports_cache_class = True
128
+
129
+ def __init__(self, *inputs, **kwargs):
130
+ super().__init__(*inputs, **kwargs)
131
+
132
+ def _init_weights(
133
+ self,
134
+ module: nn.Module,
135
+ prenorm_residual_strategy: Optional[str] = None,
136
+ num_residuals_per_layer: int = 2,
137
+ ):
138
+ if isinstance(module, (nn.Linear, nn.Conv1d)):
139
+ # Slightly different from the TF version which uses truncated_normal for initialization
140
+ # cf https://github.com/pytorch/pytorch/pull/5617
141
+ nn.init.normal_(module.weight, mean=0.0, std=self.config.initializer_range)
142
+ if module.bias is not None:
143
+ nn.init.zeros_(module.bias)
144
+ elif isinstance(module, nn.Embedding):
145
+ nn.init.normal_(module.weight, mean=0.0, std=self.config.initializer_range)
146
+ elif hasattr(module, 'reset_parameters'):
147
+ module.reset_parameters()
148
+
149
+ if prenorm_residual_strategy is not None:
150
+ # Reinitialize selected weights subject to the OpenAI GPT-2 Paper Scheme:
151
+ # > A modified initialization which accounts for the accumulation on the residual path with model depth. Scale
152
+ # > the weights of residual layers at initialization by a factor of 1/√N where N is the # of residual layers.
153
+ # > -- GPT-2 :: https://openai.com/blog/better-language-models/
154
+ #
155
+ # Reference (Megatron-LM): https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/model/gpt_model.py
156
+ p = None
157
+ if hasattr(module, 'o_proj'):
158
+ p = module.o_proj.weight
159
+ elif hasattr(module, 'down_proj'):
160
+ p = module.down_proj.weight
161
+ if p is not None:
162
+ # Special Scaled Initialization --> There are 2 Layer Norms per Transformer Block
163
+ # Following Pytorch init, except scale by 1/sqrt(2 * n_layer)
164
+ # We need to reinit p since this code could be called multiple times
165
+ # Having just p *= scale would repeatedly scale it down
166
+ if prenorm_residual_strategy == 'rescale':
167
+ nn.init.kaiming_uniform_(p, a=math.sqrt(5))
168
+ with torch.no_grad():
169
+ p /= math.sqrt(num_residuals_per_layer * self.config.num_hidden_layers)
170
+ elif prenorm_residual_strategy == 'zero':
171
+ nn.init.zeros_(p)
172
+ else:
173
+ raise ValueError(f"Invalid prenorm_residual_strategy: {prenorm_residual_strategy}")
174
+
175
+
176
+ class RavenModel(RavenPreTrainedModel):
177
+
178
+ def __init__(self, config: RavenConfig):
179
+ super().__init__(config)
180
+ self.padding_idx = config.pad_token_id
181
+ self.vocab_size = config.vocab_size
182
+
183
+ self.embeddings = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx)
184
+ self.layers = nn.ModuleList([RavenBlock(config, layer_idx) for layer_idx in range(config.num_hidden_layers)])
185
+ self.norm = (RMSNorm if config.fuse_norm else nn.RMSNorm)(config.hidden_size, eps=config.norm_eps)
186
+
187
+ self.gradient_checkpointing = False
188
+
189
+ self.post_init()
190
+
191
+ def get_input_embeddings(self):
192
+ return self.embeddings
193
+
194
+ def set_input_embeddings(self, value):
195
+ self.embeddings = value
196
+
197
+ def forward(
198
+ self,
199
+ input_ids: Optional[torch.LongTensor] = None,
200
+ attention_mask: Optional[torch.Tensor] = None, # noqa
201
+ inputs_embeds: Optional[torch.FloatTensor] = None,
202
+ past_key_values: Optional[Union[Cache, List[torch.FloatTensor]]] = None,
203
+ use_cache: Optional[bool] = None,
204
+ output_attentions: Optional[bool] = None,
205
+ output_hidden_states: Optional[bool] = None,
206
+ return_dict: Optional[bool] = None,
207
+ **kwargs: Unpack[Dict]
208
+ ) -> Union[Tuple, BaseModelOutputWithPast]:
209
+ if output_attentions:
210
+ warnings.warn("`RavenModel` does not `output_attentions` now, setting it to `False`.")
211
+ output_attentions = False
212
+ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
213
+ output_hidden_states = output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
214
+ use_cache = use_cache if use_cache is not None else (self.config.use_cache if not self.training else False)
215
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
216
+
217
+ # retrieve input_ids and inputs_embeds
218
+ if input_ids is not None and inputs_embeds is not None:
219
+ raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")
220
+ if input_ids is None and inputs_embeds is None:
221
+ raise ValueError("You have to specify either input_ids or inputs_embeds")
222
+
223
+ if inputs_embeds is None:
224
+ inputs_embeds = self.embeddings(input_ids)
225
+ hidden_states = inputs_embeds
226
+
227
+ if use_cache and not isinstance(past_key_values, Cache):
228
+ past_key_values = Cache.from_legacy_cache(past_key_values)
229
+
230
+ if self.gradient_checkpointing and self.training and use_cache:
231
+ logger.warning_once("`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...")
232
+ use_cache = False
233
+
234
+ all_hidden_states = () if output_hidden_states else None
235
+ all_attns = () if output_attentions else None
236
+ for layer in self.layers:
237
+ if output_hidden_states:
238
+ all_hidden_states += (hidden_states,)
239
+
240
+ if self.gradient_checkpointing and self.training:
241
+ hidden_states, attentions, past_key_values = self._gradient_checkpointing_func(
242
+ layer.__call__,
243
+ hidden_states,
244
+ attention_mask,
245
+ past_key_values,
246
+ use_cache,
247
+ output_attentions,
248
+ **kwargs
249
+ )
250
+ else:
251
+ hidden_states, attentions, past_key_values = layer(
252
+ hidden_states,
253
+ attention_mask=attention_mask,
254
+ past_key_values=past_key_values,
255
+ use_cache=use_cache,
256
+ output_attentions=output_attentions,
257
+ **kwargs
258
+ )
259
+
260
+ if output_attentions:
261
+ all_attns += (attentions,)
262
+
263
+ hidden_states = self.norm(hidden_states)
264
+
265
+ # add hidden states from the last decoder layer
266
+ if output_hidden_states:
267
+ all_hidden_states += (hidden_states,)
268
+
269
+ if not return_dict:
270
+ return tuple(i for i in [hidden_states, past_key_values, all_hidden_states, all_attns] if i is not None)
271
+ return BaseModelOutputWithPast(
272
+ last_hidden_state=hidden_states,
273
+ past_key_values=past_key_values,
274
+ hidden_states=all_hidden_states,
275
+ attentions=all_attns
276
+ )
277
+
278
+
279
+ class RavenForCausalLM(RavenPreTrainedModel, GenerationMixin):
280
+
281
+ _tied_weights_keys = ["lm_head.weight"]
282
+
283
+ def __init__(self, config):
284
+
285
+ super().__init__(config)
286
+ self.model = RavenModel(config)
287
+ self.vocab_size = config.vocab_size
288
+ self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
289
+ self.criterion = None
290
+
291
+ # Initialize weights and apply final processing
292
+ self.post_init()
293
+
294
+ def get_input_embeddings(self):
295
+ return self.model.embeddings
296
+
297
+ def set_input_embeddings(self, value):
298
+ self.model.embeddings = value
299
+
300
+ def get_output_embeddings(self):
301
+ return self.lm_head
302
+
303
+ def set_output_embeddings(self, new_embeddings):
304
+ self.lm_head = new_embeddings
305
+
306
+ def set_decoder(self, decoder):
307
+ self.model = decoder
308
+
309
+ def get_decoder(self):
310
+ return self.model
311
+
312
+ def generate(self, *args, **kwargs):
313
+ try:
314
+ return super().generate(*args, **kwargs)
315
+ except AttributeError as exception:
316
+ if 'past_key_values' in str(exception):
317
+ raise AttributeError(
318
+ f"You tried to call `generate` with a decoding strategy that manipulates `past_key_values`, "
319
+ f"which is not supported for {self.__class__.__name__}. "
320
+ f"Try another generation strategy instead. "
321
+ f"For the available generation strategies, check this doc: "
322
+ f"https://huggingface.co/docs/transformers/en/generation_strategies#decoding-strategies"
323
+ )
324
+ else:
325
+ raise exception
326
+
327
+ @deprecate_kwarg("num_logits_to_keep", version="4.50", new_name="logits_to_keep")
328
+ def prepare_inputs_for_generation(
329
+ self,
330
+ input_ids: torch.LongTensor = None,
331
+ past_key_values: Optional[Union[Cache, List[torch.FloatTensor]]] = None,
332
+ attention_mask: Optional[torch.Tensor] = None,
333
+ inputs_embeds: Optional[torch.Tensor] = None,
334
+ use_cache: bool = True,
335
+ logits_to_keep: Optional[int] = None,
336
+ **kwargs
337
+ ):
338
+ # only last token for `inputs_ids` if the `past_key_values` is not empty.
339
+ if past_key_values is not None and len(past_key_values) > 0:
340
+ input_ids = input_ids[:, -1:]
341
+ # if `inputs_embeds` are passed, we only want to use them in the 1st generation step
342
+ if inputs_embeds is not None and len(past_key_values) == 0:
343
+ model_inputs = {'inputs_embeds': inputs_embeds}
344
+ else:
345
+ # The `contiguous()` here is necessary to have a static stride during decoding. torchdynamo otherwise
346
+ # recompiles graphs as the stride of the inputs is a guard.
347
+ # Ref: https://github.com/huggingface/transformers/pull/29114
348
+ # TODO: use `next_tokens` directly instead.
349
+ model_inputs = {'input_ids': input_ids.contiguous()}
350
+
351
+ if logits_to_keep is not None:
352
+ model_inputs['logits_to_keep'] = logits_to_keep
353
+
354
+ model_inputs.update({
355
+ 'past_key_values': past_key_values,
356
+ 'use_cache': use_cache,
357
+ 'attention_mask': attention_mask,
358
+ })
359
+ return model_inputs
360
+
361
+ @deprecate_kwarg("num_logits_to_keep", version="4.50", new_name="logits_to_keep")
362
+ def forward(
363
+ self,
364
+ input_ids: torch.LongTensor = None,
365
+ attention_mask: Optional[torch.Tensor] = None,
366
+ inputs_embeds: Optional[torch.Tensor] = None,
367
+ past_key_values: Optional[Union[Cache, List[torch.FloatTensor]]] = None,
368
+ labels: Optional[torch.LongTensor] = None,
369
+ use_cache: Optional[bool] = None,
370
+ output_attentions: Optional[bool] = None,
371
+ output_hidden_states: Optional[bool] = None,
372
+ return_dict: Optional[bool] = None,
373
+ logits_to_keep: Optional[int] = 0,
374
+ **kwargs: Unpack[Dict]
375
+ ) -> Union[Tuple, CausalLMOutputWithPast]:
376
+ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
377
+ output_hidden_states = (
378
+ output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
379
+ )
380
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
381
+
382
+ outputs = self.model(
383
+ input_ids=input_ids,
384
+ attention_mask=attention_mask,
385
+ inputs_embeds=inputs_embeds,
386
+ past_key_values=past_key_values,
387
+ use_cache=use_cache,
388
+ output_attentions=output_attentions,
389
+ output_hidden_states=output_hidden_states,
390
+ return_dict=return_dict,
391
+ **kwargs
392
+ )
393
+
394
+ hidden_states = outputs[0]
395
+ fuse_linear_and_cross_entropy = self.config.fuse_cross_entropy and self.training and labels is not None
396
+
397
+ loss, logits = None, None
398
+ if not fuse_linear_and_cross_entropy or labels is None:
399
+ logits = self.lm_head(hidden_states if logits_to_keep is None else hidden_states[:, -logits_to_keep:])
400
+ if labels is not None:
401
+ if getattr(self, 'criterion', None) is None:
402
+ if fuse_linear_and_cross_entropy:
403
+ criterion = FusedLinearCrossEntropyLoss(use_l2warp=self.config.use_l2warp)
404
+ elif self.config.fuse_cross_entropy:
405
+ criterion = FusedCrossEntropyLoss(inplace_backward=True)
406
+ else:
407
+ criterion = nn.CrossEntropyLoss()
408
+ else:
409
+ criterion = self.criterion
410
+ # Enable model parallelism
411
+ labels = labels.to(hidden_states.device)
412
+ labels = torch.cat((labels[..., 1:], torch.full_like(labels[:, :1], criterion.ignore_index)), 1)
413
+ if fuse_linear_and_cross_entropy:
414
+ loss = criterion(hidden_states, labels, self.lm_head.weight, self.lm_head.bias)
415
+ else:
416
+ loss = criterion(logits.view(labels.numel(), -1), labels.view(-1))
417
+ loss = l2_warp(loss, logits) if self.config.use_l2warp else loss
418
+
419
+ if not return_dict:
420
+ output = (logits,) + outputs[1:]
421
+ return (loss,) + output if loss is not None else output
422
+
423
+ return CausalLMOutputWithPast(
424
+ loss=loss,
425
+ logits=logits,
426
+ past_key_values=outputs.past_key_values,
427
+ hidden_states=outputs.hidden_states,
428
+ attentions=outputs.attentions,
429
+ )