Update README.md
Browse files
README.md
CHANGED
|
@@ -1,19 +0,0 @@
|
|
| 1 |
-
# koai-stack-0_1B (Transformers)
|
| 2 |
-
|
| 3 |
-
## Purpose
|
| 4 |
-
- Daily conversation model
|
| 5 |
-
- Non-coding assistant behavior
|
| 6 |
-
|
| 7 |
-
## Architecture
|
| 8 |
-
- Custom architecture: AxaiForCausalLM
|
| 9 |
-
|
| 10 |
-
- hidden_size: 768
|
| 11 |
-
- intermediate_size: 3072
|
| 12 |
-
- layers: 8
|
| 13 |
-
- heads: 12
|
| 14 |
-
- kv_heads: 6
|
| 15 |
-
- max_seq_len: 128
|
| 16 |
-
- rope_theta: 10000
|
| 17 |
-
- qk_norm: True
|
| 18 |
-
- local_mixer: True
|
| 19 |
-
- neftune_alpha: 0.0
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|