v0.2
Browse files- README.md +10 -9
- config.json +6 -4
- model.safetensors +2 -2
README.md
CHANGED
|
@@ -18,7 +18,7 @@ model-index:
|
|
| 18 |
|
| 19 |
# Csermely (MLX)
|
| 20 |
|
| 21 |
-
**MLX version of Csermely** — a
|
| 22 |
|
| 23 |
This is the native MLX bfloat16 checkpoint. For the HuggingFace transformers version, see [emese-tech/csermely](https://huggingface.co/emese-tech/csermely).
|
| 24 |
|
|
@@ -26,24 +26,25 @@ This is the native MLX bfloat16 checkpoint. For the HuggingFace transformers ver
|
|
| 26 |
|
| 27 |
| | |
|
| 28 |
|---|---|
|
| 29 |
-
| **
|
|
|
|
| 30 |
| **Architecture** | LLaMA-style (decoder-only transformer) |
|
| 31 |
-
| **Context length** |
|
| 32 |
-
| **Training context** |
|
| 33 |
| **Precision** | bfloat16 |
|
| 34 |
| **Vocabulary** | 32,000 (SentencePiece Unigram, Hungarian) |
|
| 35 |
-
| **Training data** | ~
|
| 36 |
| **Framework** | MLX (Apple Silicon) |
|
| 37 |
| **License** | MIT |
|
| 38 |
|
| 39 |
## Architecture
|
| 40 |
|
| 41 |
- 16 transformer layers
|
| 42 |
-
-
|
| 43 |
-
-
|
| 44 |
-
-
|
| 45 |
- RMSNorm pre-layer normalization
|
| 46 |
-
- Rotary positional embeddings (RoPE) with YaRN extension
|
| 47 |
- SwiGLU feed-forward activation
|
| 48 |
- Tied input/output embeddings
|
| 49 |
|
|
|
|
| 18 |
|
| 19 |
# Csermely (MLX)
|
| 20 |
|
| 21 |
+
**MLX version of Csermely** — a 190M parameter Hungarian language model optimized for Apple Silicon. Part of the [Emese](https://emese.tech) model family.
|
| 22 |
|
| 23 |
This is the native MLX bfloat16 checkpoint. For the HuggingFace transformers version, see [emese-tech/csermely](https://huggingface.co/emese-tech/csermely).
|
| 24 |
|
|
|
|
| 26 |
|
| 27 |
| | |
|
| 28 |
|---|---|
|
| 29 |
+
| **Version** | 0.2 |
|
| 30 |
+
| **Parameters** | 190.2M |
|
| 31 |
| **Architecture** | LLaMA-style (decoder-only transformer) |
|
| 32 |
+
| **Context length** | 4,096 tokens (YaRN RoPE, 4× factor) |
|
| 33 |
+
| **Training context** | 1,024 tokens |
|
| 34 |
| **Precision** | bfloat16 |
|
| 35 |
| **Vocabulary** | 32,000 (SentencePiece Unigram, Hungarian) |
|
| 36 |
+
| **Training data** | ~2B tokens of Hungarian text |
|
| 37 |
| **Framework** | MLX (Apple Silicon) |
|
| 38 |
| **License** | MIT |
|
| 39 |
|
| 40 |
## Architecture
|
| 41 |
|
| 42 |
- 16 transformer layers
|
| 43 |
+
- 896 hidden dimension
|
| 44 |
+
- 14 attention heads
|
| 45 |
+
- 2560 FFN intermediate size
|
| 46 |
- RMSNorm pre-layer normalization
|
| 47 |
+
- Rotary positional embeddings (RoPE) with YaRN extension (4× factor, base 1024)
|
| 48 |
- SwiGLU feed-forward activation
|
| 49 |
- Tied input/output embeddings
|
| 50 |
|
config.json
CHANGED
|
@@ -1,14 +1,16 @@
|
|
| 1 |
{
|
| 2 |
"model_type": "llama",
|
|
|
|
| 3 |
"vocab_size": 32000,
|
| 4 |
-
"d_model":
|
| 5 |
"n_layers": 16,
|
| 6 |
-
"n_heads":
|
| 7 |
-
"d_ff":
|
| 8 |
-
"max_seq_len":
|
| 9 |
"rope_theta": 10000.0,
|
| 10 |
"rope_yarn": true,
|
| 11 |
"rope_yarn_scale": 4.0,
|
|
|
|
| 12 |
"rope_yarn_alpha": 1.0,
|
| 13 |
"rope_yarn_beta": 32.0,
|
| 14 |
"rms_norm_eps": 1e-5,
|
|
|
|
| 1 |
{
|
| 2 |
"model_type": "llama",
|
| 3 |
+
"model_version": "0.2",
|
| 4 |
"vocab_size": 32000,
|
| 5 |
+
"d_model": 896,
|
| 6 |
"n_layers": 16,
|
| 7 |
+
"n_heads": 14,
|
| 8 |
+
"d_ff": 2560,
|
| 9 |
+
"max_seq_len": 4096,
|
| 10 |
"rope_theta": 10000.0,
|
| 11 |
"rope_yarn": true,
|
| 12 |
"rope_yarn_scale": 4.0,
|
| 13 |
+
"rope_yarn_original_len": 1024,
|
| 14 |
"rope_yarn_alpha": 1.0,
|
| 15 |
"rope_yarn_beta": 32.0,
|
| 16 |
"rms_norm_eps": 1e-5,
|
model.safetensors
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
-
size
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e136f0e7f305d126af9c1ab74f463689f3db36eae8b7623b6c11e9641e0f7de4
|
| 3 |
+
size 380378766
|