Update README.md
Browse files
README.md
CHANGED
|
@@ -12,8 +12,14 @@ from transformers import Zamba2Config, Zamba2ForCausalLM
|
|
| 12 |
|
| 13 |
# === Step 1: Define tiny model config ===
|
| 14 |
config = Zamba2Config(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 15 |
d_model=16,
|
| 16 |
-
n_layer=46, # Match number of Mamba/Hybrid blocks
|
| 17 |
d_state=32,
|
| 18 |
expand=2,
|
| 19 |
conv_kernel=3,
|
|
|
|
| 12 |
|
| 13 |
# === Step 1: Define tiny model config ===
|
| 14 |
config = Zamba2Config(
|
| 15 |
+
num_hidden_layers=4,
|
| 16 |
+
layers_block_type=[
|
| 17 |
+
"mamba",
|
| 18 |
+
"mamba",
|
| 19 |
+
"hybrid",
|
| 20 |
+
"mamba",
|
| 21 |
+
],
|
| 22 |
d_model=16,
|
|
|
|
| 23 |
d_state=32,
|
| 24 |
expand=2,
|
| 25 |
conv_kernel=3,
|