Update README.md
Browse files
README.md
CHANGED
|
@@ -11,9 +11,11 @@ tags: []
|
|
| 11 |
import torch
|
| 12 |
from transformers import HeliumConfig, HeliumForCausalLM, AutoTokenizer
|
| 13 |
|
|
|
|
| 14 |
model_id = 'kyutai/helium-1-preview-2b'
|
| 15 |
config = HeliumConfig.from_pretrained(
|
| 16 |
model_id,
|
|
|
|
| 17 |
hidden_size=32,
|
| 18 |
intermediate_size=64,
|
| 19 |
num_attention_heads=4,
|
|
|
|
| 11 |
import torch
|
| 12 |
from transformers import HeliumConfig, HeliumForCausalLM, AutoTokenizer
|
| 13 |
|
| 14 |
+
|
| 15 |
model_id = 'kyutai/helium-1-preview-2b'
|
| 16 |
config = HeliumConfig.from_pretrained(
|
| 17 |
model_id,
|
| 18 |
+
head_dim=8,
|
| 19 |
hidden_size=32,
|
| 20 |
intermediate_size=64,
|
| 21 |
num_attention_heads=4,
|