Upload configs/model/small-50M.yaml with huggingface_hub
Browse files- configs/model/small-50M.yaml +12 -0
configs/model/small-50M.yaml
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
name: small-50M
|
| 2 |
+
type: ddit
|
| 3 |
+
hidden_size: 640
|
| 4 |
+
cond_dim: 128
|
| 5 |
+
length: 512
|
| 6 |
+
n_blocks: 10
|
| 7 |
+
n_heads: 10
|
| 8 |
+
scale_by_sigma: True
|
| 9 |
+
dropout: 0.1
|
| 10 |
+
tie_word_embeddings: True
|
| 11 |
+
adaln: False
|
| 12 |
+
attn_backend: flash_attn
|