Update bdh.py

bdh.py

@@ -8,18 +8,18 @@ import torch.nn.functional as F
 
 @dataclasses.dataclass
 class BDHConfig:
-    n_layer: int = 6
-    n_embd: int = 256
+    n_layer: int = 32
+    n_embd: int = 4096
     dropout: float = 0.1
-    n_head: int = 4
-    mlp_internal_dim_multiplier: int = 4
+    n_head: int = 32
+    mlp_internal_dim_multiplier: int = 1
     vocab_size: int = 256
     use_alibi: bool = True
     use_l1_norm: bool = True
     relu_threshold: float = 0.0
     rotary_embedding: str = "rope"
     rope_theta: float = 65536.0
-    use_plasticity: bool = False
+    use_plasticity: bool = True
     plasticity_lr: float = 0.01
     consolidation_rate: float = 0.01
     forget_rate: float = 0.1