Create configuration_pebble.py

configuration_pebble.py

@@ -0,0 +1,41 @@
+from transformers import PretrainedConfig
+
+class PebbleConfig(PretrainedConfig):
+    model_type = "pebble_10m"
+
+    def __init__(
+        self,
+        vocab_size=2048,
+        hidden_size=384,
+        intermediate_size=1536,
+        num_hidden_layers=8,
+        num_attention_heads=6,
+        block_pattern="mmma|mmma",
+        hybrid_ratio="3:1 mamba2:attention",
+        max_position_embeddings=512,
+        rms_norm_eps=1e-6,
+        tie_word_embeddings=True,
+        mamba2=None,
+        attention=None,
+        **kwargs,
+    ):
+        self.vocab_size = vocab_size
+        self.hidden_size = hidden_size
+        self.intermediate_size = intermediate_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+        self.block_pattern = block_pattern
+        self.hybrid_ratio = hybrid_ratio
+        self.max_position_embeddings = max_position_embeddings
+        self.rms_norm_eps = rms_norm_eps
+        self.tie_word_embeddings = tie_word_embeddings
+        
+        # Default dictionaries if not provided in config.json
+        self.mamba2 = mamba2 or {
+            "d_state": 128, "d_conv": 4, "expand": 2, 
+            "headdim": 96, "use_mem_eff_path": True
+        }
+        self.attention = attention or {
+            "rope_theta": 10000.0, "is_causal": True
+        }
+        super().__init__(**kwargs)