Initial release of the Slopiest-49M Base Model
Browse files
model.py
CHANGED
|
@@ -16,6 +16,11 @@ class SlopiestConfig(PretrainedConfig):
|
|
| 16 |
self.layer_size = layer_size
|
| 17 |
self.block_size = block_size
|
| 18 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 19 |
class SlopiestForCausalLM(PreTrainedModel, GenerationMixin):
|
| 20 |
config_class = SlopiestConfig
|
| 21 |
|
|
|
|
| 16 |
self.layer_size = layer_size
|
| 17 |
self.block_size = block_size
|
| 18 |
|
| 19 |
+
self.hidden_size = embeddings_size
|
| 20 |
+
self.num_hidden_layers = layer_size
|
| 21 |
+
self.num_attention_heads = head_size
|
| 22 |
+
self.max_position_embeddings = block_size
|
| 23 |
+
|
| 24 |
class SlopiestForCausalLM(PreTrainedModel, GenerationMixin):
|
| 25 |
config_class = SlopiestConfig
|
| 26 |
|