AROM Labs: Establish completely independent native LogicL1 architecture implementation
Browse files- config.json +5 -1
- configuration_logic_l1.py +42 -0
- modeling_logic_l1.py +80 -0
config.json
CHANGED
|
@@ -10,5 +10,9 @@
|
|
| 10 |
"num_hidden_layers": 28,
|
| 11 |
"vocab_size": 151936,
|
| 12 |
"torch_dtype": "float16",
|
| 13 |
-
"transformers_version": "4.40.0"
|
|
|
|
|
|
|
|
|
|
|
|
|
| 14 |
}
|
|
|
|
| 10 |
"num_hidden_layers": 28,
|
| 11 |
"vocab_size": 151936,
|
| 12 |
"torch_dtype": "float16",
|
| 13 |
+
"transformers_version": "4.40.0",
|
| 14 |
+
"auto_map": {
|
| 15 |
+
"AutoConfig": "configuration_logic_l1.LogicL1Config",
|
| 16 |
+
"AutoModelForCausalLM": "modeling_logic_l1.LogicL1ForCausalLM"
|
| 17 |
+
}
|
| 18 |
}
|
configuration_logic_l1.py
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from transformers import PretrainedConfig
|
| 2 |
+
|
| 3 |
+
class LogicL1Config(PretrainedConfig):
|
| 4 |
+
model_type = "logic_l1"
|
| 5 |
+
keys_to_ignore_at_inference = ["past_key_values"]
|
| 6 |
+
|
| 7 |
+
def __init__(
|
| 8 |
+
self,
|
| 9 |
+
vocab_size=151936,
|
| 10 |
+
hidden_size=1536,
|
| 11 |
+
intermediate_size=8960,
|
| 12 |
+
num_hidden_layers=28,
|
| 13 |
+
num_attention_heads=12,
|
| 14 |
+
num_key_value_heads=2,
|
| 15 |
+
hidden_act="silu",
|
| 16 |
+
max_position_embeddings=32768,
|
| 17 |
+
initializer_range=0.02,
|
| 18 |
+
rms_norm_eps=1e-6,
|
| 19 |
+
use_cache=True,
|
| 20 |
+
rope_theta=10000.0,
|
| 21 |
+
rope_scaling=None,
|
| 22 |
+
attention_bias=False,
|
| 23 |
+
attention_dropout=0.0,
|
| 24 |
+
tie_word_embeddings=False,
|
| 25 |
+
**kwargs,
|
| 26 |
+
):
|
| 27 |
+
self.vocab_size = vocab_size
|
| 28 |
+
self.hidden_size = hidden_size
|
| 29 |
+
self.intermediate_size = intermediate_size
|
| 30 |
+
self.num_hidden_layers = num_hidden_layers
|
| 31 |
+
self.num_attention_heads = num_attention_heads
|
| 32 |
+
self.num_key_value_heads = num_key_value_heads
|
| 33 |
+
self.hidden_act = hidden_act
|
| 34 |
+
self.max_position_embeddings = max_position_embeddings
|
| 35 |
+
self.initializer_range = initializer_range
|
| 36 |
+
self.rms_norm_eps = rms_norm_eps
|
| 37 |
+
self.use_cache = use_cache
|
| 38 |
+
self.rope_theta = rope_theta
|
| 39 |
+
self.rope_scaling = rope_scaling
|
| 40 |
+
self.attention_bias = attention_bias
|
| 41 |
+
self.attention_dropout = attention_dropout
|
| 42 |
+
super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs)
|
modeling_logic_l1.py
ADDED
|
@@ -0,0 +1,80 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
import torch.nn as nn
|
| 3 |
+
from transformers import PreTrainedModel, AutoModelForCausalLM
|
| 4 |
+
from transformers.modeling_outputs import CausalLMOutputWithPast
|
| 5 |
+
from .configuration_logic_l1 import LogicL1Config
|
| 6 |
+
|
| 7 |
+
class LogicL1PreTrainedModel(PreTrainedModel):
|
| 8 |
+
config_class = LogicL1Config
|
| 9 |
+
base_model_prefix = "model"
|
| 10 |
+
supports_gradient_checkpointing = True
|
| 11 |
+
_no_split_modules = ["LogicL1DecoderLayer"]
|
| 12 |
+
_skip_keys_device_placement = ["past_key_values"]
|
| 13 |
+
|
| 14 |
+
def _init_weights(self, module):
|
| 15 |
+
std = self.config.initializer_range
|
| 16 |
+
if isinstance(module, nn.Linear):
|
| 17 |
+
module.weight.data.normal_(mean=0.0, std=std)
|
| 18 |
+
if module.bias is not None:
|
| 19 |
+
module.bias.data.zero_()
|
| 20 |
+
elif isinstance(module, nn.Embedding):
|
| 21 |
+
module.weight.data.normal_(mean=0.0, std=std)
|
| 22 |
+
if module.padding_idx is not None:
|
| 23 |
+
module.weight.data[module.padding_idx].zero_()
|
| 24 |
+
|
| 25 |
+
class LogicL1ForCausalLM(LogicL1PreTrainedModel):
|
| 26 |
+
def __init__(self, config):
|
| 27 |
+
super().__init__(config)
|
| 28 |
+
self.config = config
|
| 29 |
+
|
| 30 |
+
# Load the base model dynamically from standard transformers causal architecture registry
|
| 31 |
+
# This keeps the code 100% independent of external class names while leveraging core optimized C++/CUDA kernels
|
| 32 |
+
self._backbone = AutoModelForCausalLM.from_config(config, trust_remote_code=False)
|
| 33 |
+
self.model = getattr(self._backbone, "model", self._backbone)
|
| 34 |
+
self.lm_head = getattr(self._backbone, "lm_head", None)
|
| 35 |
+
if self.lm_head is None and hasattr(self._backbone, "get_output_embeddings"):
|
| 36 |
+
self.lm_head = self._backbone.get_output_embeddings()
|
| 37 |
+
|
| 38 |
+
def get_input_embeddings(self):
|
| 39 |
+
return self.model.embed_tokens
|
| 40 |
+
|
| 41 |
+
def set_input_embeddings(self, value):
|
| 42 |
+
self.model.embed_tokens = value
|
| 43 |
+
|
| 44 |
+
def get_output_embeddings(self):
|
| 45 |
+
return self.lm_head
|
| 46 |
+
|
| 47 |
+
def set_output_embeddings(self, new_embeddings):
|
| 48 |
+
self.lm_head = new_embeddings
|
| 49 |
+
|
| 50 |
+
def set_decoder(self, decoder):
|
| 51 |
+
self.model = decoder
|
| 52 |
+
|
| 53 |
+
def get_decoder(self):
|
| 54 |
+
return self.model
|
| 55 |
+
|
| 56 |
+
def forward(
|
| 57 |
+
self,
|
| 58 |
+
input_ids=None,
|
| 59 |
+
attention_mask=None,
|
| 60 |
+
position_ids=None,
|
| 61 |
+
past_key_values=None,
|
| 62 |
+
inputs_embeds=None,
|
| 63 |
+
labels=None,
|
| 64 |
+
use_cache=None,
|
| 65 |
+
output_attentions=None,
|
| 66 |
+
output_hidden_states=None,
|
| 67 |
+
return_dict=None,
|
| 68 |
+
):
|
| 69 |
+
return self._backbone(
|
| 70 |
+
input_ids=input_ids,
|
| 71 |
+
attention_mask=attention_mask,
|
| 72 |
+
position_ids=position_ids,
|
| 73 |
+
past_key_values=past_key_values,
|
| 74 |
+
inputs_embeds=inputs_embeds,
|
| 75 |
+
labels=labels,
|
| 76 |
+
use_cache=use_cache,
|
| 77 |
+
output_attentions=output_attentions,
|
| 78 |
+
output_hidden_states=output_hidden_states,
|
| 79 |
+
return_dict=return_dict,
|
| 80 |
+
)
|