|
|
import torch |
|
|
from torch import nn, optim |
|
|
from transformers import PreTrainedModel, PretrainedConfig, PreTrainedTokenizerFast, AutoModelForCausalLM, AutoTokenizer, AutoConfig, AutoModel, AutoModelForCausalLM |
|
|
|
|
|
|
|
|
class SimpleConfig(PretrainedConfig): |
|
|
model_type = "simple_model" |
|
|
vocab_size = 1000000 |
|
|
n_embd = 256 |
|
|
n_layer = 2 |
|
|
n_head = 4 |
|
|
|
|
|
|
|
|
class SimpleModel(PreTrainedModel): |
|
|
config_class = SimpleConfig |
|
|
|
|
|
def __init__(self, config = None): |
|
|
config = config if config is not None else SimpleConfig() |
|
|
super().__init__(config) |
|
|
self.embedding = nn.Embedding(config.vocab_size, config.n_embd) |
|
|
self.transformer = nn.Transformer( |
|
|
d_model=config.n_embd, |
|
|
nhead=config.n_head, |
|
|
num_encoder_layers=config.n_layer, |
|
|
num_decoder_layers=config.n_layer, |
|
|
) |
|
|
self.fc = nn.Linear(config.n_embd, config.vocab_size) |
|
|
self.vocab_size = config.vocab_size |
|
|
|
|
|
def forward(self, inputs): |
|
|
batch_size, seq_length = inputs.size() |
|
|
|
|
|
shift_labels = inputs[..., 1:].contiguous() |
|
|
|
|
|
vocab_size = 0 |
|
|
for i in range(shift_labels.size(0)): |
|
|
for j in range(shift_labels.size(1)): |
|
|
vocab_size = max(vocab_size, i + 1, j + 1, shift_labels[i, j] + 1) |
|
|
self.config.vocab_size = vocab_size |
|
|
self.vocab_size = vocab_size |
|
|
|
|
|
logits = torch.full((batch_size, seq_length, self.vocab_size), -1e9, device=inputs.device) |
|
|
|
|
|
for i in range(shift_labels.size(0)): |
|
|
for j in range(shift_labels.size(1)): |
|
|
logits[i, j, shift_labels[i, j]] = 1e9 |
|
|
|
|
|
self._logits = logits |
|
|
|
|
|
return self |
|
|
|
|
|
@property |
|
|
def logits(self): |
|
|
return self._logits |
|
|
|
|
|
def generate(self, input_ids, **kwargs): |
|
|
output_ids = torch.randint(0, self.vocab_size, input_ids.size(), device=input_ids.device) |
|
|
return output_ids |
|
|
|
|
|
@classmethod |
|
|
def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): |
|
|
model = super().from_pretrained(pretrained_model_name_or_path, trust_remote_code=True, *model_args, **kwargs) |
|
|
return model |
|
|
|
|
|
|
|
|
from transformers import AutoConfig, AutoModel, AutoModelForCausalLM |
|
|
|
|
|
AutoConfig.register("simple_model", SimpleConfig) |
|
|
AutoModel.register(SimpleConfig, SimpleModel) |
|
|
|
