Upload 5 files

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenized_datasets/c4_realnewslike.json filter=lfs diff=lfs merge=lfs -text

Inference.py

@@ -0,0 +1,66 @@
+import torch
+import Train
+import Tokenizer
+
+
+def load_model(model_path, d_model, ffn_hidden, num_heads, drop_prob, num_layers):
+    model = Train.Transformer(d_model, ffn_hidden, num_heads, drop_prob, num_layers)
+    model.load_state_dict(torch.load(model_path))
+    model.eval()
+    return model
+
+
+def prepare_input(input_text):
+    input_tokens = Tokenizer.tokenize_sequence(input_text)
+    # input_tokens = Tokenizer.pad_to_length(input_tokens, Train.max_sequence_length)
+    input_ids = torch.tensor(input_tokens).unsqueeze(0)  # Add batch dimension
+    return input_ids
+
+
+# def generate_output(model, input_ids):
+#     with torch.no_grad():
+#         output_logits = model(input_ids)
+#     predicted_token_ids = torch.argmax(output_logits, dim=-1)
+#     output_text = Tokenizer.detokenize_sequence(predicted_token_ids[0].tolist())
+#     return output_text
+
+
+# def generate_output(model, input_ids, max_length, eos_token=Tokenizer.vocabulary.get('<EOS>')):
+#     with torch.no_grad():  # No need to track gradients during inference
+#         input_tensor = torch.tensor(input_ids)
+#         output_seq = []
+#
+#         for i in range(50):
+#             output = model.generate(input_tensor)
+#             print(f'output.size(): {output.size()}')
+#             next_token = torch.argmax(output[0, i, :], dim=-1).item()  # Take last token from sequence
+#             output_seq.append(next_token)
+#             if next_token == eos_token:
+#                 break  # Stop if EOS token is generated
+#             next_token_tensor = torch.tensor([[next_token]]).to(Train.device)  # Convert and move to device
+#             input_tensor = torch.cat([input_tensor, next_token_tensor], dim=1)  # Concatenate
+#         print(f'Generated tokens: {output_seq}')
+#         AI_response = Tokenizer.detokenize_sequence(output_seq)
+#         return AI_response
+
+
+def generate_output(model, input_ids, max_length, eos_token=Tokenizer.vocabulary.get('<EOS>')):
+    out = model.generate(input_ids)
+    preds = torch.argmax(out, dim=-1)
+    output_tokens = []
+    for token in preds[0]:
+        output_tokens.append(token.item())
+    AI_response = Tokenizer.detokenize_sequence(output_tokens)
+    return AI_response
+
+
+# Example usage
+model_path = 'models/my_model.pt'
+input_text = ''
+
+model = load_model(model_path, Train.d_model, Train.ffn_hidden, Train.num_heads, Train.drop_prob, Train.num_layers)
+model.to(Train.device)
+input_ids = prepare_input(input_text)
+output_text = generate_output(model, input_ids.to(Train.device), Train.max_sequence_length)
+
+print("Generated Output:", output_text)

Tokenizer.py

@@ -0,0 +1,73 @@
+import re
+import torch
+
+vocabulary = {}
+token_vocabulary = {}
+# vocabulary_length = ['<EOS>']
+
+with open('cl100k_base_vocab_list.txt', 'r', encoding='utf-8') as file:
+    for line_count, line in enumerate(file):
+        line = line.rstrip('\n')
+        if (line.startswith('\'') and line.endswith('\'')) or (line.startswith('\"') and line.endswith('\"')):
+            line = line[1:-1]
+            vocabulary[line] = line_count
+        else:
+            vocabulary[line] = line_count
+token_vocabulary = {v: k for k, v in vocabulary.items()}
+
+def get_vocabulary():
+    return vocabulary
+
+
+def get_token_vocabulary():
+    return token_vocabulary
+
+# def check_vocabulary_length(word):
+#     append_length = True
+#     for vocab in vocabulary_length:
+#         if word == vocab:
+#             append_length = False
+#             break
+#     if append_length == True:
+#         vocabulary_length.append(word)
+#
+# def return_vocabulary_length():
+#     return vocabulary_length
+
+def tokenize_sequence(sentence):
+    # tokenized_seq = [vocabulary.get('<SOS>')]
+    tokenized_seq = []
+    regex = r'(\s+\w+|\S+)'
+    words = re.split(regex, sentence)
+    for word in words:
+        if word in vocabulary:
+            tokenized_seq.append(vocabulary.get(word, vocabulary.get('<UNK>')))
+        else:
+            i = 0
+            while i < len(word):
+                subword_len = 1
+                for j in range(len(word), i - 1, -1):
+                    subword = word[i:j]
+                    if subword in vocabulary:
+                        tokenized_seq.append(vocabulary.get(subword, vocabulary.get('<UNK>')))
+                        subword_len = len(subword)
+                        break
+                    if j - i == 1:
+                        tokenized_seq.append(vocabulary.get('<UNK>'))
+                        break
+                i += subword_len
+    tokenized_seq.append(vocabulary.get('<EOS>'))
+    return tokenized_seq
+
+
+def detokenize_sequence(tokenized_seq):
+    decoded_sentence = ''
+    for token in tokenized_seq:
+        decoded_sentence += token_vocabulary[token]
+    return decoded_sentence
+
+
+def pad_to_length(seq, length):
+    padded_seq = torch.full((length,), fill_value=0, dtype=torch.long)
+    padded_seq[:len(seq)] = torch.tensor(seq, dtype=torch.long)
+    return padded_seq

Train.py

@@ -0,0 +1,290 @@
+import torch
+import math
+from torch import nn
+import torch.nn.functional as F
+import Tokenizer
+from datasets import load_dataset
+import time
+import json
+from transformers import AdamW, get_scheduler
+from sklearn.model_selection import train_test_split
+from torch.nn.utils.rnn import pad_sequence
+
+
+### TOKENIZER ##########################################################################################################
+vocabulary = Tokenizer.get_vocabulary()
+token_vocabulary = Tokenizer.get_token_vocabulary()
+
+
+### TRANSFORMER ########################################################################################################
+d_model = 384
+num_heads = 6
+drop_prob = 0.1
+batch_size = 38  # batch_size must be divisible by num_heads / len(train_input) must be divisible by batch_size
+max_sequence_length = 256
+ffn_hidden = d_model * 4
+num_layers = 6
+save_path = 'models/my_model.pt'
+device = 'cuda' if torch.cuda.is_available() else 'cpu'
+
+
+def scaled_dot_product(q, k, v, mask=None):
+    d_k = q.size()[-1]
+    scaled = torch.matmul(q, k.transpose(-1, -2)) / math.sqrt(d_k)
+    if mask is not None:
+        scaled += mask.to(device)
+    attention = F.softmax(scaled, dim=-1)
+    values = torch.matmul(attention, v)
+    return values, attention
+
+
+class MultiHeadAttention(nn.Module):
+    def __init__(self, d_model, num_heads):
+        super().__init__()
+        self.d_model = d_model
+        self.num_heads = num_heads
+        self.head_dim = d_model // num_heads
+        self.qkv_layer = nn.Linear(d_model, 3 * d_model)
+        self.linear_layer = nn.Linear(d_model, d_model)
+
+    def forward(self, x, mask=None):
+        batch_size, max_sequence_length, d_model = x.size()
+        qkv = self.qkv_layer(x)
+        qkv = qkv.reshape(batch_size, max_sequence_length, self.num_heads, 3 * self.head_dim)
+        qkv = qkv.permute(0, 2, 1, 3)
+        q, k, v = qkv.chunk(3, dim=-1)
+        values, attention = scaled_dot_product(q, k, v, mask)
+        values = values.reshape(batch_size, max_sequence_length, self.num_heads * self.head_dim)
+        out = self.linear_layer(values)
+        return out
+
+
+class LayerNormalization(nn.Module):
+    def __init__(self, parameters_shape, eps=1e-5):
+        super().__init__()
+        self.parameters_shape = parameters_shape
+        self.eps = eps
+        self.gamma = nn.Parameter(torch.ones(parameters_shape))
+        self.beta = nn.Parameter(torch.zeros(parameters_shape))
+
+    def forward(self, inputs):
+        dims = [-(i + 1) for i in range(len(self.parameters_shape))]
+        mean = inputs.mean(dim=dims, keepdim=True)
+        var = ((inputs - mean) ** 2).mean(dim=dims, keepdim=True)
+        std = (var + self.eps).sqrt()
+        y = (inputs - mean) / std
+        out = self.gamma * y + self.beta
+        return out
+
+
+class PositionwiseFeedForward(nn.Module):
+    def __init__(self, d_model, hidden, drop_prob=0.1):
+        super(PositionwiseFeedForward, self).__init__()
+        self.linear1 = nn.Linear(d_model, hidden)
+        self.linear2 = nn.Linear(hidden, d_model)
+        self.dropout = nn.Dropout(p=drop_prob)
+
+    def forward(self, x):
+        x = self.linear1(x)
+        x = F.gelu(x)
+        x = self.dropout(x)
+        x = self.linear2(x)
+        return x
+
+
+class PositionalEncoding(nn.Module):
+    def __init__(self, d_model):
+        super().__init__()
+        self.d_model = d_model
+
+    def forward(self, sequence_length):
+        even_i = torch.arange(0, self.d_model, 2).float()
+        denominator = torch.pow(10000, even_i / self.d_model)
+        position = torch.arange(sequence_length).reshape(sequence_length, 1)
+        even_PE = torch.sin(position / denominator)
+        odd_PE = torch.cos(position / denominator)
+        stacked = torch.stack([even_PE, odd_PE], dim=2)
+        PE = torch.flatten(stacked, start_dim=1, end_dim=2)
+        return PE
+
+
+class TransformerLayer(nn.Module):
+    def __init__(self, d_model, ffn_hidden, num_heads, drop_prob):
+        super(TransformerLayer, self).__init__()
+        self.attention = MultiHeadAttention(d_model=d_model, num_heads=num_heads)
+        self.norm1 = LayerNormalization(parameters_shape=[d_model])
+        self.dropout1 = nn.Dropout(p=drop_prob)
+        self.ffn = PositionwiseFeedForward(d_model=d_model, hidden=ffn_hidden, drop_prob=drop_prob)
+        self.norm2 = LayerNormalization(parameters_shape=[d_model])
+        self.dropout2 = nn.Dropout(p=drop_prob)
+
+    def forward(self, x, original_inputs):
+        input_pad_mask = (original_inputs != 0)
+        index = torch.argmax(input_pad_mask.sum(dim=1))
+        max_length = 0
+        for element in original_inputs[index]:
+            if element != 0:
+                max_length += 1
+            else:
+                break
+        seq_len = x.size()[1]
+        causal_mask = torch.tril(torch.ones(seq_len, seq_len))
+        mask = torch.where(causal_mask == 0, torch.tensor(float('-inf')), causal_mask)
+        mask[mask == 1] = 0
+        mask[max_length:, max_length:] = float('-inf')
+
+        residual_x = x
+        x = self.attention(x, mask=mask)
+        # x = self.dropout1(x)
+        x = self.norm1(x + residual_x)
+        residual_x = x
+        x = self.ffn(x)
+        # x = self.dropout2(x)
+        x = self.norm2(x + residual_x)
+        return x
+
+
+class SequentialTransformer(nn.Sequential):
+    def forward(self, *inputs):
+        x, original_inputs = inputs
+        for module in self._modules.values():
+            new_x = module(x, original_inputs)
+        return new_x
+
+
+class Transformer(nn.Module):
+    def __init__(self, d_model, ffn_hidden, num_heads, drop_prob, num_layers):
+        super().__init__()
+        self.d_model = d_model
+        self.token_embedding = nn.Embedding(len(vocabulary), d_model)
+        # self.token_embedding = nn.Embedding(len(true_vocabulary), d_model)
+        self.positional_encoding = PositionalEncoding(d_model)
+        self.layers = SequentialTransformer(*[TransformerLayer(d_model, ffn_hidden, num_heads, drop_prob)
+                                              for _ in range(num_layers)])
+        self.output_layers = nn.Linear(d_model, len(vocabulary))
+        # self.output_layers = nn.Linear(d_model, len(true_vocabulary))
+
+    def forward(self, x, targets):
+        original_inputs = x
+        token_embeddings = self.token_embedding(x) * math.sqrt(self.d_model)
+        pos_encoding = self.positional_encoding(x.size()[1]).to(device).unsqueeze(0).repeat(x.size(0), 1, 1)
+        x = token_embeddings + pos_encoding
+        x = self.layers(x, original_inputs)
+        logits = self.output_layers(x)
+        loss = F.cross_entropy(logits.view(-1, logits.size(-1)), targets.view(-1))
+        return logits, loss
+
+    def generate(self, x):
+        original_inputs = x
+        token_embeddings = self.token_embedding(x) * math.sqrt(self.d_model)
+        pos_encoding = self.positional_encoding(x.size()[1]).to(device).unsqueeze(0).repeat(x.size(0), 1, 1)
+        x = token_embeddings + pos_encoding
+        x = self.layers(x, original_inputs)
+        x = self.output_layers(x)
+        return F.softmax(x, dim=-1)
+
+
+### DATA PREPROCESSING #################################################################################################
+print('Data Preprocessing...')
+start_time = time.time()
+
+def save_tokenized_data(name, tokenized_dataset):
+    with open(name, 'w') as file:
+        json.dump(tokenized_dataset, file)
+
+def load_tokenized_data(name):
+    with open(f'tokenized_datasets/{name}', 'r') as file:
+        loaded_tokenized_data = json.load(file)
+    return loaded_tokenized_data
+
+# raw_dataset = load_dataset('c4', 'realnewslike')  #***********************************#
+# raw_dataset = raw_dataset['train'].select(range(round(len(raw_dataset['train']) / 1000)))
+# raw_dataset = [Tokenizer.tokenize_sequence(raw_dataset['text'][i]) for i in range(len(raw_dataset['text']))]
+# save_tokenized_data('tokenized_datasets/c4_realnewslike.json', raw_dataset)
+
+raw_dataset = load_tokenized_data('c4_realnewslike.json')  #***********************************#
+token_dataset = []
+for i in range(len(raw_dataset)):
+    for j in range(len(raw_dataset[i])):
+        token_dataset.append(raw_dataset[i][j])
+token_dataset = token_dataset[:round(max_sequence_length * math.floor(len(token_dataset) / max_sequence_length))]
+train_input = [[] for i in range(math.floor(len(token_dataset) / (max_sequence_length * 2)))]
+train_output = [[] for i in range(math.floor(len(token_dataset) / (max_sequence_length * 2)))]
+for i in range(0, len(token_dataset) - max_sequence_length, max_sequence_length * 2):
+    for j in range(max_sequence_length):
+        train_input[round(i / (max_sequence_length * 2))].append(token_dataset[i + j])
+        train_output[round(i / (max_sequence_length * 2))].append(token_dataset[i + j + max_sequence_length])
+print(f'len(train_input) = {len(train_input)}')
+
+# # raw_train_dataset, raw_eval_dataset = train_test_split(raw_dataset['train'].select(range(round(len(raw_dataset['train']) / 25))), test_size=0.2)
+train_input = [seq[:max_sequence_length] if len(seq) > max_sequence_length else seq for seq in train_input]
+train_output = [seq[:max_sequence_length] if len(seq) > max_sequence_length else seq for seq in train_output]
+train_input = [torch.tensor(seq, dtype=torch.long) for seq in train_input]
+train_output = [torch.tensor(seq, dtype=torch.long) for seq in train_output]
+# train_input = [Tokenizer.pad_to_length(seq, max_sequence_length) for seq in train_input]
+# train_output = [Tokenizer.pad_to_length(seq, max_sequence_length) for seq in train_output]
+train_dataset = [(train_input[i], train_output[i]) for i in range(len(train_input))]
+# train_dataset = [pad_sequence(train_dataset[i], batch_first=True, padding_value=0) for i in range(len(train_dataset))]
+train_batch = [[[] for i in range(round(len(train_dataset) / batch_size))] for j in range(2)]
+train_batch_count = 0
+for i in range(0, len(train_dataset), batch_size):
+    for j in range(batch_size):
+        train_batch[0][train_batch_count].append(train_dataset[i + j][0])
+        train_batch[1][train_batch_count].append(train_dataset[i + j][1])
+    train_batch_count += 1
+
+
+### TRAINING ###########################################################################################################
+print('Training...')
+model = Transformer(d_model, ffn_hidden, num_heads, drop_prob, num_layers)
+print(f'model parameters: {sum(p.numel() for p in model.parameters())}')
+model.to(device)
+epochs = 5
+optimizer = torch.optim.AdamW(model.parameters(), lr=0.01)
+
+num_training_steps = epochs * len(train_dataset)
+lr_scheduler = get_scheduler(
+    name='linear',
+    optimizer=optimizer,
+    num_warmup_steps=0,
+    num_training_steps=num_training_steps
+)
+train_epoch_average_loss = []
+train_loss_total = 0
+for epoch in range(epochs):
+    model.train()
+    train_loss = 0
+    for i in range(len(train_batch[0])):
+        inputs = torch.stack(train_batch[0][i]).to(device)
+        labels = torch.stack(train_batch[1][i]).to(device)
+        logits, loss = model.forward(inputs, labels)
+        optimizer.zero_grad()
+        loss.backward()
+        optimizer.step()
+        lr_scheduler.step()
+        train_loss_total += loss
+        if i % 10 == 0:
+            print('TRAINING...')
+            print(f'EPOCH {epoch}, batch {i}/{len(train_batch[0])}')
+            print(f'loss: {loss}')
+    train_epoch_average_loss.append((train_loss_total / len(train_batch[0])))
+    train_loss_total = 0
+    # model.eval()
+    # eval_loss = 0
+    # with torch.no_grad():
+    #     for i, batch in enumerate(eval_dataset):
+    #         inputs = batch[0].unsqueeze(0).to(device)
+    #         labels = batch[1].unsqueeze(0).to(device)
+    #         logits, loss = model(inputs, labels)
+    #         if i % 10 == 0:
+    #             print('EVALUATING...')
+    #             print(f'EPOCH {epoch}, batch {i}/{len(eval_dataset)}')
+    #             print(f'loss: {loss}')
+for i in range(len(train_epoch_average_loss)):
+    print(f'EPOCH {i} AVERAGE LOSS: {train_epoch_average_loss[i]}')
+torch.save(model.state_dict(), save_path)
+
+
+end_time = time.time()
+total_time = end_time - start_time
+print(f'{total_time} seconds')

tokenized_datasets/c4_realnewslike.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e6fcbb4ebe2e58cf94a59ab469390d801b808a68ae7564ff1d98fe537997e4ab
+size 43787042