Upload 3 files

LCTLM.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ba04a030e9aaa5d1c88def1f8738ec6c465491f8ac91bad3a38e39c4d3df6a23
+size 176879430

lctlm1.py

@@ -0,0 +1,95 @@
+# -*- coding: utf-8 -*-
+"""LCTLM1.ipynb
+
+Automatically generated by Colab.
+
+Original file is located at
+    https://colab.research.google.com/drive/1WtvvYAajPbW2YCEkE5Cg0IT8lKN-lPfk
+"""
+
+import torch
+from torch import nn
+from typing import Optional
+class LCMBlock (nn.Module) :
+  """
+    LCm (Laten Connected Model ) block, looking attention as two preception and icreasing it
+    to N multiple magnitude values.
+  """
+
+  def __init__ (self,d_model :int, drop_rate : float = 0.1) :
+    """
+      args:
+        d_model : int
+          dimention of model
+
+        drop_rate : float
+          rate of dropout mechanism
+    """
+    super().__init__()
+    self.step1 = nn.Linear(d_model,d_model)
+    self.step2 = nn.Linear(d_model,d_model)
+    self.magnitude = nn.Linear(d_model,d_model)
+    self.drop = nn.Dropout(drop_rate)
+    self.gelu1 = nn.GELU(approximate='tanh')
+    self.gelu2 = nn.GELU(approximate='tanh')
+    self.tanh = nn.Tanh()
+    self.norm = nn.LayerNorm(d_model)
+
+  def forward(self,x) :
+    normx = self.norm(x)
+    step1 = self.step1(normx)
+    step1 = self.gelu1(step1)
+    step2 = self.step2(normx)
+    step2 = self.gelu2(step2)
+    laten = step1 + step2
+    laten - self.drop(laten)
+    laten = self.magnitude(laten)
+    laten = self.tanh(laten)
+    return x + laten
+
+class LMLCTBlock (nn.Module) :
+  def __init__ (self,d_model,drop_rate) :
+    super().__init__()
+    self.attention = nn.MultiheadAttention(embed_dim=d_model,num_heads=8,dropout=drop_rate,batch_first=True)
+    self.norm = nn.LayerNorm(d_model)
+    self.lcmblock = LCMBlock(d_model,drop_rate)
+
+
+  def forward(self,x,mask) :
+
+    normx = self.norm(x)
+    attention,_ = self.attention(normx,normx,normx,attn_mask=mask)
+    x = x + attention
+    x = self.lcmblock(x)
+    return x
+
+import math
+class LMLCT1(nn.Module):
+    def __init__(self, d_model=512, vocab_size=30001, num_layers=6, drop_rate=0.1, maxpos=500):
+        super().__init__()
+        self.d_model = d_model
+        self.embedding = nn.Embedding(vocab_size, d_model, padding_idx=0)
+        self.pos_embedding = nn.Embedding(maxpos, d_model)
+        self.scale = math.sqrt(d_model)
+        self.ffn = nn.Sequential(
+            nn.Linear(d_model, d_model*4),
+            nn.GELU(),
+            nn.Linear(d_model*4, d_model),
+         )
+        self.layers = nn.ModuleList([LMLCTBlock(d_model, drop_rate) for _ in range(num_layers)])
+        self.out = nn.Linear(d_model, vocab_size)
+        mask = torch.triu(torch.ones(maxpos, maxpos), diagonal=1).bool()
+        self.register_buffer("causal_mask", mask)
+
+    def forward(self, x):
+        B, S = x.size()
+        pos_idx = torch.arange(S, device=x.device)
+        x = self.embedding(x) * self.scale
+        pos = self.pos_embedding(pos_idx).unsqueeze(0)
+        x = x + pos
+        mask = self.causal_mask[:S, :S]
+        for layer in self.layers:
+            x = layer(x, attn_mask=mask)
+        x = self.ffn(x)
+        logits = self.out(x)
+        return logits