Update README.md

README.md

@@ -1,3 +1,54 @@
----
-license: apache-2.0
----
+---
+license: apache-2.0
+---
+class TransformerBlock(nn.Module):
+    def __init__(self, sizeVector=256, numHeads=8, dropout=0.1):
+        super().__init__()
+        self.ln1 = nn.LayerNorm(sizeVector)
+        self.attn = nn.MultiheadAttention(sizeVector, numHeads, batch_first=True)
+        self.dropout_attn = nn.Dropout(dropout)
+        self.ln2 = nn.LayerNorm(sizeVector)
+        self.ff = nn.Sequential(
+            nn.Linear(sizeVector, sizeVector*4),
+            nn.GELU(),
+            nn.Linear(sizeVector*4, sizeVector),
+            nn.Dropout(dropout)
+        )
+    
+    def forward(self, x, attention_mask=None):
+        key_padding_mask = ~attention_mask.bool() if attention_mask is not None else None
+        h = self.ln1(x)
+        attn_out, _ = self.attn(h, h, h, key_padding_mask=key_padding_mask)
+        x = x + self.dropout_attn(attn_out)
+        x = x + self.ff(self.ln2(x))
+        return x
+
+
+class TransformerRun(nn.Module):
+    def __init__(self, vocabSize=120000, maxLen=100, sizeVector=256, numBlocks=4, numHeads=8, numClasses=3, dropout=0.1):
+        super().__init__()
+        self.token_emb = nn.Embedding(vocabSize, sizeVector)
+        self.pos_emb = nn.Embedding(maxLen, sizeVector)
+        self.layers = nn.ModuleList([
+            TransformerBlock(sizeVector=sizeVector, numHeads=numHeads, dropout=dropout)
+            for _ in range(numBlocks)
+        ])
+        self.dropout = nn.Dropout(dropout)
+        self.ln = nn.LayerNorm(sizeVector*2)
+        self.classifier = nn.Linear(sizeVector*2, numClasses)
+
+    def forward(self, x, attention_mask=None):
+        B, T = x.shape
+        tok = self.token_emb(x)
+        pos = self.pos_emb(torch.arange(T, device=x.device).unsqueeze(0).expand(B, T))
+        h = tok + pos
+
+        for layer in self.layers:
+            h = layer(h, attention_mask)
+
+        cls_token = h[:,0,:]
+        mean_pool = h.mean(dim=1)
+        combined = torch.cat([cls_token, mean_pool], dim=1)
+        combined = self.ln(self.dropout(combined))
+        logits = self.classifier(combined)
+        return logits