Updated Space

app.py

@@ -0,0 +1,118 @@
+import torch
+import gradio as gr
+from transformers import AutoTokenizer
+import torch.nn as nn
+import joblib
+
+# Load the label encoder
+label_encoder = joblib.load("label_encoder.pkl")  # Save it during training using joblib.dump(label_encoder, "label_encoder.pkl")
+
+class TransformerEncoderLayer(nn.Module):
+    def __init__(self, embed_size, num_heads, ff_size, dropout=0.1):
+        super(TransformerEncoderLayer, self).__init__()
+        self.attention = nn.MultiheadAttention(embed_size, num_heads, dropout=dropout)
+        self.ffn = nn.Sequential(
+            nn.Linear(embed_size, ff_size),
+            nn.ReLU(),
+            nn.Linear(ff_size, embed_size)
+        )
+        self.norm1 = nn.LayerNorm(embed_size)
+        self.norm2 = nn.LayerNorm(embed_size)
+        self.dropout = nn.Dropout(dropout)
+
+    def forward(self, x):
+        # Multi-head attention
+        attn_output, _ = self.attention(x, x, x)
+        x = self.norm1(x + self.dropout(attn_output))
+
+        # Feed-forward layer
+        ffn_output = self.ffn(x)
+        x = self.norm2(x + self.dropout(ffn_output))
+        
+        return x
+
+
+class CustomTransformerEncoder(nn.Module):
+    def __init__(self, vocab_size, embed_size=768, num_heads=8, num_layers=6, ff_size=2048, max_len=512):
+        super(CustomTransformerEncoder, self).__init__()
+        
+        # Token embedding layer
+        self.embedding = nn.Embedding(vocab_size, embed_size)
+        
+        # Positional encoding
+        self.positional_encoding = nn.Parameter(torch.zeros(1, max_len, embed_size))
+        
+        # Transformer encoder layers
+        self.layers = nn.ModuleList([
+            TransformerEncoderLayer(embed_size, num_heads, ff_size) for _ in range(num_layers)
+        ])
+        
+    def forward(self, input_ids):
+        # Token embeddings
+        token_embeddings = self.embedding(input_ids)
+        
+        # Add positional encoding
+        seq_len = input_ids.size(1)
+        embeddings_with_pos = token_embeddings + self.positional_encoding[:, :seq_len, :]
+        
+        # Pass through the transformer layers
+        x = embeddings_with_pos.transpose(0, 1)  # Transpose for multihead attention (seq_len, batch_size, embed_size)
+        for layer in self.layers:
+            x = layer(x)
+        
+        return x.transpose(0, 1)  # Transpose back to (batch_size, seq_len, embed_size)
+
+
+class CustomTransformerClassifier(nn.Module):
+    def __init__(self, vocab_size, num_classes=2, embed_size=768, num_heads=8, num_layers=6, ff_size=2048, max_len=512):
+        super(CustomTransformerClassifier, self).__init__()
+        
+        # Custom Transformer Encoder
+        self.encoder = CustomTransformerEncoder(vocab_size, embed_size, num_heads, num_layers, ff_size, max_len)
+        
+        # Classification head
+        self.fc = nn.Linear(embed_size, num_classes)
+        
+        # Dropout for regularization
+        self.dropout = nn.Dropout(0.3)
+
+    def forward(self, input_ids):
+        # Pass the input through the encoder
+        encoder_output = self.encoder(input_ids)
+        
+        # Use the output of the [CLS] token (first token) for classification
+        cls_output = encoder_output[:, 0, :]
+        
+        # Dropout for regularization
+        cls_output = self.dropout(cls_output)
+        
+        # Final classification layer
+        logits = self.fc(cls_output)
+        
+        return logits
+# model = CustomTransformerClassifier(vocab_size=len(tokenizer), num_classes= len(label_encoder.classes_))
+
+
+# Load tokenizer
+tokenizer = AutoTokenizer.from_pretrained("bert-base-multilingual-cased")
+
+# Load saved model
+model = CustomTransformerClassifier(vocab_size=len(tokenizer), num_classes=3)
+model.load_state_dict(torch.load("model.pth"))
+model.eval()
+
+def predict(text):
+    encoding = tokenizer(text, max_length=256, padding="max_length", truncation=True, return_tensors="pt")
+    input_ids = encoding["input_ids"]
+    
+    with torch.no_grad():
+        logits = model(input_ids)
+        prediction_index = torch.argmax(logits, dim=1).item()
+    
+    predicted_label = label_encoder.inverse_transform([prediction_index])[0]
+    return f"Predicted Label: {predicted_label}"
+
+
+# Gradio UI
+demo = gr.Interface(fn=predict, inputs=gr.Textbox(placeholder="Enter Nepali text"), outputs="text")
+demo.launch()

label_encoder.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3826f7213efe9876f26275927bbf3cda44572a4bbbdf106ab0c4436c268d1596
+size 483

model.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d0e05b3a344f29c574d6e623d2ce077563ff696aea968680ce791083fac4f573
+size 501188690