Phase 1: Environment setup and training pipeline

.gitignore

@@ -0,0 +1,12 @@
+models/
+data/
+logs/
+__pycache__/
+urdu_env/
+venv/
+*.docx
+*.html
+*.pdf
+.env
+remove_comments_script.py
+*.tsv

README.md

@@ -0,0 +1,36 @@
+# Urdu Sentiment and Emotion Analysis Engine
+
+Welcome to the Urdu Sentiment and Emotion Analysis Engine project! This repository contains the code for a multilingual NLP system that classifies sentiment (Positive, Negative, Neutral) and emotion (Joy, Anger, Fear, Sadness) from Urdu, Roman Urdu, and mixed-language text using a fine-tuned XLM-RoBERTa transformer.
+
+## Phase 1: Environment Setup & Training Pipeline
+This initial commit includes **Phase 1** of our project roadmap: setting up the environment, establishing the dataset loaders, and building the initial training scripts for the sentiment and emotion models.
+
+### Repository Structure (Phase 1)
+- `requirements.txt`: Environment dependencies required for training and inference.
+- `training/`: Contains the core scripts for data processing and model fine-tuning.
+  - `dataset.py`: PyTorch `Dataset` implementation for loading and tokenizing text using XLM-RoBERTa.
+  - `train_sentiment.py`: Training script for the sentiment classification model.
+  - `train_emotion.py`: Training script for the emotion classification model.
+- `test_models.py`: A utility script to load trained models and run interactive inference.
+
+### Environment Setup
+To get started, create a virtual environment and install the required dependencies:
+
+```bash
+# Create a virtual environment
+python -m venv urdu_env
+
+# Activate the virtual environment
+# On Windows:
+urdu_env\Scripts\activate
+# On Linux/Mac:
+source urdu_env/bin/activate
+
+# Install dependencies
+pip install -r requirements.txt
+```
+
+### Next Steps
+The models are trained using HuggingFace's Trainer API on local datasets (which have been kept separate from the repository due to size constraints). In upcoming phases, we will introduce the Flask Backend (REST API), Frontend Dashboard, and cloud deployment via HuggingFace Spaces.
+
+Stay tuned for Phase 2 updates!

static/script.js

@@ -0,0 +1,334 @@
+document.addEventListener('DOMContentLoaded', () => {
+    // Theme toggling
+    const themeToggle = document.getElementById('themeToggle');
+    const body = document.body;
+    
+    themeToggle.addEventListener('click', () => {
+        body.classList.toggle('light-theme');
+        // Update charts on theme change
+        updateChartColors();
+    });
+
+    // Auto-resize textarea
+    const textInput = document.getElementById('textInput');
+    textInput.addEventListener('input', function() {
+        this.style.height = 'auto';
+        this.style.height = (this.scrollHeight) + 'px';
+        
+        // Auto-detect direction (simple heuristic)
+        const urduRegex = /[\u0600-\u06FF]/;
+        if (urduRegex.test(this.value)) {
+            this.style.direction = 'rtl';
+            this.classList.add('urdu-text');
+        } else {
+            this.style.direction = 'ltr';
+            this.classList.remove('urdu-text');
+        }
+    });
+
+    // Handle Analysis
+    const analyzeBtn = document.getElementById('analyzeBtn');
+    const resultsCard = document.getElementById('resultsCard');
+    
+    analyzeBtn.addEventListener('click', async () => {
+        const text = textInput.value.trim();
+        if (!text) return;
+        
+        // UI Loading state
+        const originalBtnContent = analyzeBtn.innerHTML;
+        analyzeBtn.innerHTML = `<span>Analyzing...</span><svg class="spinner" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" style="width:20px;height:20px;"><path stroke-linecap="round" stroke-linejoin="round" d="M4 4v5h.582m15.356 2A8.001 8.001 0 004.582 9m0 0H9m11 11v-5h-.581m0 0a8.003 8.003 0 01-15.357-2m15.357 2H15"></path></svg>`;
+        analyzeBtn.disabled = true;
+
+        try {
+            // Attempt to hit real backend
+            let data;
+            try {
+                const response = await fetch('/analyze', {
+                    method: 'POST',
+                    headers: { 'Content-Type': 'application/json' },
+                    body: JSON.stringify({ text })
+                });
+                if (response.ok) {
+                    data = await response.json();
+                } else {
+                    throw new Error('Backend not ready');
+                }
+            } catch (err) {
+                console.log("Mocking response since backend isn't available:", err);
+                // Mock fallback for UI demonstration
+                await new Promise(r => setTimeout(r, 1200)); // Simulate delay
+                data = generateMockResult(text);
+            }
+            
+            displayResults(data);
+            updateAnalyticsMock();
+        } catch (error) {
+            console.error('Analysis error:', error);
+            alert('Error analyzing text. Please try again.');
+        } finally {
+            analyzeBtn.innerHTML = originalBtnContent;
+            analyzeBtn.disabled = false;
+        }
+    });
+
+    // Charts Initialization
+    initCharts();
+    
+    // Start Live Feed Simulation
+    startLiveFeed();
+    
+    // Init Keywords
+    initKeywords();
+});
+
+// Mock Data Generator
+function generateMockResult(text) {
+    const isUrdu = /[\u0600-\u06FF]/.test(text);
+    const lang = isUrdu ? 'Urdu' : 'Roman Urdu / English';
+    
+    // Randomize for demo
+    const sentiments = ['Positive', 'Negative', 'Neutral'];
+    const emotions = ['Joy', 'Anger', 'Fear', 'Sadness'];
+    
+    const sentiment = sentiments[Math.floor(Math.random() * sentiments.length)];
+    const emotion = emotions[Math.floor(Math.random() * emotions.length)];
+    const confS = (Math.random() * 20 + 75).toFixed(1); // 75-95%
+    const confE = (Math.random() * 30 + 60).toFixed(1); // 60-90%
+
+    // Attention map mock
+    const words = text.split(/\s+/);
+    const attentionWords = words.map(word => {
+        return {
+            word: word,
+            weight: Math.random() // 0 to 1
+        };
+    });
+
+    return {
+        language: lang,
+        sentiment: { label: sentiment, confidence: parseFloat(confS) },
+        emotion: { label: emotion, confidence: parseFloat(confE) },
+        attention: attentionWords
+    };
+}
+
+// Display Results
+function displayResults(data) {
+    document.getElementById('detectedLang').textContent = data.language;
+    document.getElementById('resultsCard').style.display = 'block';
+
+    // Update Sentiment
+    const sBox = document.getElementById('sentimentResult');
+    const sConfBar = document.getElementById('sentimentConfidence');
+    const sConfText = document.getElementById('sentimentConfText');
+    
+    let sEmoji = '😐', sColor = 'var(--neutral)';
+    if (data.sentiment.label === 'Positive') { sEmoji = '😊'; sColor = 'var(--positive)'; }
+    if (data.sentiment.label === 'Negative') { sEmoji = '😠'; sColor = 'var(--negative)'; }
+    
+    sBox.innerHTML = `<span class="emoji">${sEmoji}</span><span class="label" style="color: ${sColor}">${data.sentiment.label}</span>`;
+    sConfBar.style.width = '0%'; // Reset for animation
+    sConfBar.style.backgroundColor = sColor;
+    setTimeout(() => {
+        sConfBar.style.width = `${data.sentiment.confidence}%`;
+    }, 100);
+    sConfText.textContent = `${data.sentiment.confidence}% Confidence`;
+
+    // Update Emotion
+    const eBox = document.getElementById('emotionResult');
+    const eConfBar = document.getElementById('emotionConfidence');
+    const eConfText = document.getElementById('emotionConfText');
+    
+    let eEmoji = '🤔', eColor = 'var(--primary)';
+    if (data.emotion.label === 'Joy') { eEmoji = '😄'; eColor = 'var(--joy)'; }
+    if (data.emotion.label === 'Anger') { eEmoji = '😡'; eColor = 'var(--anger)'; }
+    if (data.emotion.label === 'Fear') { eEmoji = '😨'; eColor = 'var(--fear)'; }
+    if (data.emotion.label === 'Sadness') { eEmoji = '😢'; eColor = 'var(--sadness)'; }
+
+    eBox.innerHTML = `<span class="emoji">${eEmoji}</span><span class="label" style="color: ${eColor}">${data.emotion.label}</span>`;
+    eConfBar.style.width = '0%'; // Reset for animation
+    eConfBar.style.backgroundColor = eColor;
+    setTimeout(() => {
+        eConfBar.style.width = `${data.emotion.confidence}%`;
+    }, 100);
+    eConfText.textContent = `${data.emotion.confidence}% Confidence`;
+
+    // Render Attention Map
+    const attContainer = document.getElementById('attentionResult');
+    attContainer.innerHTML = '';
+    
+    const isRtl = /[\u0600-\u06FF]/.test(data.attention.map(w=>w.word).join(' '));
+    attContainer.style.direction = isRtl ? 'rtl' : 'ltr';
+
+    data.attention.forEach(item => {
+        const span = document.createElement('span');
+        span.textContent = item.word + ' ';
+        span.className = 'attention-word';
+        
+        span.style.backgroundColor = `rgba(99, 102, 241, ${item.weight * 0.5})`;
+        span.title = `Attention Weight: ${(item.weight).toFixed(2)}`;
+        
+        attContainer.appendChild(span);
+    });
+    
+    // Scroll to results
+    setTimeout(() => {
+        document.getElementById('resultsCard').scrollIntoView({ behavior: 'smooth', block: 'nearest' });
+    }, 100);
+}
+
+// Charts
+let sentimentChart, emotionChart;
+
+function initCharts() {
+    const textColor = getComputedStyle(document.body).getPropertyValue('--text-primary').trim() || '#f9fafb';
+    
+    // Sentiment Donut
+    const ctxS = document.getElementById('sentimentChart').getContext('2d');
+    sentimentChart = new Chart(ctxS, {
+        type: 'doughnut',
+        data: {
+            labels: ['Positive', 'Negative', 'Neutral'],
+            datasets: [{
+                data: [45, 30, 25], // initial mock data
+                backgroundColor: ['#10b981', '#ef4444', '#6b7280'],
+                borderWidth: 0,
+                hoverOffset: 4
+            }]
+        },
+        options: {
+            responsive: true,
+            maintainAspectRatio: false,
+            plugins: {
+                legend: { position: 'bottom', labels: { color: textColor } },
+                title: { display: true, text: 'Sentiment Distribution', color: textColor }
+            },
+            cutout: '70%'
+        }
+    });
+
+    // Emotion Bar
+    const ctxE = document.getElementById('emotionChart').getContext('2d');
+    emotionChart = new Chart(ctxE, {
+        type: 'bar',
+        data: {
+            labels: ['Joy', 'Anger', 'Fear', 'Sadness'],
+            datasets: [{
+                label: 'Emotion Count',
+                data: [40, 25, 15, 20], // initial mock data
+                backgroundColor: ['#f59e0b', '#ef4444', '#8b5cf6', '#3b82f6'],
+                borderRadius: 4
+            }]
+        },
+        options: {
+            responsive: true,
+            maintainAspectRatio: false,
+            plugins: {
+                legend: { display: false },
+                title: { display: true, text: 'Emotion Frequency', color: textColor }
+            },
+            scales: {
+                y: { beginAtZero: true, grid: { color: 'rgba(156, 163, 175, 0.1)' }, ticks: { color: textColor } },
+                x: { grid: { display: false }, ticks: { color: textColor } }
+            }
+        }
+    });
+}
+
+function updateChartColors() {
+    const textColor = getComputedStyle(document.body).getPropertyValue('--text-primary').trim() || '#111827';
+    if (sentimentChart) {
+        sentimentChart.options.plugins.legend.labels.color = textColor;
+        sentimentChart.options.plugins.title.color = textColor;
+        sentimentChart.update();
+    }
+    if (emotionChart) {
+        emotionChart.options.plugins.title.color = textColor;
+        emotionChart.options.scales.x.ticks.color = textColor;
+        emotionChart.options.scales.y.ticks.color = textColor;
+        emotionChart.update();
+    }
+}
+
+function updateAnalyticsMock() {
+    if (sentimentChart && emotionChart) {
+        // Slightly jitter data for demo
+        const sData = sentimentChart.data.datasets[0].data;
+        sData[0] += Math.floor(Math.random() * 3);
+        sData[1] += Math.floor(Math.random() * 3);
+        sData[2] += Math.floor(Math.random() * 3);
+        sentimentChart.update();
+
+        const eData = emotionChart.data.datasets[0].data;
+        eData[Math.floor(Math.random() * 4)] += 1;
+        emotionChart.update();
+    }
+}
+
+// Live Feed Simulation
+const mockTweets = [
+    { text: "آج کا دن بہت اچھا گزر رہا ہے! 🌞", user: "@ahmed_pk", pos: true, neu: false, neg: false },
+    { text: "traffic ne bohat tang kiya hua hai aaj 😡", user: "@sana_tweets", pos: false, neu: false, neg: true },
+    { text: "Weather update: It might rain tomorrow in Lahore.", user: "@news_update", pos: false, neu: true, neg: false },
+    { text: "مجھے ڈر ہے کہ کل کا امتحان کیسا ہوگا 😰", user: "@student_life", pos: false, neu: false, neg: true },
+    { text: "ye movie bohat zbardast thi, highly recommended! 🎬", user: "@cinephile", pos: true, neu: false, neg: false }
+];
+
+function startLiveFeed() {
+    const container = document.getElementById('liveFeed');
+    
+    // Initial tweets
+    for (let i = 0; i < 3; i++) {
+        addTweetToFeed(container, mockTweets[i]);
+    }
+
+    // Add new tweet periodically
+    setInterval(() => {
+        const tweet = mockTweets[Math.floor(Math.random() * mockTweets.length)];
+        addTweetToFeed(container, tweet);
+    }, 8000);
+}
+
+function addTweetToFeed(container, data) {
+    const el = document.createElement('div');
+    el.className = 'tweet';
+    
+    let tagsHtml = '';
+    if (data.pos) tagsHtml += '<span class="tag pos">Positive</span>';
+    if (data.neg) tagsHtml += '<span class="tag neg">Negative</span>';
+    if (data.neu) tagsHtml += '<span class="tag neu">Neutral</span>';
+    
+    el.innerHTML = `
+        <div class="tweet-header">
+            <span class="tweet-user">${data.user}</span>
+            <span class="tweet-time">just now</span>
+        </div>
+        <div class="tweet-text ${/[\u0600-\u06FF]/.test(data.text) ? 'urdu-text' : ''}">
+            ${data.text}
+        </div>
+        <div class="tweet-tags">
+            ${tagsHtml}
+        </div>
+    `;
+    
+    container.prepend(el);
+    if (container.children.length > 10) {
+        container.removeChild(container.lastChild);
+    }
+}
+
+function initKeywords() {
+    const keywords = ['اچھا (Good)', 'برا (Bad)', 'خوش (Happy)', 'traffic', 'movie', 'امتحان', 'khushi', 'zbardast', 'غصہ', 'barish'];
+    const container = document.getElementById('trendingKeywords');
+    
+    keywords.forEach(kw => {
+        const span = document.createElement('span');
+        span.className = 'keyword';
+        span.textContent = kw;
+        span.onclick = () => {
+            document.getElementById('textInput').value = kw.split(' ')[0];
+            document.getElementById('analyzeBtn').click();
+        };
+        container.appendChild(span);
+    });
+}

static/style.css

@@ -0,0 +1,532 @@
+:root {
+    /* Dark Theme (Default) */
+    --bg-color: #0b0f19;
+    --surface: #111827;
+    --surface-hover: #1f2937;
+    --border: #374151;
+    --text-primary: #f9fafb;
+    --text-secondary: #9ca3af;
+    --primary: #6366f1;
+    --primary-hover: #4f46e5;
+    
+    /* Sentiments */
+    --positive: #10b981;
+    --negative: #ef4444;
+    --neutral: #6b7280;
+    
+    /* Emotions */
+    --joy: #f59e0b;
+    --anger: #ef4444;
+    --fear: #8b5cf6;
+    --sadness: #3b82f6;
+
+    /* Effects */
+    --glass-bg: rgba(17, 24, 39, 0.7);
+    --glass-border: rgba(255, 255, 255, 0.1);
+    --shadow-sm: 0 1px 2px 0 rgba(0, 0, 0, 0.05);
+    --shadow-md: 0 4px 6px -1px rgba(0, 0, 0, 0.1), 0 2px 4px -1px rgba(0, 0, 0, 0.06);
+    --shadow-lg: 0 10px 15px -3px rgba(0, 0, 0, 0.1), 0 4px 6px -2px rgba(0, 0, 0, 0.05);
+}
+
+/* Light Theme Variables */
+body.light-theme {
+    --bg-color: #f3f4f6;
+    --surface: #ffffff;
+    --surface-hover: #f9fafb;
+    --border: #e5e7eb;
+    --text-primary: #111827;
+    --text-secondary: #4b5563;
+    --glass-bg: rgba(255, 255, 255, 0.7);
+    --glass-border: rgba(0, 0, 0, 0.1);
+}
+
+* {
+    margin: 0;
+    padding: 0;
+    box-sizing: border-box;
+}
+
+body {
+    font-family: 'Inter', sans-serif;
+    background-color: var(--bg-color);
+    color: var(--text-primary);
+    min-height: 100vh;
+    transition: background-color 0.3s ease, color 0.3s ease;
+}
+
+.urdu-text {
+    font-family: 'Noto Nastaliq Urdu', serif;
+    direction: rtl;
+}
+
+.urdu-input {
+    font-family: 'Noto Nastaliq Urdu', serif;
+}
+
+/* App Container */
+.app-container {
+    max-width: 1400px;
+    margin: 0 auto;
+    padding: 0 1rem;
+}
+
+/* Header */
+.glass-header {
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+    padding: 1rem 2rem;
+    margin: 1rem 0 2rem 0;
+    background: var(--glass-bg);
+    backdrop-filter: blur(12px);
+    border: 1px solid var(--glass-border);
+    border-radius: 1rem;
+    position: sticky;
+    top: 1rem;
+    z-index: 50;
+    box-shadow: var(--shadow-md);
+}
+
+.logo {
+    display: flex;
+    align-items: center;
+    gap: 1rem;
+}
+
+.logo-icon {
+    width: 48px;
+    height: 48px;
+    background: linear-gradient(135deg, var(--primary), var(--fear));
+    border-radius: 12px;
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    font-size: 1.25rem;
+    font-weight: bold;
+    color: white;
+    font-family: 'Noto Nastaliq Urdu', serif;
+}
+
+.logo-text h1 {
+    font-size: 1.25rem;
+    font-weight: 700;
+    margin: 0;
+    background: linear-gradient(to right, #6366f1, #8b5cf6);
+    -webkit-background-clip: text;
+    -webkit-text-fill-color: transparent;
+}
+
+.badge {
+    font-size: 0.75rem;
+    padding: 0.2rem 0.5rem;
+    background: rgba(99, 102, 241, 0.2);
+    color: #818cf8;
+    border-radius: 99px;
+    font-weight: 600;
+}
+
+.icon-btn {
+    background: transparent;
+    border: none;
+    color: var(--text-primary);
+    width: 40px;
+    height: 40px;
+    border-radius: 50%;
+    cursor: pointer;
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    transition: background 0.2s;
+}
+
+.icon-btn:hover {
+    background: var(--surface-hover);
+}
+
+.icon-btn svg {
+    width: 20px;
+    height: 20px;
+}
+
+/* Dashboard Layout */
+.dashboard {
+    display: grid;
+    grid-template-columns: 1fr 350px;
+    gap: 2rem;
+    padding-bottom: 2rem;
+}
+
+@media (max-width: 1024px) {
+    .dashboard {
+        grid-template-columns: 1fr;
+    }
+}
+
+/* Cards */
+.card {
+    background: var(--surface);
+    border: 1px solid var(--border);
+    border-radius: 1rem;
+    padding: 1.5rem;
+    box-shadow: var(--shadow-md);
+    transition: transform 0.2s, box-shadow 0.2s;
+}
+
+.card:hover {
+    box-shadow: var(--shadow-lg);
+}
+
+.card-header {
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+    margin-bottom: 1rem;
+}
+
+.card-header h2 {
+    font-size: 1.125rem;
+    font-weight: 600;
+}
+
+/* Input Section */
+.input-wrapper {
+    position: relative;
+    margin-bottom: 1rem;
+}
+
+textarea {
+    width: 100%;
+    min-height: 120px;
+    background: var(--bg-color);
+    border: 1px solid var(--border);
+    border-radius: 0.75rem;
+    padding: 1rem;
+    color: var(--text-primary);
+    font-size: 1rem;
+    resize: vertical;
+    transition: border-color 0.2s, box-shadow 0.2s;
+}
+
+textarea:focus {
+    outline: none;
+    border-color: var(--primary);
+    box-shadow: 0 0 0 3px rgba(99, 102, 241, 0.2);
+}
+
+.actions {
+    display: flex;
+    justify-content: flex-end;
+}
+
+.primary-btn {
+    background: var(--primary);
+    color: white;
+    border: none;
+    padding: 0.75rem 1.5rem;
+    border-radius: 0.75rem;
+    font-size: 1rem;
+    font-weight: 600;
+    cursor: pointer;
+    display: flex;
+    align-items: center;
+    gap: 0.5rem;
+    transition: background-color 0.2s, transform 0.1s;
+}
+
+.primary-btn:hover {
+    background: var(--primary-hover);
+    transform: translateY(-1px);
+}
+
+.primary-btn:active {
+    transform: translateY(0);
+}
+
+.primary-btn svg {
+    width: 20px;
+    height: 20px;
+}
+
+.primary-btn:disabled {
+    opacity: 0.7;
+    cursor: not-allowed;
+    transform: none;
+}
+
+.spinner {
+    animation: spin 1s linear infinite;
+}
+
+@keyframes spin {
+    from { transform: rotate(0deg); }
+    to { transform: rotate(360deg); }
+}
+
+/* Language Badge */
+.language-badge {
+    background: rgba(99, 102, 241, 0.1);
+    color: var(--primary);
+    padding: 0.25rem 0.75rem;
+    border-radius: 99px;
+    font-size: 0.875rem;
+    font-weight: 500;
+}
+
+/* Results Section */
+.results-card {
+    margin-top: 2rem;
+    animation: slideUp 0.4s ease-out;
+}
+
+@keyframes slideUp {
+    from { opacity: 0; transform: translateY(20px); }
+    to { opacity: 1; transform: translateY(0); }
+}
+
+.results-grid {
+    display: grid;
+    grid-template-columns: 1fr 1fr;
+    gap: 1.5rem;
+}
+
+@media (max-width: 640px) {
+    .results-grid {
+        grid-template-columns: 1fr;
+    }
+}
+
+.result-box {
+    background: var(--bg-color);
+    border: 1px solid var(--border);
+    border-radius: 0.75rem;
+    padding: 1.25rem;
+    text-align: center;
+}
+
+.result-box h3 {
+    font-size: 0.875rem;
+    color: var(--text-secondary);
+    text-transform: uppercase;
+    letter-spacing: 0.05em;
+    margin-bottom: 1rem;
+}
+
+.primary-result {
+    display: flex;
+    flex-direction: column;
+    align-items: center;
+    gap: 0.5rem;
+    margin-bottom: 1rem;
+}
+
+.primary-result .emoji {
+    font-size: 2.5rem;
+    line-height: 1;
+}
+
+.primary-result .label {
+    font-size: 1.25rem;
+    font-weight: 600;
+}
+
+.confidence-bar {
+    height: 6px;
+    background: var(--surface);
+    border-radius: 99px;
+    overflow: hidden;
+    margin-bottom: 0.5rem;
+}
+
+.confidence-bar .fill {
+    height: 100%;
+    background: var(--primary);
+    border-radius: 99px;
+    transition: width 1s cubic-bezier(0.4, 0, 0.2, 1);
+}
+
+.conf-text {
+    font-size: 0.75rem;
+    color: var(--text-secondary);
+}
+
+/* Attention Map */
+.attention-box {
+    margin-top: 1.5rem;
+    background: var(--bg-color);
+    border: 1px solid var(--border);
+    border-radius: 0.75rem;
+    padding: 1.25rem;
+}
+
+.attention-box h3 {
+    font-size: 0.875rem;
+    color: var(--text-secondary);
+    margin-bottom: 1rem;
+    display: flex;
+    align-items: center;
+    gap: 0.5rem;
+}
+
+.info-icon {
+    cursor: help;
+    color: var(--primary);
+}
+
+.attention-content {
+    font-size: 1.5rem;
+    line-height: 2;
+    padding: 1rem;
+}
+
+.attention-word {
+    padding: 0.1rem 0.2rem;
+    border-radius: 0.25rem;
+    transition: background-color 0.3s;
+}
+
+/* Analytics */
+.analytics-card {
+    margin-top: 2rem;
+}
+
+.charts-container {
+    display: grid;
+    grid-template-columns: 1fr 1fr;
+    gap: 1.5rem;
+    margin-top: 1.5rem;
+}
+
+.chart-wrapper {
+    position: relative;
+    height: 250px;
+    width: 100%;
+}
+
+@media (max-width: 768px) {
+    .charts-container {
+        grid-template-columns: 1fr;
+    }
+}
+
+/* Sidebar Elements */
+.mt-4 {
+    margin-top: 1.5rem;
+}
+
+.live-indicator {
+    display: flex;
+    align-items: center;
+    gap: 0.5rem;
+}
+
+.dot {
+    width: 8px;
+    height: 8px;
+    background-color: #ef4444;
+    border-radius: 50%;
+    animation: pulse 2s infinite;
+}
+
+@keyframes pulse {
+    0% { box-shadow: 0 0 0 0 rgba(239, 68, 68, 0.7); }
+    70% { box-shadow: 0 0 0 10px rgba(239, 68, 68, 0); }
+    100% { box-shadow: 0 0 0 0 rgba(239, 68, 68, 0); }
+}
+
+.feed-container {
+    display: flex;
+    flex-direction: column;
+    gap: 1rem;
+    max-height: 400px;
+    overflow-y: auto;
+    padding-right: 0.5rem;
+}
+
+.feed-container::-webkit-scrollbar {
+    width: 6px;
+}
+
+.feed-container::-webkit-scrollbar-track {
+    background: transparent;
+}
+
+.feed-container::-webkit-scrollbar-thumb {
+    background: var(--border);
+    border-radius: 3px;
+}
+
+.tweet {
+    background: var(--bg-color);
+    border: 1px solid var(--border);
+    padding: 1rem;
+    border-radius: 0.75rem;
+    animation: slideInRight 0.3s ease-out;
+}
+
+@keyframes slideInRight {
+    from { opacity: 0; transform: translateX(20px); }
+    to { opacity: 1; transform: translateX(0); }
+}
+
+.tweet-header {
+    display: flex;
+    justify-content: space-between;
+    margin-bottom: 0.5rem;
+    font-size: 0.875rem;
+}
+
+.tweet-user {
+    font-weight: 600;
+    color: var(--primary);
+}
+
+.tweet-time {
+    color: var(--text-secondary);
+}
+
+.tweet-text {
+    font-size: 1rem;
+    line-height: 1.5;
+    direction: auto;
+}
+
+.tweet-tags {
+    display: flex;
+    gap: 0.5rem;
+    margin-top: 0.75rem;
+}
+
+.tag {
+    font-size: 0.75rem;
+    padding: 0.1rem 0.5rem;
+    border-radius: 99px;
+    background: var(--surface);
+    border: 1px solid var(--border);
+}
+
+.tag.pos { color: var(--positive); border-color: var(--positive); }
+.tag.neg { color: var(--negative); border-color: var(--negative); }
+.tag.neu { color: var(--neutral); border-color: var(--neutral); }
+
+/* Keywords */
+.tags-cloud {
+    display: flex;
+    flex-wrap: wrap;
+    gap: 0.5rem;
+}
+
+.keyword {
+    background: rgba(99, 102, 241, 0.1);
+    color: var(--text-primary);
+    padding: 0.4rem 0.8rem;
+    border-radius: 0.5rem;
+    font-size: 0.875rem;
+    cursor: pointer;
+    transition: background 0.2s, transform 0.2s;
+    border: 1px solid rgba(99, 102, 241, 0.2);
+}
+
+.keyword:hover {
+    background: rgba(99, 102, 241, 0.2);
+    transform: scale(1.05);
+}

test_models.py

@@ -0,0 +1,63 @@
+import os 
+import torch 
+import numpy as np 
+from transformers import AutoTokenizer ,AutoModelForSequenceClassification 
+
+def main ():
+    print ("="*60 )
+    print ("  Loading Urdu Sentiment & Emotion Models...")
+    print ("="*60 )
+
+    base_dir =os .path .dirname (os .path .abspath (__file__ ))
+    sentiment_dir =os .path .join (base_dir ,"models","sentiment_model")
+    emotion_dir =os .path .join (base_dir ,"models","emotion_model")
+
+
+
+    sentiment_map ={0 :"Positive 😊",1 :"Negative 😠",2 :"Neutral 😐"}
+    emotion_map ={0 :"Joy 😄",1 :"Anger 😡",2 :"Fear 😨",3 :"Sadness 😢"}
+
+
+    try :
+        print ("Loading Tokenizer...")
+        tokenizer =AutoTokenizer .from_pretrained (sentiment_dir )
+
+        print ("Loading Sentiment Model...")
+        sentiment_model =AutoModelForSequenceClassification .from_pretrained (sentiment_dir )
+
+        print ("Loading Emotion Model...")
+        emotion_model =AutoModelForSequenceClassification .from_pretrained (emotion_dir )
+    except Exception as e :
+        print (f"Error loading models. Are you sure they finished training? ({e })")
+        return 
+
+    print ("\n✅ Models loaded successfully!")
+    print ("Type an Urdu sentence (Roman or Script) to test them. Type 'exit' to quit.\n")
+
+
+    while True :
+        text =input ("Enter Urdu text: ")
+        if text .strip ().lower ()in ['exit','quit','q']:
+            break 
+        if not text .strip ():
+            continue 
+
+
+        inputs =tokenizer (text ,return_tensors ="pt",truncation =True ,max_length =128 )
+
+
+        with torch .no_grad ():
+            sentiment_out =sentiment_model (**inputs ).logits 
+            emotion_out =emotion_model (**inputs ).logits 
+
+
+        sentiment_idx =np .argmax (sentiment_out .numpy (),axis =-1 )[0 ]
+        emotion_idx =np .argmax (emotion_out .numpy (),axis =-1 )[0 ]
+
+        print ("-"*40 )
+        print (f"Sentiment : {sentiment_map [sentiment_idx ]}")
+        print (f"Emotion   : {emotion_map [emotion_idx ]}")
+        print ("-"*40 +"\n")
+
+if __name__ =="__main__":
+    main ()

training/dataset.py

@@ -0,0 +1,91 @@
+import torch 
+from torch .utils .data import Dataset 
+import pandas as pd 
+
+
+class UrduTextDataset (Dataset ):
+    def __init__ (self ,csv_paths ,tokenizer ,max_length =128 ,task ="sentiment"):
+        """
+        Custom PyTorch Dataset for Urdu Sentiment and Emotion Analysis.
+
+        NOTE: Tokenization here does NOT pad. Padding is applied per-batch by a
+        DataCollatorWithPadding in the training script (dynamic padding), which
+        is much faster than padding every sample to max_length -- especially on
+        CPU, where wasted FLOPs on padding tokens dominate runtime.
+
+        Args:
+            csv_paths (list or str): Path(s) to the cleaned CSV files.
+            tokenizer: HuggingFace tokenizer (XLM-RoBERTa).
+            max_length (int): Max token length for truncation.
+            task (str): 'sentiment' or 'emotion'.
+        """
+        self .tokenizer =tokenizer 
+        self .max_length =max_length 
+        self .task =task 
+
+        if isinstance (csv_paths ,str ):
+            csv_paths =[csv_paths ]
+
+
+        dfs =[pd .read_csv (path )for path in csv_paths ]
+        self .data =pd .concat (dfs ,ignore_index =True )
+
+        before =len (self .data )
+
+        self .data =self .data .dropna (subset =['text','label'])
+
+
+        if task =="sentiment":
+
+            label_map ={
+            'P':2 ,'O':1 ,'N':0 ,
+            2 :2 ,1 :1 ,0 :0 ,
+            '2':2 ,'1':1 ,'0':0 ,
+            }
+        elif task =="emotion":
+
+            label_map ={'joy':0 ,'anger':1 ,'fear':2 ,'sadness':3 }
+        else :
+            raise ValueError ("Task must be either 'sentiment' or 'emotion'")
+
+        self .data ['label']=self .data ['label'].map (label_map )
+
+
+        unmapped =int (self .data ['label'].isna ().sum ())
+        self .data =self .data .dropna (subset =['label'])
+        self .data ['label']=self .data ['label'].astype (int )
+        after =len (self .data )
+        print (f"  [{task }] kept {after }/{before } rows "
+        f"({before -after } dropped, {unmapped } had invalid labels)")
+        print (f"  [{task }] class counts: "
+        f"{self .data ['label'].value_counts ().sort_index ().to_dict ()}")
+
+        self .texts =self .data ['text'].astype (str ).tolist ()
+        self .labels =self .data ['label'].tolist ()
+
+    def __len__ (self ):
+        return len (self .texts )
+
+    def __getitem__ (self ,idx ):
+
+
+        encoding =self .tokenizer (
+        self .texts [idx ],
+        add_special_tokens =True ,
+        max_length =self .max_length ,
+        truncation =True ,
+        )
+        return {
+        'input_ids':encoding ['input_ids'],
+        'attention_mask':encoding ['attention_mask'],
+        'labels':torch .tensor (self .labels [idx ],dtype =torch .long ),
+        }
+
+    def get_class_weights (self ,num_labels ):
+        """Inverse-frequency class weights for a weighted loss (handles imbalance)."""
+        counts =self .data ['label'].value_counts ().sort_index ()
+        counts =counts .reindex (range (num_labels ),fill_value =0 )
+        total =counts .sum ()
+
+        weights =total /(num_labels *counts .replace (0 ,1 ))
+        return torch .tensor (weights .values ,dtype =torch .float )

training/phase3_verify.py

@@ -0,0 +1,64 @@
+import os 
+import sys 
+
+
+sys .path .append (os .path .dirname (os .path .abspath (__file__ )))
+
+from transformers import AutoTokenizer 
+from dataset import UrduTextDataset 
+from torch .utils .data import DataLoader 
+
+def main ():
+    print ("Loading XLM-RoBERTa Tokenizer...")
+    tokenizer =AutoTokenizer .from_pretrained ("xlm-roberta-base")
+
+    data_dir =os .path .join (os .path .dirname (os .path .dirname (os .path .abspath (__file__ ))),'data')
+
+    print ("\n--- Testing Sentiment Dataset ---")
+    sentiment_train_files =[
+    os .path .join (data_dir ,'roman_urdu_sentiment_train.csv'),
+    os .path .join (data_dir ,'urdu_sentiment_corpus_train.csv')
+    ]
+
+    sentiment_dataset =UrduTextDataset (
+    csv_paths =sentiment_train_files ,
+    tokenizer =tokenizer ,
+    max_length =128 ,
+    task ="sentiment"
+    )
+
+    print (f"Total Sentiment Training Samples: {len (sentiment_dataset )}")
+
+
+    sample =sentiment_dataset [0 ]
+    print (f"\nSample Text: {sample ['text']}")
+    print (f"Sample Label: {sample ['label']} (0=Neg, 1=Neu, 2=Pos)")
+    print (f"Input IDs shape: {sample ['input_ids'].shape }")
+    print (f"Attention Mask shape: {sample ['attention_mask'].shape }")
+
+
+    loader =DataLoader (sentiment_dataset ,batch_size =4 ,shuffle =True )
+    batch =next (iter (loader ))
+    print (f"\nBatch Input IDs shape: {batch ['input_ids'].shape }")
+    print (f"Batch Labels: {batch ['label']}")
+
+    print ("\n--- Testing Emotion Dataset ---")
+    emotion_train_files =[
+    os .path .join (data_dir ,'semeval_emotion_train.csv')
+    ]
+
+    emotion_dataset =UrduTextDataset (
+    csv_paths =emotion_train_files ,
+    tokenizer =tokenizer ,
+    max_length =128 ,
+    task ="emotion"
+    )
+
+    print (f"Total Emotion Training Samples: {len (emotion_dataset )}")
+    emotion_sample =emotion_dataset [0 ]
+    print (f"Sample Label: {emotion_sample ['label']} (0=Joy, 1=Anger, 2=Fear, 3=Sadness)")
+
+    print ("\nPhase 3 Verification Successful! Dataset class and tokenization work perfectly.")
+
+if __name__ =="__main__":
+    main ()

training/train_emotion.py

@@ -0,0 +1,142 @@
+import os 
+import sys 
+import numpy as np 
+import torch 
+import torch .nn as nn 
+from transformers import (
+AutoTokenizer ,AutoModelForSequenceClassification ,
+Trainer ,TrainingArguments ,DataCollatorWithPadding ,
+EarlyStoppingCallback ,
+)
+from sklearn .metrics import accuracy_score ,f1_score ,precision_recall_fscore_support 
+
+
+sys .path .append (os .path .dirname (os .path .abspath (__file__ )))
+from dataset import UrduTextDataset 
+
+NUM_LABELS =4 
+MAX_LENGTH =128 
+
+
+def compute_metrics (eval_pred ):
+    logits ,labels =eval_pred 
+    preds =np .argmax (logits ,axis =-1 )
+    precision ,recall ,f1 ,_ =precision_recall_fscore_support (
+    labels ,preds ,average ='weighted',zero_division =0 )
+    macro_f1 =f1_score (labels ,preds ,average ='macro',zero_division =0 )
+    return {
+    'accuracy':accuracy_score (labels ,preds ),
+    'f1':f1 ,
+    'macro_f1':macro_f1 ,
+    'precision':precision ,
+    'recall':recall ,
+    }
+
+
+class WeightedTrainer (Trainer ):
+    """Trainer that applies class weights (+ optional label smoothing) in the loss."""
+    def __init__ (self ,*args ,class_weights =None ,label_smoothing =0.0 ,**kwargs ):
+        super ().__init__ (*args ,**kwargs )
+        self .class_weights =class_weights 
+        self .label_smoothing =label_smoothing 
+
+    def compute_loss (self ,model ,inputs ,return_outputs =False ,**kwargs ):
+        labels =inputs .pop ("labels")
+        outputs =model (**inputs )
+        weight =(self .class_weights .to (model .device )
+        if self .class_weights is not None else None )
+        loss_fn =nn .CrossEntropyLoss (
+        weight =weight ,label_smoothing =self .label_smoothing )
+        loss =loss_fn (outputs .logits .view (-1 ,NUM_LABELS ),labels .view (-1 ))
+        return (loss ,outputs )if return_outputs else loss 
+
+
+def main ():
+    print ("="*60 )
+    print ("  Phase 4: Training Emotion Model")
+    print ("="*60 )
+
+
+    torch .set_num_threads (os .cpu_count ()or 1 )
+
+    model_name ="xlm-roberta-base"
+
+
+    print ("\n[1/5] Loading tokenizer and XLM-RoBERTa model...")
+    tokenizer =AutoTokenizer .from_pretrained (model_name )
+    model =AutoModelForSequenceClassification .from_pretrained (
+    model_name ,num_labels =NUM_LABELS )
+
+
+    print ("\n[2/5] Loading datasets...")
+    base =os .path .dirname (os .path .dirname (os .path .abspath (__file__ )))
+    data_dir =os .path .join (base ,'data')
+    train_files =[os .path .join (data_dir ,'semeval_emotion_train.csv')]
+    val_files =[os .path .join (data_dir ,'semeval_emotion_val.csv')]
+
+    train_dataset =UrduTextDataset (train_files ,tokenizer ,max_length =MAX_LENGTH ,task ="emotion")
+    val_dataset =UrduTextDataset (val_files ,tokenizer ,max_length =MAX_LENGTH ,task ="emotion")
+    print (f"Train samples: {len (train_dataset )} | Val samples: {len (val_dataset )}")
+
+
+    class_weights =train_dataset .get_class_weights (NUM_LABELS )
+    print (f"Class weights: {class_weights .tolist ()}")
+    data_collator =DataCollatorWithPadding (tokenizer =tokenizer )
+
+
+    print ("\n[3/5] Setting up Training Arguments...")
+    output_dir =os .path .join (base ,'models','emotion_model')
+
+    training_args =TrainingArguments (
+    output_dir =output_dir ,
+    num_train_epochs =6 ,
+    per_device_train_batch_size =8 ,
+    per_device_eval_batch_size =16 ,
+    gradient_accumulation_steps =4 ,
+    learning_rate =2e-5 ,
+    warmup_ratio =0.1 ,
+    weight_decay =0.01 ,
+    lr_scheduler_type ="cosine",
+    max_grad_norm =1.0 ,
+
+    dataloader_pin_memory =False ,
+    dataloader_num_workers =0 ,
+    logging_dir ='./logs/emotion',
+    logging_steps =50 ,
+    eval_strategy ="epoch",
+    save_strategy ="epoch",
+    save_total_limit =2 ,
+    load_best_model_at_end =True ,
+    metric_for_best_model ="macro_f1",
+    greater_is_better =True ,
+    seed =42 ,
+    report_to ="none",
+    )
+
+
+    print ("\n[4/5] Initializing Trainer...")
+    trainer =WeightedTrainer (
+    model =model ,
+    args =training_args ,
+    train_dataset =train_dataset ,
+    eval_dataset =val_dataset ,
+    processing_class =tokenizer ,
+    data_collator =data_collator ,
+    compute_metrics =compute_metrics ,
+    class_weights =class_weights ,
+    label_smoothing =0.1 ,
+    callbacks =[EarlyStoppingCallback (early_stopping_patience =2 )],
+    )
+
+
+    print ("\n[5/5] Starting training loop...")
+    trainer .train ()
+
+    print (f"\nTraining complete! Saving best model to {output_dir }")
+    trainer .save_model (output_dir )
+    tokenizer .save_pretrained (output_dir )
+    print ("Emotion model saved successfully!")
+
+
+if __name__ =="__main__":
+    main ()

training/train_sentiment.py

@@ -0,0 +1,148 @@
+import os 
+import sys 
+import numpy as np 
+import torch 
+import torch .nn as nn 
+from transformers import (
+AutoTokenizer ,AutoModelForSequenceClassification ,
+Trainer ,TrainingArguments ,DataCollatorWithPadding ,
+EarlyStoppingCallback ,
+)
+from sklearn .metrics import accuracy_score ,f1_score ,precision_recall_fscore_support 
+
+
+sys .path .append (os .path .dirname (os .path .abspath (__file__ )))
+from dataset import UrduTextDataset 
+
+NUM_LABELS =3 
+MAX_LENGTH =128 
+
+
+def compute_metrics (eval_pred ):
+    logits ,labels =eval_pred 
+    preds =np .argmax (logits ,axis =-1 )
+    precision ,recall ,f1 ,_ =precision_recall_fscore_support (
+    labels ,preds ,average ='weighted',zero_division =0 )
+    macro_f1 =f1_score (labels ,preds ,average ='macro',zero_division =0 )
+    return {
+    'accuracy':accuracy_score (labels ,preds ),
+    'f1':f1 ,
+    'macro_f1':macro_f1 ,
+    'precision':precision ,
+    'recall':recall ,
+    }
+
+
+class WeightedTrainer (Trainer ):
+    """Trainer that applies class weights (+ optional label smoothing) in the loss."""
+    def __init__ (self ,*args ,class_weights =None ,label_smoothing =0.0 ,**kwargs ):
+        super ().__init__ (*args ,**kwargs )
+        self .class_weights =class_weights 
+        self .label_smoothing =label_smoothing 
+
+    def compute_loss (self ,model ,inputs ,return_outputs =False ,**kwargs ):
+        labels =inputs .pop ("labels")
+        outputs =model (**inputs )
+        weight =(self .class_weights .to (model .device )
+        if self .class_weights is not None else None )
+        loss_fn =nn .CrossEntropyLoss (
+        weight =weight ,label_smoothing =self .label_smoothing )
+        loss =loss_fn (outputs .logits .view (-1 ,NUM_LABELS ),labels .view (-1 ))
+        return (loss ,outputs )if return_outputs else loss 
+
+
+def main ():
+    print ("="*60 )
+    print ("  Phase 4: Training Sentiment Model")
+    print ("="*60 )
+
+
+    torch .set_num_threads (os .cpu_count ()or 1 )
+
+    model_name ="xlm-roberta-base"
+
+
+    print ("\n[1/5] Loading tokenizer and XLM-RoBERTa model...")
+    tokenizer =AutoTokenizer .from_pretrained (model_name )
+    model =AutoModelForSequenceClassification .from_pretrained (
+    model_name ,num_labels =NUM_LABELS )
+
+
+    print ("\n[2/5] Loading datasets...")
+    base =os .path .dirname (os .path .dirname (os .path .abspath (__file__ )))
+    data_dir =os .path .join (base ,'data')
+    train_files =[
+    os .path .join (data_dir ,'roman_urdu_sentiment_train.csv'),
+    os .path .join (data_dir ,'urdu_sentiment_corpus_train.csv'),
+    ]
+    val_files =[
+    os .path .join (data_dir ,'roman_urdu_sentiment_val.csv'),
+    os .path .join (data_dir ,'urdu_sentiment_corpus_val.csv'),
+    ]
+
+    train_dataset =UrduTextDataset (train_files ,tokenizer ,max_length =MAX_LENGTH ,task ="sentiment")
+    val_dataset =UrduTextDataset (val_files ,tokenizer ,max_length =MAX_LENGTH ,task ="sentiment")
+    print (f"Train samples: {len (train_dataset )} | Val samples: {len (val_dataset )}")
+
+
+    class_weights =train_dataset .get_class_weights (NUM_LABELS )
+    print (f"Class weights: {class_weights .tolist ()}")
+    data_collator =DataCollatorWithPadding (tokenizer =tokenizer )
+
+
+    print ("\n[3/5] Setting up Training Arguments...")
+    output_dir =os .path .join (base ,'models','sentiment_model')
+
+    training_args =TrainingArguments (
+    output_dir =output_dir ,
+    num_train_epochs =6 ,
+    per_device_train_batch_size =8 ,
+    per_device_eval_batch_size =16 ,
+    gradient_accumulation_steps =4 ,
+    learning_rate =2e-5 ,
+    warmup_ratio =0.1 ,
+    weight_decay =0.01 ,
+    lr_scheduler_type ="cosine",
+    max_grad_norm =1.0 ,
+
+    dataloader_pin_memory =False ,
+    dataloader_num_workers =0 ,
+    logging_dir ='./logs/sentiment',
+    logging_steps =50 ,
+    eval_strategy ="epoch",
+    save_strategy ="epoch",
+    save_total_limit =2 ,
+    load_best_model_at_end =True ,
+    metric_for_best_model ="macro_f1",
+    greater_is_better =True ,
+    seed =42 ,
+    report_to ="none",
+    )
+
+
+    print ("\n[4/5] Initializing Trainer...")
+    trainer =WeightedTrainer (
+    model =model ,
+    args =training_args ,
+    train_dataset =train_dataset ,
+    eval_dataset =val_dataset ,
+    processing_class =tokenizer ,
+    data_collator =data_collator ,
+    compute_metrics =compute_metrics ,
+    class_weights =class_weights ,
+    label_smoothing =0.1 ,
+    callbacks =[EarlyStoppingCallback (early_stopping_patience =2 )],
+    )
+
+
+    print ("\n[5/5] Starting training loop...")
+    trainer .train ()
+
+    print (f"\nTraining complete! Saving best model to {output_dir }")
+    trainer .save_model (output_dir )
+    tokenizer .save_pretrained (output_dir )
+    print ("Sentiment model saved successfully!")
+
+
+if __name__ =="__main__":
+    main ()