Add source code

annotation/active_learning/human_feedback_loop.py

@@ -0,0 +1,338 @@
+"""Human feedback loop for active learning.
+
+Manages the cycle of:
+1. Model prediction
+2. Uncertainty sampling
+3. Human annotation
+4. Model retraining
+"""
+
+import json
+import logging
+from dataclasses import dataclass, field
+from datetime import datetime
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Tuple
+
+import pandas as pd
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class FeedbackRecord:
+    """Record of human feedback for a sample."""
+    sample_id: str
+    text: str
+    original_prediction: str
+    human_label: str
+    confidence_feedback: float  # 0-1, did model seem confident?
+    notes: str = ""
+    timestamp: str = ""
+    
+    def to_dict(self) -> Dict:
+        return {
+            "sample_id": self.sample_id,
+            "text": self.text,
+            "original_prediction": self.original_prediction,
+            "human_label": self.human_label,
+            "confidence_feedback": self.confidence_feedback,
+            "notes": self.notes,
+            "timestamp": self.timestamp or datetime.now().isoformat(),
+        }
+
+
+@dataclass
+class FeedbackLoopConfig:
+    """Configuration for feedback loop."""
+    min_feedback_samples: int = 50
+    max_feedback_samples: int = 500
+    retrain_threshold: int = 100  # Retrain after this many new samples
+    disagreement_threshold: float = 0.3  # Retrain if disagreement rate > this
+    batch_size: int = 32
+    
+    
+@dataclass 
+class LoopState:
+    """State of the feedback loop."""
+    iteration: int = 0
+    total_annotated: int = 0
+    total_retrained: int = 0
+    disagreement_rate: float = 0.0
+    model_performance: Dict = field(default_factory=dict)
+    history: List[Dict] = field(default_factory=list)
+
+
+class HumanFeedbackLoop:
+    """Manages the human-in-the-loop training cycle."""
+    
+    def __init__(
+        self,
+        config: Optional[FeedbackLoopConfig] = None,
+        output_dir: str = "outputs/active_learning",
+    ):
+        self.config = config or FeedbackLoopConfig()
+        self.output_dir = Path(output_dir)
+        self.output_dir.mkdir(parents=True, exist_ok=True)
+        
+        self.state = LoopState()
+        self.feedback_records: List[FeedbackRecord] = []
+        self.labeled_samples: List[Dict] = []
+        
+    def add_feedback(
+        self,
+        sample_id: str,
+        text: str,
+        original_prediction: str,
+        human_label: str,
+        confidence_feedback: float = 0.5,
+        notes: str = "",
+    ) -> None:
+        """Add human feedback for a sample."""
+        record = FeedbackRecord(
+            sample_id=sample_id,
+            text=text,
+            original_prediction=original_prediction,
+            human_label=human_label,
+            confidence_feedback=confidence_feedback,
+            notes=notes,
+            timestamp=datetime.now().isoformat(),
+        )
+        
+        self.feedback_records.append(record)
+        
+        # Add to labeled samples
+        self.labeled_samples.append({
+            "id": sample_id,
+            "text": text,
+            "label": human_label,
+            "source": "human_feedback",
+        })
+        
+        self.state.total_annotated += 1
+        
+        logger.info(
+            f"Added feedback for {sample_id}: "
+            f"{original_prediction} -> {human_label}"
+        )
+    
+    def batch_add_feedback(
+        self,
+        feedback_list: List[Dict],
+    ) -> None:
+        """Add multiple feedback records at once."""
+        for fb in feedback_list:
+            self.add_feedback(
+                sample_id=fb.get("sample_id", fb.get("id")),
+                text=fb.get("text", ""),
+                original_prediction=fb.get("original_prediction", "unknown"),
+                human_label=fb.get("human_label", fb.get("label")),
+                confidence_feedback=fb.get("confidence_feedback", 0.5),
+                notes=fb.get("notes", ""),
+            )
+    
+    def should_retrain(self) -> Tuple[bool, str]:
+        """Check if model should be retrained.
+        
+        Returns:
+            (should_retrain, reason)
+        """
+        n_new = len(self.feedback_records)
+        
+        # Check minimum samples
+        if n_new < self.config.min_feedback_samples:
+            return False, f"Only {n_new} samples (min: {self.config.min_feedback_samples})"
+        
+        # Check retrain threshold
+        if n_new >= self.config.retrain_threshold:
+            self._calculate_disagreement_rate()
+            if self.state.disagreement_rate > self.config.disagreement_threshold:
+                return True, f"High disagreement ({self.state.disagreement_rate:.1%})"
+            return True, f"Reached {n_new} samples threshold"
+        
+        return False, f"Not enough samples: {n_new}"
+    
+    def _calculate_disagreement_rate(self) -> float:
+        """Calculate disagreement rate between model and human."""
+        if not self.feedback_records:
+            self.state.disagreement_rate = 0.0
+            return 0.0
+        
+        disagreements = sum(
+            1 for r in self.feedback_records
+            if r.original_prediction != r.human_label
+        )
+        
+        self.state.disagreement_rate = disagreements / len(self.feedback_records)
+        return self.state.disagreement_rate
+    
+    def get_training_data(
+        self,
+        include_previous: bool = True,
+    ) -> List[Dict]:
+        """Get accumulated training data.
+        
+        Args:
+            include_previous: Include previously retrained data
+        
+        Returns:
+            List of samples with labels
+        """
+        if include_previous:
+            return self.labeled_samples
+        else:
+            # Only return new samples since last retrain
+            return self.labeled_samples[-self.config.retrain_threshold:]
+    
+    def export_training_data(
+        self,
+        path: Optional[str] = None,
+        format: str = "jsonl",
+    ) -> str:
+        """Export training data to file."""
+        if path is None:
+            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+            path = self.output_dir / f"training_data_{timestamp}.{format}"
+        
+        if format == "jsonl":
+            with open(path, "w", encoding="utf-8") as f:
+                for sample in self.labeled_samples:
+                    f.write(json.dumps(sample, ensure_ascii=False) + "\n")
+        elif format == "csv":
+            df = pd.DataFrame(self.labeled_samples)
+            df.to_csv(path, index=False)
+        
+        logger.info(f"Exported {len(self.labeled_samples)} samples to {path}")
+        return str(path)
+    
+    def mark_retrained(self, performance: Optional[Dict] = None) -> None:
+        """Mark that retraining has occurred."""
+        self.state.iteration += 1
+        self.state.total_retrained += 1
+        
+        if performance:
+            self.state.model_performance = performance
+        
+        # Record history
+        self.history.append({
+            "iteration": self.state.iteration,
+            "timestamp": datetime.now().isoformat(),
+            "total_annotated": self.state.total_annotated,
+            "disagreement_rate": self.state.disagreement_rate,
+            "performance": performance,
+        })
+        
+        logger.info(
+            f"Model retrained (iteration {self.state.iteration}). "
+            f"Total annotated: {self.state.total_annotated}"
+        )
+    
+    def get_statistics(self) -> Dict[str, Any]:
+        """Get loop statistics."""
+        return {
+            "iteration": self.state.iteration,
+            "total_annotated": self.state.total_annotated,
+            "total_retrained": self.state.total_retrained,
+            "disagreement_rate": self.state.disagreement_rate,
+            "should_retrain": self.should_retrain()[0],
+            "pending_samples": len(self.feedback_records),
+            "recent_history": self.history[-5:] if self.history else [],
+        }
+    
+    def get_label_distribution(self) -> Dict[str, int]:
+        """Get distribution of labels."""
+        from collections import Counter
+        labels = [r.human_label for r in self.feedback_records]
+        return dict(Counter(labels))
+    
+    def analyze_errors(self) -> Dict[str, Any]:
+        """Analyze patterns in model errors."""
+        errors = [
+            r for r in self.feedback_records
+            if r.original_prediction != r.human_label
+        ]
+        
+        if not errors:
+            return {"total_errors": 0}
+        
+        # Group by confusion pairs
+        confusion_pairs = {}
+        for e in errors:
+            pair = (e.original_prediction, e.human_label)
+            confusion_pairs[pair] = confusion_pairs.get(pair, 0) + 1
+        
+        return {
+            "total_errors": len(errors),
+            "error_rate": len(errors) / len(self.feedback_records),
+            "confusion_matrix": confusion_pairs,
+            "most_common_error": max(
+                confusion_pairs.items(),
+                key=lambda x: x[1]
+            ) if confusion_pairs else None,
+        }
+    
+    def save_state(self, path: Optional[str] = None) -> str:
+        """Save loop state to file."""
+        if path is None:
+            path = self.output_dir / "loop_state.json"
+        
+        state_data = {
+            "config": {
+                "min_feedback_samples": self.config.min_feedback_samples,
+                "max_feedback_samples": self.config.max_feedback_samples,
+                "retrain_threshold": self.config.retrain_threshold,
+                "disagreement_threshold": self.config.disagreement_threshold,
+            },
+            "state": {
+                "iteration": self.state.iteration,
+                "total_annotated": self.state.total_annotated,
+                "total_retrained": self.state.total_retrained,
+                "disagreement_rate": self.state.disagreement_rate,
+            },
+            "history": self.history,
+        }
+        
+        with open(path, "w", encoding="utf-8") as f:
+            json.dump(state_data, f, indent=2)
+        
+        return str(path)
+    
+    def load_state(self, path: str) -> None:
+        """Load loop state from file."""
+        with open(path, "r", encoding="utf-8") as f:
+            state_data = json.load(f)
+        
+        config_dict = state_data.get("config", {})
+        self.config = FeedbackLoopConfig(**config_dict)
+        
+        state_dict = state_data.get("state", {})
+        self.state = LoopState(**state_dict)
+        
+        self.history = state_data.get("history", [])
+
+
+def create_feedback_loop(
+    config: Optional[Dict] = None,
+) -> HumanFeedbackLoop:
+    """Factory function to create feedback loop."""
+    loop_config = None
+    if config:
+        loop_config = FeedbackLoopConfig(**config)
+    
+    return HumanFeedbackLoop(config=loop_config)
+
+
+if __name__ == "__main__":
+    loop = create_feedback_loop()
+    
+    # Simulate feedback
+    loop.add_feedback(
+        sample_id="utt_001",
+        text="ကျေးဇူးပါ",
+        original_prediction="positive",
+        human_label="sarcastic",
+        notes="Voice tone suggests complaint",
+    )
+    
+    print(f"Should retrain: {loop.should_retrain()}")
+    print(f"Stats: {loop.get_statistics()}")

annotation/active_learning/uncertainty_sampler.py

@@ -0,0 +1,342 @@
+"""Uncertainty sampling for active learning.
+
+Selects samples where the model has lowest confidence,
+indicating areas where human annotation would be most valuable.
+"""
+
+import json
+import logging
+from dataclasses import dataclass
+from enum import Enum
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Tuple
+
+import numpy as np
+import torch
+import torch.nn as nn
+from torch.utils.data import DataLoader, Dataset
+from tqdm import tqdm
+
+logger = logging.getLogger(__name__)
+
+
+class UncertaintyMethod(str, Enum):
+    """Methods for calculating uncertainty."""
+    LEAST_CONFIDENCE = "least_confidence"  # 1 - max probability
+    MARGIN = "margin"  # Difference between top 2 probabilities
+    ENTROPY = "entropy"  # Shannon entropy
+    RATIO = "ratio"  # Ratio of top to second probability
+    VARIANCE = "variance"  # Prediction variance (ensemble)
+
+
+@dataclass
+class UncertaintySample:
+    """Sample with uncertainty score."""
+    sample_id: str
+    text: str
+    uncertainty_score: float
+    predicted_class: str
+    predicted_prob: float
+    second_prob: float = 0.0
+    metadata: Dict = None
+    
+    def to_dict(self) -> Dict:
+        return {
+            "sample_id": self.sample_id,
+            "text": self.text,
+            "uncertainty_score": self.uncertainty_score,
+            "predicted_class": self.predicted_class,
+            "predicted_prob": self.predicted_prob,
+            "second_prob": self.second_prob,
+            "metadata": self.metadata or {},
+        }
+
+
+class PredictionDataset(Dataset):
+    """Dataset for model predictions."""
+    
+    def __init__(self, samples: List[Dict], tokenizer, max_length: int = 128):
+        self.samples = samples
+        self.tokenizer = tokenizer
+        self.max_length = max_length
+    
+    def __len__(self) -> int:
+        return len(self.samples)
+    
+    def __getitem__(self, idx: int) -> Tuple[torch.Tensor, torch.Tensor, str]:
+        sample = self.samples[idx]
+        text = sample.get("text", "")
+        
+        encoding = self.tokenizer(
+            text,
+            truncation=True,
+            max_length=self.max_length,
+            padding="max_length",
+            return_tensors="pt",
+        )
+        
+        return (
+            encoding["input_ids"].squeeze(0),
+            encoding["attention_mask"].squeeze(0),
+            sample.get("id", f"sample_{idx}"),
+        )
+
+
+class UncertaintySampler:
+    """Sample uncertain instances for active learning."""
+    
+    def __init__(
+        self,
+        model: nn.Module,
+        method: UncertaintyMethod = UncertaintyMethod.LEAST_CONFIDENCE,
+        device: str = "cuda" if torch.cuda.is_available() else "cpu",
+    ):
+        self.model = model
+        self.method = method
+        self.device = device
+        self.model.to(device)
+        self.model.eval()
+    
+    def _compute_uncertainty(
+        self,
+        logits: torch.Tensor,
+    ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
+        """Compute uncertainty scores from logits.
+        
+        Returns:
+            uncertainty: Uncertainty scores
+            predicted_class: Predicted class indices
+            probs: Probability distributions
+            second_probs: Second highest probabilities (for margin)
+        """
+        probs = torch.softmax(logits, dim=-1)
+        
+        if self.method == UncertaintyMethod.ENTROPY:
+            # Shannon entropy: -sum(p * log(p))
+            entropy = -torch.sum(probs * torch.log(probs + 1e-10), dim=-1)
+            uncertainty = entropy
+        elif self.method == UncertaintyMethod.LEAST_CONFIDENCE:
+            # 1 - max probability
+            max_prob, _ = probs.max(dim=-1)
+            uncertainty = 1 - max_prob
+        elif self.method == UncertaintyMethod.MARGIN:
+            # Difference between top 2 probabilities
+            sorted_probs, _ = torch.sort(probs, dim=-1, descending=True)
+            margin = sorted_probs[:, 0] - sorted_probs[:, 1]
+            uncertainty = 1 - margin
+        elif self.method == UncertaintyMethod.RATIO:
+            # Ratio of top to second probability
+            sorted_probs, _ = torch.sort(probs, dim=-1, descending=True)
+            ratio = sorted_probs[:, 1] / (sorted_probs[:, 0] + 1e-10)
+            uncertainty = ratio
+        else:
+            raise ValueError(f"Unknown method: {self.method}")
+        
+        predicted_class = probs.argmax(dim=-1)
+        max_probs = probs.max(dim=-1).values
+        
+        # Second highest probability
+        sorted_probs_detached = probs.detach().cpu()
+        sorted_indices = torch.argsort(sorted_probs_detached, dim=-1, descending=True)
+        second_probs = torch.gather(
+            probs, 1, sorted_indices[:, 1:2]
+        ).squeeze(-1)
+        
+        return uncertainty, predicted_class, max_probs, second_probs
+    
+    def score_samples(
+        self,
+        samples: List[Dict],
+        tokenizer,
+        batch_size: int = 32,
+    ) -> List[UncertaintySample]:
+        """Score samples by uncertainty."""
+        dataset = PredictionDataset(samples, tokenizer)
+        dataloader = DataLoader(dataset, batch_size=batch_size, shuffle=False)
+        
+        all_uncertainty = []
+        all_predicted = []
+        all_probs = []
+        all_second_probs = []
+        all_ids = []
+        
+        with torch.no_grad():
+            for input_ids, attention_mask, sample_ids in tqdm(
+                dataloader, desc="Computing uncertainty"
+            ):
+                input_ids = input_ids.to(self.device)
+                attention_mask = attention_mask.to(self.device)
+                
+                outputs = self.model(input_ids, attention_mask)
+                logits = outputs.logits if hasattr(outputs, "logits") else outputs
+                
+                uncertainty, pred_class, probs, second_probs = self._compute_uncertainty(logits)
+                
+                all_uncertainty.extend(uncertainty.cpu().tolist())
+                all_predicted.extend(pred_class.cpu().tolist())
+                all_probs.extend(probs.cpu().tolist())
+                all_second_probs.extend(second_probs.cpu().tolist())
+                all_ids.extend(sample_ids)
+        
+        # Create uncertainty samples
+        uncertain_samples = []
+        class_names = ["negative", "neutral", "positive", "sarcastic"]
+        
+        for i, sample in enumerate(samples):
+            us = UncertaintySample(
+                sample_id=all_ids[i],
+                text=sample.get("text", ""),
+                uncertainty_score=all_uncertainty[i],
+                predicted_class=class_names[all_predicted[i]] if all_predicted[i] < len(class_names) else "unknown",
+                predicted_prob=all_probs[i],
+                second_prob=all_second_probs[i],
+                metadata=sample,
+            )
+            uncertain_samples.append(us)
+        
+        return uncertain_samples
+    
+    def select_samples(
+        self,
+        samples: List[Dict],
+        tokenizer,
+        n_samples: int = 100,
+        batch_size: int = 32,
+        exclude_ids: Optional[List[str]] = None,
+    ) -> List[UncertaintySample]:
+        """Select most uncertain samples for annotation.
+        
+        Args:
+            samples: List of samples to score
+            tokenizer: Tokenizer for the model
+            n_samples: Number of samples to select
+            batch_size: Batch size for inference
+            exclude_ids: Sample IDs to exclude (already annotated)
+        
+        Returns:
+            List of selected uncertain samples, sorted by uncertainty
+        """
+        # Filter out already annotated
+        if exclude_ids:
+            samples = [s for s in samples if s.get("id") not in exclude_ids]
+        
+        # Score all samples
+        uncertain_samples = self.score_samples(samples, tokenizer, batch_size)
+        
+        # Sort by uncertainty (highest first)
+        uncertain_samples.sort(key=lambda x: x.uncertainty_score, reverse=True)
+        
+        # Select top n_samples
+        selected = uncertain_samples[:n_samples]
+        
+        logger.info(
+            f"Selected {len(selected)} most uncertain samples "
+            f"(uncertainty range: {selected[0].uncertainty_score:.4f} - "
+            f"{selected[-1].uncertainty_score:.4f})"
+        )
+        
+        return selected
+    
+    def diversity_sample(
+        self,
+        samples: List[Dict],
+        tokenizer,
+        n_samples: int = 100,
+        batch_size: int = 32,
+        n_clusters: int = 10,
+    ) -> List[UncertaintySample]:
+        """Select diverse uncertain samples using clustering.
+        
+        Combines uncertainty with diversity to avoid selecting
+        similar samples.
+        """
+        from sklearn.cluster import MiniBatchKMeans
+        from sklearn.feature_extraction.text import TfidfVectorizer
+        
+        # Score samples
+        uncertain_samples = self.score_samples(samples, tokenizer, batch_size)
+        
+        # Create embeddings for clustering
+        vectorizer = TfidfVectorizer(max_features=1000)
+        texts = [s.text for s in samples]
+        embeddings = vectorizer.fit_transform(texts)
+        
+        # Cluster
+        kmeans = MiniBatchKMeans(n_clusters=n_clusters, random_state=42)
+        cluster_labels = kmeans.fit_predict(embeddings)
+        
+        # Select from each cluster
+        selected = []
+        for cluster_id in range(n_clusters):
+            cluster_indices = [
+                i for i, label in enumerate(cluster_labels)
+                if label == cluster_id
+            ]
+            cluster_uncertain = [
+                uncertain_samples[i] for i in cluster_indices
+            ]
+            cluster_uncertain.sort(key=lambda x: x.uncertainty_score, reverse=True)
+            
+            # Take top samples from each cluster
+            n_per_cluster = max(1, n_samples // n_clusters)
+            selected.extend(cluster_uncertain[:n_per_cluster])
+        
+        # Sort by uncertainty
+        selected.sort(key=lambda x: x.uncertainty_score, reverse=True)
+        
+        return selected[:n_samples]
+    
+    def batch_sample(
+        self,
+        samples: List[Dict],
+        tokenizer,
+        strategy: str = "greedy",
+        n_samples: int = 100,
+        batch_size: int = 32,
+    ) -> List[UncertaintySample]:
+        """Sample using batch mode for efficiency.
+        
+        Strategies:
+            - greedy: Select top n_samples by uncertainty
+            - diverse: Cluster-based diverse sampling
+            - random: Random baseline
+        """
+        if strategy == "random":
+            import random
+            random.seed(42)
+            indices = random.sample(range(len(samples)), min(n_samples, len(samples)))
+            return [UncertaintySample(
+                sample_id=samples[i].get("id", f"sample_{i}"),
+                text=samples[i].get("text", ""),
+                uncertainty_score=0.0,
+                predicted_class="unknown",
+                predicted_prob=0.0,
+            ) for i in indices]
+        
+        elif strategy == "diverse":
+            return self.diversity_sample(
+                samples, tokenizer, n_samples, batch_size
+            )
+        
+        else:  # greedy
+            return self.select_samples(
+                samples, tokenizer, n_samples, batch_size
+            )
+
+
+def save_selected_samples(
+    samples: List[UncertaintySample],
+    output_path: str,
+) -> None:
+    """Save selected samples to JSON file."""
+    output_data = [s.to_dict() for s in samples]
+    
+    with open(output_path, "w", encoding="utf-8") as f:
+        json.dump(output_data, f, indent=2, ensure_ascii=False)
+    
+    logger.info(f"Saved {len(samples)} samples to {output_path}")
+
+
+if __name__ == "__main__":
+    print("UncertaintySampler module loaded")
+    print(f"Available methods: {[m.value for m in UncertaintyMethod]}")

annotation/automatic_verifier.py

@@ -0,0 +1,387 @@
+"""Automatic verification of annotations using rule-based checks."""
+
+import json
+import logging
+import re
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Tuple
+
+import pandas as pd
+import yaml
+
+logger = logging.getLogger(__name__)
+
+
+class VerificationRule:
+    """Base class for verification rules."""
+    
+    def __init__(self, rule_id: str, severity: str = "error"):
+        self.rule_id = rule_id
+        self.severity = severity  # error, warning, info
+    
+    def verify(self, sample: Dict) -> Tuple[bool, Optional[str]]:
+        """Verify a sample. Returns (passed, error_message)."""
+        raise NotImplementedError
+
+
+class TextLengthRule(VerificationRule):
+    """Check if text length is within acceptable range."""
+    
+    def __init__(self, min_length: int = 1, max_length: int = 500):
+        super().__init__("text_length")
+        self.min_length = min_length
+        self.max_length = max_length
+    
+    def verify(self, sample: Dict) -> Tuple[bool, Optional[str]]:
+        text = sample.get("text", "")
+        length = len(text.strip())
+        
+        if length < self.min_length:
+            return False, f"Text too short: {length} chars (min: {self.min_length})"
+        if length > self.max_length:
+            return False, f"Text too long: {length} chars (max: {self.max_length})"
+        
+        return True, None
+
+
+class SentimentConsistencyRule(VerificationRule):
+    """Check sentiment consistency between text and prosody."""
+    
+    # Sentiment keywords mapping
+    POSITIVE_KEYWORDS = ["ကျေးဇူး", "ပါး", "မင်္ဂလာ", "ဝမ်းသာ", "ပျော်"]
+    NEGATIVE_KEYWORDS = ["မကျေနပ်", "ဒေါသ", "စိတ်ဓာတ်ကျ", "ပူ", "ဆူ"]
+    
+    def __init__(self):
+        super().__init__("sentiment_consistency")
+    
+    def verify(self, sample: Dict) -> Tuple[bool, Optional[str]]:
+        text = sample.get("text", "")
+        prosody = sample.get("prosody", {})
+        
+        text_positive = any(kw in text for kw in self.POSITIVE_KEYWORDS)
+        text_negative = any(kw in text for kw in self.NEGATIVE_KEYWORDS)
+        
+        sentiment = sample.get("sentiment", "neutral")
+        
+        # Warning only (not error) - prosody and text can diverge
+        if sentiment == "positive" and text_negative and not text_positive:
+            return True, "Warning: Text has negative keywords but labeled positive"
+        if sentiment == "negative" and text_positive and not text_negative:
+            return True, "Warning: Text has positive keywords but labeled negative"
+        
+        return True, None
+
+
+class LabelDistributionRule(VerificationRule):
+    """Check if label distribution is reasonable."""
+    
+    def __init__(
+        self,
+        min_samples_per_class: int = 10,
+        max_imbalance_ratio: float = 10.0,
+    ):
+        super().__init__("label_distribution", severity="warning")
+        self.min_samples_per_class = min_samples_per_class
+        self.max_imbalance_ratio = max_imbalance_ratio
+    
+    def verify(self, samples: List[Dict]) -> Tuple[bool, Optional[str]]:
+        if not samples:
+            return False, "No samples provided"
+        
+        from collections import Counter
+        sentiments = [s.get("sentiment", "unknown") for s in samples]
+        counts = Counter(sentiments)
+        
+        if len(counts) == 0:
+            return False, "No sentiment labels found"
+        
+        # Check minimum samples per class
+        for sentiment, count in counts.items():
+            if count < self.min_samples_per_class:
+                return False, f"Class '{sentiment}' has only {count} samples (min: {self.min_samples_per_class})"
+        
+        # Check imbalance
+        max_count = max(counts.values())
+        min_count = min(counts.values())
+        if max_count / min_count > self.max_imbalance_ratio:
+            return True, f"Warning: Imbalance ratio {max_count/min_count:.1f} > {self.max_imbalance_ratio}"
+        
+        return True, None
+
+
+class DuplicateTextRule(VerificationRule):
+    """Check for duplicate text entries."""
+    
+    def __init__(self, threshold: float = 0.9):
+        super().__init__("duplicate_text", severity="warning")
+        self.threshold = threshold
+    
+    def verify(self, samples: List[Dict]) -> Tuple[bool, Optional[str]]:
+        texts = [s.get("text", "").strip().lower() for s in samples]
+        
+        duplicates = []
+        seen = {}
+        
+        for i, text in enumerate(texts):
+            if text in seen:
+                duplicates.append((seen[text], i))
+            else:
+                seen[text] = i
+        
+        if duplicates:
+            return True, f"Found {len(duplicates)} potential duplicates"
+        
+        return True, None
+
+
+class AutomaticVerifier:
+    """Verify annotations using rule-based checks."""
+    
+    def __init__(self, rules_config: Optional[str] = None):
+        self.rules: List[VerificationRule] = []
+        
+        if rules_config and Path(rules_config).exists():
+            self._load_config(rules_config)
+        else:
+            self._setup_default_rules()
+    
+    def _setup_default_rules(self) -> None:
+        """Set up default verification rules."""
+        self.rules = [
+            TextLengthRule(min_length=1, max_length=500),
+            SentimentConsistencyRule(),
+            LabelDistributionRule(),
+            DuplicateTextRule(),
+        ]
+    
+    def _load_config(self, config_path: str) -> None:
+        """Load rules from config file."""
+        with open(config_path, "r", encoding="utf-8") as f:
+            config = yaml.safe_load(f)
+        
+        self.rules = []
+        
+        for rule_def in config.get("rules", []):
+            rule_type = rule_def.get("type")
+            
+            if rule_type == "text_length":
+                self.rules.append(TextLengthRule(
+                    min_length=rule_def.get("min_length", 1),
+                    max_length=rule_def.get("max_length", 500),
+                ))
+            elif rule_type == "sentiment_consistency":
+                self.rules.append(SentimentConsistencyRule())
+            elif rule_type == "label_distribution":
+                self.rules.append(LabelDistributionRule(
+                    min_samples_per_class=rule_def.get("min_samples_per_class", 10),
+                    max_imbalance_ratio=rule_def.get("max_imbalance_ratio", 10.0),
+                ))
+            elif rule_type == "duplicate_text":
+                self.rules.append(DuplicateTextRule(
+                    threshold=rule_def.get("threshold", 0.9),
+                ))
+    
+    def verify_sample(self, sample: Dict) -> Dict[str, Any]:
+        """Verify a single sample against all rules."""
+        results = {
+            "sample_id": sample.get("id", "unknown"),
+            "passed": True,
+            "errors": [],
+            "warnings": [],
+        }
+        
+        for rule in self.rules:
+            if isinstance(rule, LabelDistributionRule) or isinstance(rule, DuplicateTextRule):
+                # These rules need full dataset
+                continue
+            
+            passed, message = rule.verify(sample)
+            
+            if not passed:
+                results["passed"] = False
+                if rule.severity == "error":
+                    results["errors"].append({
+                        "rule_id": rule.rule_id,
+                        "message": message,
+                    })
+                else:
+                    results["warnings"].append({
+                        "rule_id": rule.rule_id,
+                        "message": message,
+                    })
+            elif message:
+                results["warnings"].append({
+                    "rule_id": rule.rule_id,
+                    "message": message,
+                })
+        
+        return results
+    
+    def verify_dataset(
+        self,
+        samples: List[Dict],
+    ) -> Dict[str, Any]:
+        """Verify entire dataset."""
+        results = {
+            "total_samples": len(samples),
+            "sample_results": [],
+            "dataset_errors": [],
+            "statistics": {},
+        }
+        
+        # Check sample-level rules
+        for sample in samples:
+            sample_result = self.verify_sample(sample)
+            results["sample_results"].append(sample_result)
+        
+        # Check dataset-level rules
+        for rule in self.rules:
+            if isinstance(rule, (LabelDistributionRule, DuplicateTextRule)):
+                passed, message = rule.verify(samples)
+                if not passed:
+                    results["dataset_errors"].append({
+                        "rule_id": rule.rule_id,
+                        "severity": rule.severity,
+                        "message": message,
+                    })
+        
+        # Calculate statistics
+        total_errors = sum(
+            len(r.get("errors", []))
+            for r in results["sample_results"]
+        )
+        total_warnings = sum(
+            len(r.get("warnings", []))
+            for r in results["sample_results"]
+        )
+        
+        results["statistics"] = {
+            "total_errors": total_errors,
+            "total_warnings": total_warnings,
+            "samples_passed": sum(
+                1 for r in results["sample_results"] if r["passed"]
+            ),
+        }
+        
+        return results
+    
+    def filter_samples(
+        self,
+        samples: List[Dict],
+        remove_errors: bool = True,
+        remove_warnings: bool = False,
+    ) -> Tuple[List[Dict], List[Dict]]:
+        """Filter samples based on verification results."""
+        results = self.verify_dataset(samples)
+        
+        kept = []
+        removed = []
+        
+        for i, (sample, result) in enumerate(zip(samples, results["sample_results"])):
+            should_remove = False
+            
+            if remove_errors and result["errors"]:
+                should_remove = True
+            if remove_warnings and result["warnings"]:
+                should_remove = True
+            
+            if should_remove:
+                removed.append({
+                    "sample": sample,
+                    "reason": result,
+                })
+            else:
+                kept.append(sample)
+        
+        logger.info(
+            f"Filtered: {len(kept)} kept, {len(removed)} removed"
+        )
+        
+        return kept, removed
+    
+    def generate_report(
+        self,
+        samples: List[Dict],
+        output_path: Optional[str] = None,
+    ) -> str:
+        """Generate verification report."""
+        results = self.verify_dataset(samples)
+        
+        report_lines = [
+            "=" * 60,
+            "AUTOMATIC ANNOTATION VERIFICATION REPORT",
+            "=" * 60,
+            f"Total Samples: {results['total_samples']}",
+            f"Samples Passed: {results['statistics']['samples_passed']}",
+            f"Total Errors: {results['statistics']['total_errors']}",
+            f"Total Warnings: {results['statistics']['total_warnings']}",
+            "",
+            "-" * 60,
+            "DATASET-LEVEL ISSUES",
+            "-" * 60,
+        ]
+        
+        for error in results.get("dataset_errors", []):
+            report_lines.append(
+                f"[{error['severity'].upper()}] {error['rule_id']}: {error['message']}"
+            )
+        
+        if not results.get("dataset_errors"):
+            report_lines.append("No dataset-level issues found.")
+        
+        report_lines.extend([
+            "",
+            "-" * 60,
+            "SAMPLE-LEVEL ISSUES",
+            "-" * 60,
+        ])
+        
+        error_count = 0
+        for result in results["sample_results"]:
+            if result["errors"] or result["warnings"]:
+                error_count += 1
+                report_lines.append(f"\nSample: {result['sample_id']}")
+                for error in result["errors"]:
+                    report_lines.append(f"  ERROR: {error['message']}")
+                for warning in result["warnings"]:
+                    report_lines.append(f"  WARNING: {warning['message']}")
+                
+                if error_count >= 20:
+                    report_lines.append("\n... (showing first 20 samples with issues)")
+                    break
+        
+        report = "\n".join(report_lines)
+        
+        if output_path:
+            with open(output_path, "w", encoding="utf-8") as f:
+                f.write(report)
+            logger.info(f"Report saved to {output_path}")
+        
+        return report
+
+
+def create_verifier(config_path: Optional[str] = None) -> AutomaticVerifier:
+    """Factory function to create verifier."""
+    return AutomaticVerifier(rules_config=config_path)
+
+
+if __name__ == "__main__":
+    verifier = create_verifier()
+    
+    # Test samples
+    test_samples = [
+        {
+            "id": "utt_001",
+            "text": "ကျေးဇူးပါ",
+            "sentiment": "positive",
+            "prosody": {"mean_pitch": 150},
+        },
+        {
+            "id": "utt_002",
+            "text": "",
+            "sentiment": "negative",
+        },
+    ]
+    
+    results = verifier.verify_dataset(test_samples)
+    print(f"Verification results: {results['statistics']}")

annotation/labeler_tool.py

@@ -0,0 +1,333 @@
+"""GUI Tool for manual annotation of Myanmar speech data."""
+
+import json
+import logging
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Callable, Dict, List, Optional
+
+import pandas as pd
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class AnnotationLabel:
+    """Single annotation label."""
+    utterance_id: str
+    text: str
+    sentiment: str
+    intensity: float  # 0.0 - 1.0
+    confidence: float  # 0.0 - 1.0
+    notes: str = ""
+    annotator: str = "anonymous"
+    timestamp: str = ""
+
+
+@dataclass
+class AnnotationSession:
+    """Annotation session data."""
+    session_id: str
+    samples: List[Dict]
+    labels: List[AnnotationLabel] = field(default_factory=list)
+    completed_indices: set = field(default_factory=set)
+    current_index: int = 0
+    
+    @property
+    def progress(self) -> float:
+        if not self.samples:
+            return 0.0
+        return len(self.completed_indices) / len(self.samples)
+
+
+class MyanmarAnnotationTool:
+    """GUI-based annotation tool for Myanmar speech data."""
+    
+    # Sentiment classes
+    SENTIMENT_CLASSES = [
+        ("positive", "အပြုသဘော"),
+        ("negative", "အနှုတ်သဘော"),
+        ("neutral", "အလယ်အလတ်"),
+        ("sarcastic", "သရော်သည်"),
+        ("confused", "ရောထွေး"),
+    ]
+    
+    def __init__(self, output_dir: str = "data/annotations/human_annotated"):
+        self.output_dir = Path(output_dir)
+        self.output_dir.mkdir(parents=True, exist_ok=True)
+        self.session: Optional[AnnotationSession] = None
+        
+    def load_dataset(self, path: str) -> AnnotationSession:
+        """Load dataset for annotation."""
+        if path.endswith(".jsonl"):
+            samples = []
+            with open(path, "r", encoding="utf-8") as f:
+                for line in f:
+                    samples.append(json.loads(line))
+        elif path.endswith(".csv"):
+            df = pd.read_csv(path)
+            samples = df.to_dict("records")
+        else:
+            raise ValueError(f"Unsupported file format: {path}")
+        
+        session_id = Path(path).stem
+        self.session = AnnotationSession(
+            session_id=session_id,
+            samples=samples,
+        )
+        
+        logger.info(f"Loaded {len(samples)} samples for annotation")
+        return self.session
+    
+    def get_current_sample(self) -> Optional[Dict]:
+        """Get current sample to annotate."""
+        if not self.session:
+            return None
+        if self.session.current_index >= len(self.session.samples):
+            return None
+        return self.session.samples[self.session.current_index]
+    
+    def submit_annotation(
+        self,
+        sentiment: str,
+        intensity: float,
+        confidence: float,
+        notes: str = "",
+        annotator: str = "anonymous",
+    ) -> bool:
+        """Submit an annotation for the current sample."""
+        if not self.session:
+            logger.error("No active session")
+            return False
+        
+        sample = self.get_current_sample()
+        if not sample:
+            logger.error("No current sample")
+            return False
+        
+        from datetime import datetime
+        
+        label = AnnotationLabel(
+            utterance_id=sample.get("id", f"utt_{self.session.current_index}"),
+            text=sample.get("text", ""),
+            sentiment=sentiment,
+            intensity=intensity,
+            confidence=confidence,
+            notes=notes,
+            annotator=annotator,
+            timestamp=datetime.now().isoformat(),
+        )
+        
+        self.session.labels.append(label)
+        self.session.completed_indices.add(self.session.current_index)
+        
+        # Move to next incomplete sample
+        self._advance_to_next()
+        
+        logger.info(f"Annotated sample {label.utterance_id} as {sentiment}")
+        return True
+    
+    def _advance_to_next(self) -> None:
+        """Move to next incomplete sample."""
+        if not self.session:
+            return
+        
+        for i in range(self.session.current_index + 1, len(self.session.samples)):
+            if i not in self.session.completed_indices:
+                self.session.current_index = i
+                return
+        
+        for i in range(self.session.current_index):
+            if i not in self.session.completed_indices:
+                self.session.current_index = i
+                return
+    
+    def skip_sample(self) -> bool:
+        """Skip current sample without annotating."""
+        if not self.session:
+            return False
+        
+        self._advance_to_next()
+        return True
+    
+    def go_to_sample(self, index: int) -> bool:
+        """Go to specific sample index."""
+        if not self.session:
+            return False
+        if 0 <= index < len(self.session.samples):
+            self.session.current_index = index
+            return True
+        return False
+    
+    def save_session(self, path: Optional[str] = None) -> str:
+        """Save annotation session to file."""
+        if not self.session:
+            raise ValueError("No active session")
+        
+        if path is None:
+            path = self.output_dir / f"{self.session.session_id}_annotations.jsonl"
+        
+        labels_data = [
+            {
+                "utterance_id": label.utterance_id,
+                "text": label.text,
+                "sentiment": label.sentiment,
+                "intensity": label.intensity,
+                "confidence": label.confidence,
+                "notes": label.notes,
+                "annotator": label.annotator,
+                "timestamp": label.timestamp,
+            }
+            for label in self.session.labels
+        ]
+        
+        with open(path, "w", encoding="utf-8") as f:
+            for item in labels_data:
+                f.write(json.dumps(item, ensure_ascii=False) + "\n")
+        
+        logger.info(f"Saved {len(labels_data)} annotations to {path}")
+        return str(path)
+    
+    def export_to_dataframe(self) -> pd.DataFrame:
+        """Export annotations as DataFrame."""
+        if not self.session:
+            raise ValueError("No active session")
+        
+        return pd.DataFrame([
+            {
+                "utterance_id": label.utterance_id,
+                "text": label.text,
+                "sentiment": label.sentiment,
+                "intensity": label.intensity,
+                "confidence": label.confidence,
+                "notes": label.notes,
+                "annotator": label.annotator,
+            }
+            for label in self.session.labels
+        ])
+    
+    def get_statistics(self) -> Dict[str, Any]:
+        """Get annotation statistics."""
+        if not self.session:
+            return {}
+        
+        df = self.export_to_dataframe()
+        
+        return {
+            "total_samples": len(self.session.samples),
+            "annotated": len(self.session.labels),
+            "remaining": len(self.session.samples) - len(self.session.labels),
+            "progress": self.session.progress,
+            "sentiment_distribution": df["sentiment"].value_counts().to_dict() if len(df) > 0 else {},
+            "annotator_counts": df["annotator"].value_counts().to_dict() if len(df) > 0 else {},
+        }
+
+
+# Gradio-based GUI
+def create_gradio_interface(tool: MyanmarAnnotationTool):
+    """Create Gradio-based GUI for annotation."""
+    import gradio as gr
+    
+    with gr.Blocks(title="Myanmar Annotation Tool") as app:
+        gr.Markdown("# 🇲🇲 Myanmar Speech Annotation Tool")
+        
+        with gr.Row():
+            with gr.Column():
+                sample_display = gr.Textbox(
+                    label="Text to Annotate",
+                    lines=3,
+                    interactive=False,
+                )
+                prosody_display = gr.JSON(label="Prosody Features")
+            
+            with gr.Column():
+                sentiment_dropdown = gr.Dropdown(
+                    choices=[s[0] for s in tool.SENTIMENT_CLASSES],
+                    label="Sentiment",
+                    value="neutral",
+                )
+                intensity_slider = gr.Slider(
+                    minimum=0.0,
+                    maximum=1.0,
+                    value=0.5,
+                    step=0.1,
+                    label="Intensity",
+                )
+                confidence_slider = gr.Slider(
+                    minimum=0.0,
+                    maximum=1.0,
+                    value=0.8,
+                    step=0.1,
+                    label="Confidence",
+                )
+                notes_input = gr.Textbox(
+                    label="Notes",
+                    lines=2,
+                )
+        
+        with gr.Row():
+            submit_btn = gr.Button("Submit", variant="primary")
+            skip_btn = gr.Button("Skip")
+            save_btn = gr.Button("Save Session")
+        
+        with gr.Row():
+            progress_display = gr.Textbox(label="Progress", interactive=False)
+            stats_display = gr.JSON(label="Statistics")
+        
+        def update_display():
+            sample = tool.get_current_sample()
+            if sample:
+                return (
+                    sample.get("text", ""),
+                    sample.get("prosody", {}),
+                )
+            return ("No more samples", {})
+        
+        def submit_annotation(sentiment, intensity, confidence, notes):
+            tool.submit_annotation(sentiment, intensity, confidence, notes)
+            sample = tool.get_current_sample()
+            stats = tool.get_statistics()
+            if sample:
+                return (
+                    sample.get("text", ""),
+                    sample.get("prosody", {}),
+                    f"{stats['annotated']}/{stats['total_samples']} ({stats['progress']*100:.1f}%)",
+                    stats,
+                )
+            return ("All samples annotated!", {}, "100%", stats)
+        
+        def skip():
+            tool.skip_sample()
+            return update_display()
+        
+        def save():
+            path = tool.save_session()
+            return f"Saved to {path}"
+        
+        submit_btn.click(
+            submit_annotation,
+            inputs=[sentiment_dropdown, intensity_slider, confidence_slider, notes_input],
+            outputs=[sample_display, prosody_display, progress_display, stats_display],
+        )
+        skip_btn.click(
+            skip,
+            outputs=[sample_display, prosody_display],
+        )
+        save_btn.click(
+            save,
+            outputs=[progress_display],
+        )
+        
+        # Initialize display
+        app.load(
+            update_display,
+            outputs=[sample_display, prosody_display],
+        )
+    
+    return app
+
+
+if __name__ == "__main__":
+    tool = MyanmarAnnotationTool()
+    print("MyanmarAnnotationTool initialized")
+    print(f"Available sentiment classes: {[s[0] for s in tool.SENTIMENT_CLASSES]}")

augmentation/back_translator.py

@@ -0,0 +1,242 @@
+"""Back-translation augmentation for Myanmar text.
+
+Translates text to another language and back to create
+paraphrased versions for data augmentation.
+"""
+
+import logging
+from pathlib import Path
+from typing import Dict, List, Optional, Tuple
+
+logger = logging.getLogger(__name__)
+
+
+class BackTranslator:
+    """Back-translation augmentation using translation APIs."""
+    
+    def __init__(
+        self,
+        translator_api: Optional[object] = None,
+        target_lang: str = "en",
+        source_lang: str = "my",
+    ):
+        """
+        Args:
+            translator_api: Translation API instance
+            target_lang: Target language for translation
+            source_lang: Source language
+        """
+        self.translator_api = translator_api
+        self.target_lang = target_lang
+        self.source_lang = source_lang
+    
+    def translate(
+        self,
+        text: str,
+        direction: str = "forward",
+    ) -> Optional[str]:
+        """Translate text.
+        
+        Args:
+            text: Text to translate
+            direction: "forward" (src->tgt) or "backward" (tgt->src)
+        
+        Returns:
+            Translated text or None if failed
+        """
+        if self.translator_api is None:
+            # Simulate translation for testing
+            return self._simulate_translation(text, direction)
+        
+        try:
+            if direction == "forward":
+                return self.translator_api.translate(
+                    text,
+                    src=self.source_lang,
+                    tgt=self.target_lang,
+                )
+            else:
+                return self.translator_api.translate(
+                    text,
+                    src=self.target_lang,
+                    tgt=self.source_lang,
+                )
+        except Exception as e:
+            logger.error(f"Translation failed: {e}")
+            return None
+    
+    def _simulate_translation(
+        self,
+        text: str,
+        direction: str,
+    ) -> str:
+        """Simulate translation for testing without API.
+        
+        In real use, this would call a translation service.
+        """
+        # This is a placeholder - real implementation would use
+        # Google Translate, DeepL, or similar API
+        
+        # For testing, just return the original text
+        # with a marker to indicate it was "translated"
+        marker = "[EN]" if direction == "forward" else "[MY]"
+        return f"{marker}{text}{marker}"
+    
+    def back_translate(
+        self,
+        text: str,
+    ) -> Tuple[Optional[str], Optional[str], Optional[str]]:
+        """Translate text to target language and back.
+        
+        Args:
+            text: Myanmar text
+        
+        Returns:
+            (forward_translation, back_translation, final_text)
+        """
+        # Forward translation
+        forward = self.translate(text, "forward")
+        if forward is None:
+            return None, None, None
+        
+        # Back translation
+        back = self.translate(forward, "backward")
+        if back is None:
+            return forward, None, None
+        
+        return forward, back, back
+    
+    def augment_dataset(
+        self,
+        samples: List[Dict],
+        batch_size: int = 10,
+    ) -> List[Dict]:
+        """Augment dataset using back-translation.
+        
+        Args:
+            samples: List of sample dictionaries
+            batch_size: Batch size for API calls
+        
+        Returns:
+            List of augmented samples
+        """
+        augmented = []
+        
+        for i, sample in enumerate(samples):
+            text = sample.get("text", "")
+            
+            forward, back, final = self.back_translate(text)
+            
+            if final and final != text:
+                aug_sample = sample.copy()
+                aug_sample["text"] = final
+                aug_sample["forward_translation"] = forward
+                aug_sample["back_translation"] = back
+                aug_sample["augmentation_type"] = "back_translation"
+                aug_sample["is_augmented"] = True
+                augmented.append(aug_sample)
+            
+            if (i + 1) % batch_size == 0:
+                logger.info(f"Processed {i + 1}/{len(samples)} samples")
+        
+        return augmented
+
+
+class TranslationAugmenter:
+    """Advanced translation-based augmentation."""
+    
+    def __init__(
+        self,
+        translator_api: Optional[object] = None,
+        languages: Optional[List[str]] = None,
+    ):
+        """
+        Args:
+            translator_api: Translation API instance
+            languages: List of intermediate languages for multi-hop translation
+        """
+        self.translator_api = translator_api
+        self.languages = languages or ["en", "zh", "ja", "ko"]
+    
+    def multi_hop_translate(
+        self,
+        text: str,
+        intermediate_langs: Optional[List[str]] = None,
+    ) -> str:
+        """Translate through multiple intermediate languages.
+        
+        Args:
+            text: Text to translate
+            intermediate_langs: Languages to translate through
+        
+        Returns:
+            Final translated text
+        """
+        if intermediate_langs is None:
+            intermediate_langs = random.sample(
+                self.languages,
+                k=min(2, len(self.languages))
+            )
+        
+        current_text = text
+        
+        for lang in intermediate_langs:
+            # Translate to intermediate language
+            if self.translator_api:
+                current_text = self.translator_api.translate(
+                    current_text,
+                    src="my",
+                    tgt=lang,
+                )
+            
+            # Translate back to Myanmar
+            if self.translator_api:
+                current_text = self.translator_api.translate(
+                    current_text,
+                    src=lang,
+                    tgt="my",
+                )
+        
+        return current_text
+    
+    def paraphrase_with_context(
+        self,
+        text: str,
+        context: str,
+    ) -> str:
+        """Paraphrase text while maintaining context.
+        
+        Args:
+            text: Text to paraphrase
+            context: Additional context to help translation
+        
+        Returns:
+            Paraphrased text
+        """
+        # Combine text with context
+        combined = f"{context}: {text}"
+        
+        # Translate and back-translate
+        translator = BackTranslator(self.translator_api)
+        _, _, paraphrased = translator.back_translate(combined)
+        
+        return paraphrased if paraphrased else text
+
+
+def create_back_translator(
+    translator_api: Optional[object] = None,
+    target_lang: str = "en",
+) -> BackTranslator:
+    """Factory function to create back translator."""
+    return BackTranslator(
+        translator_api=translator_api,
+        target_lang=target_lang,
+    )
+
+
+if __name__ == "__main__":
+    print("BackTranslator loaded")
+    print("For production use, integrate with translation APIs like:")
+    print("  - Google Cloud Translation")
+    print("  - DeepL API")
+    print("  - transformers.TranslationPipeline")

augmentation/perturbator.py

@@ -0,0 +1,368 @@
+"""Text perturbation for adversarial data augmentation.
+
+Applies various perturbations to text to create challenging
+training examples that improve model robustness.
+"""
+
+import random
+import re
+from enum import Enum
+from typing import Callable, Dict, List, Optional, Tuple
+
+
+class PerturbationType(str, Enum):
+    """Types of text perturbations."""
+    CHAR_SWAP = "char_swap"
+    CHAR_DELETE = "char_delete"
+    CHAR_DUPLICATE = "char_duplicate"
+    WORD_SWAP = "word_swap"
+    WORD_DELETE = "word_delete"
+    WORD_DUPLICATE = "word_duplicate"
+    SENTENCE_SHUFFLE = "sentence_shuffle"
+    KEYBOARD_typo = "keyboard_typo"
+    RANDOM_CASE = "random_case"
+
+
+class TextPerturbator:
+    """Apply perturbations to Myanmar text."""
+    
+    # Myanmar keyboard layout (simplified)
+    KEYBOARD_LAYOUT = {
+        "က": ["ခ", "ဂ"],
+        "ခ": ["က", "ဂ", "ဃ"],
+        "ဂ": ["က", "ခ"],
+        "ဃ": ["ခ"],
+        "င": ["စ", "ဆ"],
+        "စ": ["က", "င", "ဆ", "ဇ"],
+        "ဆ": ["င", "စ", "ဇ"],
+        "ဇ": ["စ", "ဆ", "ဈ"],
+        "ဈ": ["ဇ"],
+        "ဉ": ["ည"],
+        "ည": ["ဉ", "ဋ"],
+        "ဋ": ["ည", "ဌ"],
+        "ဌ": ["ဋ", "ဍ"],
+        "ဍ": ["ဌ", "ဎ"],
+        "ဎ": ["ဍ", "ဏ"],
+        "ဏ": ["ဎ", "တ"],
+        "တ": ["ဏ", "ထ", "ဒ"],
+        "ထ": ["တ", "ဓ"],
+        "ဓ": ["ထ", "ဒ"],
+        "ဒ": ["တ", "ဓ", "န"],
+        "န": ["ဒ", "ပ", "ဖ"],
+        "ပ": ["န", "ဖ", "ဗ"],
+        "ဖ": ["န", "ပ", "ဗ"],
+        "ဗ": ["ပ", "ဖ"],
+    }
+    
+    def __init__(self, seed: int = 42):
+        random.seed(seed)
+        self.perturbation_count = 0
+    
+    def char_swap(self, text: str, prob: float = 0.1) -> str:
+        """Swap adjacent characters."""
+        chars = list(text)
+        for i in range(len(chars) - 1):
+            if random.random() < prob:
+                chars[i], chars[i + 1] = chars[i + 1], chars[i]
+        return "".join(chars)
+    
+    def char_delete(self, text: str, prob: float = 0.05) -> str:
+        """Delete random characters."""
+        chars = list(text)
+        result = [c for c in chars if random.random() > prob]
+        return "".join(result) if result else text
+    
+    def char_duplicate(self, text: str, prob: float = 0.05) -> str:
+        """Duplicate random characters."""
+        chars = list(text)
+        result = []
+        for c in chars:
+            result.append(c)
+            if random.random() < prob:
+                result.append(c)
+        return "".join(result)
+    
+    def word_swap(self, text: str, prob: float = 0.1) -> str:
+        """Swap adjacent words."""
+        words = text.split()
+        if len(words) < 2:
+            return text
+        
+        for i in range(len(words) - 1):
+            if random.random() < prob:
+                words[i], words[i + 1] = words[i + 1], words[i]
+        
+        return " ".join(words)
+    
+    def word_delete(self, text: str, prob: float = 0.1) -> str:
+        """Delete random words."""
+        words = text.split()
+        if len(words) < 2:
+            return text
+        
+        result = [w for w in words if random.random() > prob]
+        return " ".join(result) if result else text
+    
+    def word_duplicate(self, text: str, prob: float = 0.1) -> str:
+        """Duplicate random words."""
+        words = text.split()
+        result = []
+        for w in words:
+            result.append(w)
+            if random.random() < prob:
+                result.append(w)
+        return " ".join(result)
+    
+    def keyboard_typo(self, text: str, prob: float = 0.1) -> str:
+        """Introduce keyboard typos."""
+        chars = list(text)
+        result = []
+        
+        for c in chars:
+            if random.random() < prob and c in self.KEYBOARD_LAYOUT:
+                # Replace with keyboard neighbor
+                neighbor = random.choice(self.KEYBOARD_LAYOUT[c])
+                result.append(neighbor)
+            else:
+                result.append(c)
+        
+        return "".join(result)
+    
+    def random_case(self, text: str, prob: float = 0.1) -> str:
+        """Randomly change case of characters (for mixed scripts)."""
+        # Myanmar doesn't have case, but this can affect punctuation
+        chars = list(text)
+        for i in range(len(chars)):
+            if chars[i].isupper() and random.random() < prob:
+                chars[i] = chars[i].lower()
+            elif chars[i].islower() and random.random() < prob:
+                chars[i] = chars[i].upper()
+        return "".join(chars)
+    
+    def sentence_shuffle(self, text: str) -> str:
+        """Shuffle sentences in multi-sentence text."""
+        sentences = re.split(r'[။၊।\.\!\?]+', text)
+        sentences = [s.strip() for s in sentences if s.strip()]
+        
+        if len(sentences) < 2:
+            return text
+        
+        random.shuffle(sentences)
+        return " ".join(sentences)
+    
+    def apply_perturbation(
+        self,
+        text: str,
+        perturbation_type: PerturbationType,
+        prob: float = 0.1,
+    ) -> str:
+        """Apply a specific perturbation.
+        
+        Args:
+            text: Myanmar text
+            perturbation_type: Type of perturbation
+            prob: Probability of perturbation
+        
+        Returns:
+            Perturbed text
+        """
+        if perturbation_type == PerturbationType.CHAR_SWAP:
+            return self.char_swap(text, prob)
+        elif perturbation_type == PerturbationType.CHAR_DELETE:
+            return self.char_delete(text, prob)
+        elif perturbation_type == PerturbationType.CHAR_DUPLICATE:
+            return self.char_duplicate(text, prob)
+        elif perturbation_type == PerturbationType.WORD_SWAP:
+            return self.word_swap(text, prob)
+        elif perturbation_type == PerturbationType.WORD_DELETE:
+            return self.word_delete(text, prob)
+        elif perturbation_type == PerturbationType.WORD_DUPLICATE:
+            return self.word_duplicate(text, prob)
+        elif perturbation_type == PerturbationType.KEYBOARD_typo:
+            return self.keyboard_typo(text, prob)
+        elif perturbation_type == PerturbationType.RANDOM_CASE:
+            return self.random_case(text, prob)
+        elif perturbation_type == PerturbationType.SENTENCE_SHUFFLE:
+            return self.sentence_shuffle(text)
+        else:
+            return text
+    
+    def apply_random_perturbations(
+        self,
+        text: str,
+        n_perturbations: int = 2,
+        prob: float = 0.1,
+    ) -> Tuple[str, List[PerturbationType]]:
+        """Apply random perturbations.
+        
+        Args:
+            text: Myanmar text
+            n_perturbations: Number of perturbations to apply
+            prob: Probability for each perturbation
+        
+        Returns:
+            (perturbed_text, list_of_applied_perturbations)
+        """
+        perturbations = list(PerturbationType)
+        applied = []
+        
+        current_text = text
+        
+        for _ in range(n_perturbations):
+            pert_type = random.choice(perturbations)
+            current_text = self.apply_perturbation(current_text, pert_type, prob)
+            applied.append(pert_type)
+        
+        self.perturbation_count += 1
+        
+        return current_text, applied
+    
+    def augment_dataset(
+        self,
+        samples: List[Dict],
+        n_perturbations: int = 2,
+        prob: float = 0.1,
+        n_augmentations: int = 2,
+    ) -> List[Dict]:
+        """Augment dataset with perturbations.
+        
+        Args:
+            samples: List of sample dictionaries
+            n_perturbations: Number of perturbations per augmentation
+            prob: Probability for each perturbation
+            n_augmentations: Number of augmentations per sample
+        
+        Returns:
+            List of augmented samples
+        """
+        augmented = []
+        
+        for sample in samples:
+            text = sample.get("text", "")
+            
+            for i in range(n_augmentations):
+                aug_text, applied = self.apply_random_perturbations(
+                    text,
+                    n_perturbations=n_perturbations,
+                    prob=prob,
+                )
+                
+                aug_sample = sample.copy()
+                aug_sample["text"] = aug_text
+                aug_sample["augmentation_id"] = i
+                aug_sample["perturbations"] = [p.value for p in applied]
+                aug_sample["is_augmented"] = True
+                augmented.append(aug_sample)
+        
+        return augmented
+
+
+class AdversarialPerturbator:
+    """Advanced adversarial perturbations targeting specific weaknesses."""
+    
+    def __init__(self):
+        self.base_perturbator = TextPerturbator()
+    
+    def confuse_sentiment_keywords(
+        self,
+        text: str,
+        keyword_replacements: Dict[str, str],
+    ) -> str:
+        """Replace sentiment keywords to flip or confuse sentiment.
+        
+        Args:
+            text: Myanmar text
+            keyword_replacements: Dict of keyword -> replacement
+        
+        Returns:
+            Text with keywords replaced
+        """
+        for keyword, replacement in keyword_replacements.items():
+            if keyword in text:
+                text = text.replace(keyword, replacement, 1)  # Replace first occurrence only
+        return text
+    
+    def add_distractors(
+        self,
+        text: str,
+        distractors: List[str] = None,
+    ) -> str:
+        """Add distractor phrases to text.
+        
+        Args:
+            text: Myanmar text
+            distractors: List of distractor phrases
+        
+        Returns:
+            Text with distractors added
+        """
+        if distractors is None:
+            distractors = [
+                "အဲ့ဒါကို",
+                "ဟုတ်ကဲ့",
+                "နောက်တော့",
+            ]
+        
+        distractor = random.choice(distractors)
+        words = text.split()
+        
+        if len(words) >= 3:
+            insert_pos = random.randint(1, len(words) - 1)
+            words.insert(insert_pos, distractor)
+        
+        return " ".join(words)
+    
+    def paraphrase_style(self, text: str, style: str = "formal") -> str:
+        """Change text style (formal/informal).
+        
+        Args:
+            text: Myanmar text
+            style: Target style ("formal" or "informal")
+        
+        Returns:
+            Text with changed style
+        """
+        style_markers = {
+            "formal": {
+                "add": ["သည်", "မှာ", "ကို", "ဖြင့်"],
+                "remove": ["နော်", "ဟုတ်"],
+            },
+            "informal": {
+                "add": ["နော်", "ဟုတ်"],
+                "remove": ["သည်", "မှာ", "ကို", "ဖြင့်"],
+            },
+        }
+        
+        markers = style_markers.get(style, style_markers["formal"])
+        
+        for marker in markers.get("add", []):
+            if marker not in text and random.random() < 0.3:
+                words = text.split()
+                insert_pos = random.randint(0, len(words))
+                words.insert(insert_pos, marker)
+                text = " ".join(words)
+        
+        for marker in markers.get("remove", []):
+            if marker in text and random.random() < 0.5:
+                text = text.replace(marker, "")
+        
+        return text
+
+
+def create_perturbator(seed: int = 42) -> TextPerturbator:
+    """Factory function to create perturbator."""
+    return TextPerturbator(seed=seed)
+
+
+if __name__ == "__main__":
+    perturbator = create_perturbator()
+    
+    test_text = "ကျေးဇူးပါ မင်္ဂလာပါ"
+    
+    print(f"Original: {test_text}")
+    print(f"\nRandom perturbations:")
+    
+    for i in range(3):
+        aug, applied = perturbator.apply_random_perturbations(test_text)
+        print(f"  {i+1}. {aug}")
+        print(f"     Applied: {[p.value for p in applied]}")

augmentation/synonym_replacer.py

@@ -0,0 +1,227 @@
+"""Synonym replacement for adversarial data augmentation.
+
+Replaces words with synonyms to create challenging training examples
+that help improve model robustness.
+"""
+
+import random
+from pathlib import Path
+from typing import Dict, List, Optional, Set, Tuple
+
+import yaml
+
+
+class MyanmarSynonymReplacer:
+    """Replace words with Myanmar synonyms for data augmentation."""
+    
+    # Basic synonym dictionary for Myanmar
+    SYNONYMS = {
+        "ကျေးဇူး": ["ခန့်ညား", "ဂုဏ်ပြု", "အားထုတ်"],
+        "ပါး": ["များ", "အရမ်း", "အလွန်"],
+        "သိပ်": ["အရမ်း", "များစွာ", "ပါး"],
+        "ကောင်း": ["မွန်", "သန့်", "စင်"],
+        "ဆိုး": ["မကောင်း", "ယုတ်", "ညံ့"],
+        "ပျော်": ["ရွှင်", "ပီတိ", "မင်္ဂလာ"],
+        "စိတ်မကောင်း": ["စိတ်ဓာတ်ကျ", "ဝမ်းနည်း", "ဒေါသ"],
+        "လာ": ["ရောက်", "သွား", "ပို့"],
+        "သွား": ["သွား", "ထွက်", "မောင်း"],
+        "ပေး": ["ပေးဆောင်", "မွေးစား", "အပ်"],
+        "ယူ": ["ရ", "လက်ခံ", "ရှာ"],
+        "မှန်": ["ဟုတ်", "ကျိ", "သင့်"],
+        "မှား": ["မဟုတ်", "အမှား", "ယွင်း"],
+    }
+    
+    def __init__(
+        self,
+        synonym_file: Optional[str] = None,
+        seed: int = 42,
+    ):
+        """
+        Args:
+            synonym_file: Path to YAML file with custom synonyms
+            seed: Random seed for reproducibility
+        """
+        random.seed(seed)
+        
+        self.synonyms = dict(self.SYNONYMS)
+        
+        if synonym_file and Path(synonym_file).exists():
+            self._load_synonyms(synonym_file)
+    
+    def _load_synonyms(self, path: str) -> None:
+        """Load synonyms from YAML file."""
+        with open(path, "r", encoding="utf-8") as f:
+            custom = yaml.safe_load(f)
+        
+        if custom:
+            self.synonyms.update(custom)
+    
+    def get_synonyms(self, word: str) -> List[str]:
+        """Get synonyms for a word."""
+        return self.synonyms.get(word, [])
+    
+    def replace_word(self, word: str) -> Tuple[str, bool]:
+        """Replace a word with a synonym.
+        
+        Args:
+            word: Word to replace
+        
+        Returns:
+            (replaced_word, was_replaced)
+        """
+        synonyms = self.get_synonyms(word)
+        
+        if synonyms:
+            replacement = random.choice(synonyms)
+            return replacement, True
+        
+        return word, False
+    
+    def augment_text(
+        self,
+        text: str,
+        replace_prob: float = 0.3,
+        max_replacements: int = 3,
+    ) -> Tuple[str, List[Tuple[str, str]]]:
+        """Augment text by replacing words with synonyms.
+        
+        Args:
+            text: Myanmar text
+            replace_prob: Probability of replacing each synonym word
+            max_replacements: Maximum number of replacements
+        
+        Returns:
+            (augmented_text, list_of_replacements)
+        """
+        words = text.split()
+        replacements = []
+        augmented_words = []
+        
+        num_replaced = 0
+        
+        for word in words:
+            if num_replaced >= max_replacements:
+                augmented_words.append(word)
+                continue
+            
+            if word in self.synonyms and random.random() < replace_prob:
+                new_word, replaced = self.replace_word(word)
+                if replaced:
+                    augmented_words.append(new_word)
+                    replacements.append((word, new_word))
+                    num_replaced += 1
+                else:
+                    augmented_words.append(word)
+            else:
+                augmented_words.append(word)
+        
+        return " ".join(augmented_words), replacements
+    
+    def augment_dataset(
+        self,
+        samples: List[Dict],
+        replace_prob: float = 0.3,
+        max_replacements: int = 3,
+        n_augmentations: int = 2,
+    ) -> List[Dict]:
+        """Augment entire dataset.
+        
+        Args:
+            samples: List of sample dictionaries
+            replace_prob: Probability of replacing each word
+            max_replacements: Maximum replacements per text
+            n_augmentations: Number of augmentations per sample
+        
+        Returns:
+            List of augmented samples
+        """
+        augmented = []
+        
+        for sample in samples:
+            text = sample.get("text", "")
+            
+            for i in range(n_augmentations):
+                aug_text, replacements = self.augment_text(
+                    text,
+                    replace_prob=replace_prob,
+                    max_replacements=max_replacements,
+                )
+                
+                if replacements:  # Only add if something was replaced
+                    aug_sample = sample.copy()
+                    aug_sample["text"] = aug_text
+                    aug_sample["augmentation_id"] = i
+                    aug_sample["replacements"] = replacements
+                    aug_sample["is_augmented"] = True
+                    augmented.append(aug_sample)
+        
+        return augmented
+    
+    def get_replacement_stats(self) -> Dict:
+        """Get statistics about synonym coverage."""
+        total_words = sum(len(syns) for syns in self.synonyms.values())
+        return {
+            "num_synonym_groups": len(self.synonyms),
+            "total_synonyms": total_words,
+            "avg_synonyms_per_word": total_words / len(self.synonyms) if self.synonyms else 0,
+        }
+
+
+class ContextualSynonymReplacer:
+    """Synonym replacer that considers context."""
+    
+    def __init__(self):
+        # Context-dependent synonyms
+        self.CONTEXT_SYNONYMS = {
+            "formal": {
+                "ကျေးဇူး": ["ဂုဏ်ပြုမှတ်ရှိပါ", "အထူးပင်ကျေးဇူးတင်ပါ"],
+                "သိပ်": ["အလွန်", "အထူးသဖြင့်"],
+            },
+            "informal": {
+                "ကျေးဇူး": ["ခန့်ညား", "ကျေးဇူးလည်းပါ"],
+                "ပါး": ["ပို", "အရမ်း"],
+            },
+        }
+    
+    def augment_by_context(
+        self,
+        text: str,
+        context: str = "formal",
+    ) -> str:
+        """Augment text using context-specific synonyms.
+        
+        Args:
+            text: Myanmar text
+            context: Context type ("formal" or "informal")
+        
+        Returns:
+            Augmented text
+        """
+        context_syns = self.CONTEXT_SYNONYMS.get(context, {})
+        
+        for word, synonyms in context_syns.items():
+            if word in text:
+                replacement = random.choice(synonyms)
+                text = text.replace(word, replacement, 1)
+        
+        return text
+
+
+def create_synonym_replacer(
+    synonym_file: Optional[str] = None,
+) -> MyanmarSynonymReplacer:
+    """Factory function to create synonym replacer."""
+    return MyanmarSynonymReplacer(synonym_file=synonym_file)
+
+
+if __name__ == "__main__":
+    replacer = create_synonym_replacer()
+    
+    test_text = "ကျေးဇူးပါးသိပ်ကောင်းတယ်"
+    
+    for i in range(3):
+        aug, replacements = replacer.augment_text(test_text)
+        print(f"Original: {test_text}")
+        print(f"Augmented: {aug}")
+        print(f"Replacements: {replacements}")
+        print()

data_processing/audio_processor.py

@@ -0,0 +1,215 @@
+"""Audio processing module for Myanmar Ghost project."""
+
+import logging
+from pathlib import Path
+from typing import Optional, Tuple
+
+import librosa
+import numpy as np
+import soundfile as sf
+from scipy.signal import butter, filtfilt
+
+logger = logging.getLogger(__name__)
+
+
+class AudioProcessor:
+    """Process audio files for Myanmar speech recognition."""
+
+    def __init__(
+        self,
+        sample_rate: int = 16000,
+        n_fft: int = 512,
+        hop_length: int = 160,
+        n_mels: int = 80,
+    ):
+        self.sample_rate = sample_rate
+        self.n_fft = n_fft
+        self.hop_length = hop_length
+        self.n_mels = n_mels
+
+    def load_audio(self, path: str) -> Tuple[np.ndarray, int]:
+        """Load audio file and resample to target sample rate."""
+        audio, sr = librosa.load(path, sr=self.sample_rate)
+        logger.info(f"Loaded audio from {path}: {len(audio)} samples at {sr}Hz")
+        return audio, sr
+
+    def normalize_audio(self, audio: np.ndarray) -> np.ndarray:
+        """Normalize audio to [-1, 1] range."""
+        max_val = np.abs(audio).max()
+        if max_val > 0:
+            audio = audio / max_val
+        return audio
+
+    def remove_silence(
+        self,
+        audio: np.ndarray,
+        threshold_db: float = -40,
+        min_silence_duration: float = 0.3,
+    ) -> np.ndarray:
+        """Remove silence from audio based on energy threshold."""
+        intervals = librosa.effects.split(
+            audio,
+            top_db=-threshold_db,
+            frame_length=self.n_fft,
+            hop_length=self.hop_length,
+        )
+        
+        if len(intervals) == 0:
+            return audio
+        
+        min_samples = int(min_silence_duration * self.sample_rate)
+        non_silent = []
+        
+        for start, end in intervals:
+            if end - start >= min_samples:
+                non_silent.append(audio[start:end])
+        
+        if non_silent:
+            return np.concatenate(non_silent)
+        return audio
+
+    def apply_bandpass_filter(
+        self,
+        audio: np.ndarray,
+        low_freq: float = 80,
+        high_freq: float = 7500,
+    ) -> np.ndarray:
+        """Apply bandpass filter to focus on speech frequencies."""
+        nyquist = self.sample_rate / 2
+        low = low_freq / nyquist
+        high = high_freq / nyquist
+        
+        if low < 0:
+            low = 0.001
+        if high > 1:
+            high = 0.999
+            
+        b, a = butter(4, [low, high], btype="band")
+        filtered = filtfilt(b, a, audio)
+        return filtered
+
+    def reduce_noise(
+        self,
+        audio: np.ndarray,
+        noise_profile: Optional[np.ndarray] = None,
+    ) -> np.ndarray:
+        """Reduce background noise using spectral subtraction."""
+        if noise_profile is None:
+            noise_profile = audio[: int(0.1 * self.sample_rate)]
+        
+        noise_spectrum = np.abs(np.fft.rfft(noise_profile))
+        noise_magnitude = np.mean(noise_spectrum, axis=0)
+        
+        audio_spectrum = np.abs(np.fft.rfft(audio))
+        cleaned = np.maximum(
+            audio_spectrum - noise_magnitude[:, None],
+            audio_spectrum * 0.1,
+        )
+        cleaned = cleaned * np.exp(1j * np.fft.rfft(audio).angle())
+        
+        return np.fft.irfft(cleaned)
+
+    def extract_mel_spectrogram(self, audio: np.ndarray) -> np.ndarray:
+        """Extract mel spectrogram features."""
+        mel_spec = librosa.feature.melspectrogram(
+            y=audio,
+            sr=self.sample_rate,
+            n_fft=self.n_fft,
+            hop_length=self.hop_length,
+            n_mels=self.n_mels,
+        )
+        log_mel = librosa.power_to_db(mel_spec, ref=np.max)
+        return log_mel
+
+    def extract_prosody_features(self, audio: np.ndarray) -> dict:
+        """Extract prosodic features (pitch, energy, speaking rate)."""
+        pitches, magnitudes = librosa.piptrack(
+            y=audio,
+            sr=self.sample_rate,
+            n_fft=self.n_fft,
+            hop_length=self.hop_length,
+        )
+        
+        pitch_values = []
+        for i in range(pitches.shape[1]):
+            index = magnitudes[:, i].argmax()
+            pitch = pitches[index, i]
+            if pitch > 0:
+                pitch_values.append(pitch)
+        
+        rms = librosa.feature.rms(y=audio, hop_length=self.hop_length)[0]
+        
+        return {
+            "mean_pitch": np.mean(pitch_values) if pitch_values else 0,
+            "pitch_std": np.std(pitch_values) if pitch_values else 0,
+            "pitch_range": (np.min(pitch_values) if pitch_values else 0,
+                          np.max(pitch_values) if pitch_values else 0),
+            "mean_energy": np.mean(rms),
+            "energy_std": np.std(rms),
+        }
+
+    def process_file(
+        self,
+        input_path: str,
+        output_path: str,
+        remove_silence: bool = True,
+        apply_filter: bool = True,
+    ) -> dict:
+        """Process a single audio file."""
+        audio, sr = self.load_audio(input_path)
+        audio = self.normalize_audio(audio)
+        
+        if apply_filter:
+            audio = self.apply_bandpass_filter(audio)
+        
+        if remove_silence:
+            audio = self.remove_silence(audio)
+        
+        prosody = self.extract_prosody_features(audio)
+        
+        sf.write(output_path, audio, self.sample_rate)
+        logger.info(f"Saved processed audio to {output_path}")
+        
+        return {
+            "input_path": input_path,
+            "output_path": output_path,
+            "duration": len(audio) / self.sample_rate,
+            "prosody": prosody,
+        }
+
+    def batch_process(
+        self,
+        input_dir: str,
+        output_dir: str,
+        pattern: str = "*.wav",
+    ) -> list:
+        """Process all audio files in a directory."""
+        input_path = Path(input_dir)
+        output_path = Path(output_dir)
+        output_path.mkdir(parents=True, exist_ok=True)
+        
+        results = []
+        for file_path in input_path.glob(pattern):
+            out_file = output_path / file_path.name
+            result = self.process_file(str(file_path), str(out_file))
+            results.append(result)
+        
+        return results
+
+
+def create_processor(config: dict = None) -> AudioProcessor:
+    """Factory function to create AudioProcessor from config."""
+    if config is None:
+        config = {}
+    
+    return AudioProcessor(
+        sample_rate=config.get("sample_rate", 16000),
+        n_fft=config.get("n_fft", 512),
+        hop_length=config.get("hop_length", 160),
+        n_mels=config.get("n_mels", 80),
+    )
+
+
+if __name__ == "__main__":
+    processor = create_processor()
+    print("AudioProcessor initialized successfully")

data_processing/graph_builder.py

@@ -0,0 +1,466 @@
+"""Knowledge Graph builder for Myanmar Ghost project.
+
+Represents conversational context as a knowledge graph for better
+understanding of complex social interactions.
+Example: (Speaker, Role, Customer) --[located_in]--> (Restaurant)
+"""
+
+import json
+from dataclasses import dataclass, field
+from enum import Enum
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Set, Tuple
+
+import networkx as nx
+
+
+class NodeType(str, Enum):
+    """Types of nodes in the knowledge graph."""
+    SPEAKER = "speaker"
+    UTTERANCE = "utterance"
+    LOCATION = "location"
+    ORGANIZATION = "organization"
+    EMOTION = "emotion"
+    TOPIC = "topic"
+    ACTION = "action"
+    TIME = "time"
+
+
+class RelationType(str, Enum):
+    """Types of relations between nodes."""
+    SPEAKS = "speaks"
+    LOCATED_IN = "located_in"
+    WORKS_AT = "works_at"
+    VISITS = "visits"
+    FEELS = "feels"
+    ABOUT = "about"
+    BEFORE = "before"
+    AFTER = "after"
+    IN_RESPONSE_TO = "in_response_to"
+    CONTAINS = "contains"
+    HAS_ROLE = "has_role"
+
+
+@dataclass
+class Entity:
+    """Represents an entity in the knowledge graph."""
+    id: str
+    type: NodeType
+    properties: Dict[str, Any] = field(default_factory=dict)
+    aliases: List[str] = field(default_factory=list)
+    
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "id": self.id,
+            "type": self.type.value,
+            "properties": self.properties,
+            "aliases": self.aliases,
+        }
+
+
+@dataclass
+class Relation:
+    """Represents a relation between entities."""
+    source: str  # Entity ID
+    target: str  # Entity ID
+    type: RelationType
+    properties: Dict[str, Any] = field(default_factory=dict)
+    confidence: float = 1.0
+    
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "source": self.source,
+            "target": self.target,
+            "type": self.type.value,
+            "properties": self.properties,
+            "confidence": self.confidence,
+        }
+
+
+class MyanmarKnowledgeGraph:
+    """Build and manage knowledge graph for Myanmar conversations."""
+    
+    # Common Myanmar entities
+    LOCATIONS = {
+        "စားသောက်ဆိုင်": NodeType.LOCATION,
+        "ဆေးရုံ": NodeType.LOCATION,
+        "ဈေး": NodeType.LOCATION,
+        "ရုံး": NodeType.LOCATION,
+        "အိမ်": NodeType.LOCATION,
+    }
+    
+    EMOTIONS = {
+        "ပျော်": NodeType.EMOTION,
+        "စိတ်ဓာတ်ကျ": NodeType.EMOTION,
+        "ဒေါသ": NodeType.EMOTION,
+        "ဝမ်းနည်း": NodeType.EMOTION,
+        "ပိုးပါး": NodeType.EMOTION,
+    }
+    
+    ROLES = {
+        "ဖေါ်သည်": "customer",
+        "ဝန်ထမ်း": "staff",
+        "ဆရာဝန်": "doctor",
+        "ပါးရှင်း": "patient",
+        "အရာရှိ": "manager",
+    }
+    
+    def __init__(self):
+        self.graph = nx.MultiDiGraph()
+        self.entity_index: Dict[str, Entity] = {}
+        self.session_id = 0
+    
+    def add_entity(self, entity: Entity) -> None:
+        """Add an entity to the graph."""
+        self.entity_index[entity.id] = entity
+        self.graph.add_node(
+            entity.id,
+            type=entity.type.value,
+            **entity.properties,
+        )
+    
+    def add_relation(self, relation: Relation) -> None:
+        """Add a relation between entities."""
+        self.graph.add_edge(
+            relation.source,
+            relation.target,
+            type=relation.type.value,
+            **relation.properties,
+        )
+    
+    def extract_speaker_entity(
+        self,
+        speaker_id: str,
+        role: Optional[str] = None,
+    ) -> Entity:
+        """Create a speaker entity from utterance metadata."""
+        entity = Entity(
+            id=f"speaker_{speaker_id}",
+            type=NodeType.SPEAKER,
+            properties={
+                "role": role or "unknown",
+                "session": self.session_id,
+            },
+        )
+        self.add_entity(entity)
+        return entity
+    
+    def extract_utterance_entity(
+        self,
+        text: str,
+        speaker_id: str,
+        timestamp: float,
+        prosody: Optional[Dict] = None,
+    ) -> Tuple[Entity, List[Entity], List[Relation]]:
+        """Extract utterance and related entities from text."""
+        utterance_id = f"utt_{speaker_id}_{int(timestamp * 1000)}"
+        
+        utterance = Entity(
+            id=utterance_id,
+            type=NodeType.UTTERANCE,
+            properties={
+                "text": text,
+                "timestamp": timestamp,
+                "prosody": prosody or {},
+            },
+        )
+        self.add_entity(utterance)
+        
+        # Extract related entities
+        related_entities = []
+        relations = []
+        
+        # Extract location mentions
+        for loc, _ in self.LOCATIONS.items():
+            if loc in text:
+                loc_entity = Entity(
+                    id=f"loc_{loc}_{self.session_id}",
+                    type=NodeType.LOCATION,
+                    properties={"name": loc},
+                )
+                self.add_entity(loc_entity)
+                related_entities.append(loc_entity)
+                
+                relation = Relation(
+                    source=utterance_id,
+                    target=loc_entity.id,
+                    type=RelationType.LOCATED_IN,
+                )
+                self.add_relation(relation)
+                relations.append(relation)
+        
+        # Extract emotion mentions
+        for emotion, _ in self.EMOTIONS.items():
+            if emotion in text:
+                emotion_entity = Entity(
+                    id=f"emotion_{emotion}_{self.session_id}",
+                    type=NodeType.EMOTION,
+                    properties={"name": emotion},
+                )
+                self.add_entity(emotion_entity)
+                related_entities.append(emotion_entity)
+                
+                relation = Relation(
+                    source=utterance_id,
+                    target=emotion_entity.id,
+                    type=RelationType.FEELS,
+                )
+                self.add_relation(relation)
+                relations.append(relation)
+        
+        # Link to speaker
+        speaker_entity = self.entity_index.get(f"speaker_{speaker_id}")
+        if speaker_entity:
+            relation = Relation(
+                source=speaker_entity.id,
+                target=utterance_id,
+                type=RelationType.SPEAKS,
+            )
+            self.add_relation(relation)
+            relations.append(relation)
+        
+        return utterance, related_entities, relations
+    
+    def build_from_conversation(
+        self,
+        utterances: List[Dict],
+        context: Optional[Dict] = None,
+    ) -> nx.MultiDiGraph:
+        """Build knowledge graph from conversation data."""
+        self.session_id += 1
+        
+        # Set context entities
+        if context:
+            for key, value in context.items():
+                if key == "location" and value in self.LOCATIONS:
+                    loc_entity = Entity(
+                        id=f"context_location",
+                        type=NodeType.LOCATION,
+                        properties={"name": value},
+                    )
+                    self.add_entity(loc_entity)
+        
+        prev_utterance = None
+        
+        for i, utt_data in enumerate(utterances):
+            speaker_id = utt_data.get("speaker_id", f"s_{i}")
+            text = utt_data.get("text", "")
+            timestamp = utt_data.get("timestamp", i)
+            prosody = utt_data.get("prosody")
+            role = utt_data.get("role")
+            
+            # Add speaker
+            self.extract_speaker_entity(speaker_id, role)
+            
+            # Add utterance
+            utterance, related, _ = self.extract_utterance_entity(
+                text, speaker_id, timestamp, prosody
+            )
+            
+            # Link to previous utterance (temporal relation)
+            if prev_utterance:
+                relation = Relation(
+                    source=prev_utterance.id,
+                    target=utterance.id,
+                    type=RelationType.BEFORE,
+                )
+                self.add_relation(relation)
+                
+                # In response relation
+                response_relation = Relation(
+                    source=utterance.id,
+                    target=prev_utterance.id,
+                    type=RelationType.IN_RESPONSE_TO,
+                )
+                self.add_relation(response_relation)
+            
+            prev_utterance = utterance
+        
+        return self.graph
+    
+    def query_path(
+        self,
+        source_type: NodeType,
+        target_type: NodeType,
+        relation_type: Optional[RelationType] = None,
+    ) -> List[Tuple[Entity, Entity, Relation]]:
+        """Query paths between entity types."""
+        results = []
+        
+        for source_id in self.entity_index:
+            source = self.entity_index[source_id]
+            if source.type != source_type:
+                continue
+            
+            for target_id in self.entity_index:
+                target = self.entity_index[target_id]
+                if target.type != target_type:
+                    continue
+                
+                # Find paths
+                try:
+                    if relation_type:
+                        edges = self.graph.get_edge_data(source_id, target_id)
+                        if edges:
+                            for edge_data in edges.values():
+                                if edge_data.get("type") == relation_type.value:
+                                    relation = Relation(
+                                        source=source_id,
+                                        target=target_id,
+                                        type=relation_type,
+                                        properties=edge_data,
+                                    )
+                                    results.append((source, target, relation))
+                    else:
+                        if nx.has_path(self.graph, source_id, target_id):
+                            path = nx.shortest_path(
+                                self.graph, source_id, target_id
+                            )
+                            if len(path) == 2:
+                                relation = Relation(
+                                    source=source_id,
+                                    target=target_id,
+                                    type=RelationType.CONTAINS,
+                                )
+                                results.append((source, target, relation))
+                except nx.NetworkXError:
+                    continue
+        
+        return results
+    
+    def get_utterance_context(self, utterance_id: str) -> Dict:
+        """Get full context for an utterance."""
+        if utterance_id not in self.entity_index:
+            return {}
+        
+        context = {
+            "utterance": self.entity_index[utterance_id].to_dict(),
+            "speaker": None,
+            "previous": None,
+            "next": None,
+            "locations": [],
+            "emotions": [],
+        }
+        
+        # Get speaker
+        for edge in self.graph.out_edges(utterance_id, data=True):
+            if edge[2].get("type") == RelationType.FEELS.value:
+                context["emotions"].append(self.entity_index[edge[1]].to_dict())
+            if edge[2].get("type") == RelationType.LOCATED_IN.value:
+                context["locations"].append(self.entity_index[edge[1]].to_dict())
+        
+        # Get predecessor/successor
+        predecessors = list(self.graph.predecessors(utterance_id))
+        successors = list(self.graph.successors(utterance_id))
+        
+        for pred_id in predecessors:
+            pred = self.entity_index.get(pred_id)
+            if pred and pred.type == NodeType.UTTERANCE:
+                context["previous"] = pred.to_dict()
+                break
+        
+        for succ_id in successors:
+            succ = self.entity_index.get(succ_id)
+            if succ and succ.type == NodeType.UTTERANCE:
+                context["next"] = succ.to_dict()
+                break
+        
+        return context
+    
+    def export_to_json(self, path: str) -> None:
+        """Export graph to JSON format."""
+        entities = [e.to_dict() for e in self.entity_index.values()]
+        
+        relations = []
+        for source, target, data in self.graph.edges(data=True):
+            relations.append({
+                "source": source,
+                "target": target,
+                "type": data.get("type"),
+                **data,
+            })
+        
+        output = {
+            "entities": entities,
+            "relations": relations,
+            "metadata": {
+                "num_entities": len(entities),
+                "num_relations": len(relations),
+                "session_id": self.session_id,
+            },
+        }
+        
+        with open(path, "w", encoding="utf-8") as f:
+            json.dump(output, f, indent=2, ensure_ascii=False)
+    
+    def load_from_json(self, path: str) -> None:
+        """Load graph from JSON format."""
+        with open(path, "r", encoding="utf-8") as f:
+            data = json.load(f)
+        
+        self.entity_index = {}
+        self.graph = nx.MultiDiGraph()
+        
+        for entity_data in data.get("entities", []):
+            entity = Entity(
+                id=entity_data["id"],
+                type=NodeType(entity_data["type"]),
+                properties=entity_data.get("properties", {}),
+                aliases=entity_data.get("aliases", []),
+            )
+            self.add_entity(entity)
+        
+        for rel_data in data.get("relations", []):
+            relation = Relation(
+                source=rel_data["source"],
+                target=rel_data["target"],
+                type=RelationType(rel_data["type"]),
+                properties=rel_data,
+                confidence=rel_data.get("confidence", 1.0),
+            )
+            self.add_relation(relation)
+    
+    def visualize(self) -> nx.MultiDiGraph:
+        """Return the graph for visualization."""
+        return self.graph
+
+
+def create_knowledge_graph() -> MyanmarKnowledgeGraph:
+    """Factory function to create knowledge graph."""
+    return MyanmarKnowledgeGraph()
+
+
+if __name__ == "__main__":
+    # Example usage
+    kg = create_knowledge_graph()
+    
+    # Sample conversation
+    utterances = [
+        {
+            "speaker_id": "customer_1",
+            "text": "ဆိုင်သို့ ကျွန်ုပ်လာပါပြီ",
+            "timestamp": 0,
+            "role": "customer",
+        },
+        {
+            "speaker_id": "staff_1",
+            "text": "ကြိုဆိုပါတယ်",
+            "timestamp": 1,
+            "role": "staff",
+        },
+        {
+            "speaker_id": "customer_1",
+            "text": "ကျေးဇူးပါ",
+            "timestamp": 2,
+            "prosody": {"mean_pitch": 150, "speaking_rate": 3},
+            "role": "customer",
+        },
+    ]
+    
+    context = {"location": "စားသောက်ဆိုင်"}
+    
+    kg.build_from_conversation(utterances, context)
+    
+    # Export
+    kg.export_to_json("data/graph/conversation_graph.json")
+    print(f"Graph exported with {len(kg.entity_index)} entities")

data_processing/multimodal_fusion.py

@@ -0,0 +1,371 @@
+"""Multi-modal data fusion for Myanmar Ghost project.
+
+Fuses audio (prosody) and text to understand sentiment/intensity
+in expressions like "ကျေးဇူးပါ" (thank you) which can mean:
+- Genuine gratitude (low pitch, slow)
+- Sarcasm (high pitch, fast)
+- Complaint (negative prosody)
+"""
+
+from dataclasses import dataclass
+from enum import Enum
+from typing import Any, Dict, List, Optional, Tuple
+
+import numpy as np
+import torch
+import torch.nn as nn
+from torch import Tensor
+
+
+class SentimentClass(str, Enum):
+    """Sentiment classes for thanking expressions."""
+    GENUINE = "genuine"          # ရိုးသားခြင်း
+    SARCASTIC = "sarcastic"      # သရော်ခြင်း
+    COMPLAINING = "complaining"  # မကျေနပ်ခြင်း
+    NEUTRAL = "neutral"
+
+
+@dataclass
+class ProsodyFeatures:
+    """Prosodic features extracted from audio."""
+    mean_pitch: float
+    pitch_std: float
+    pitch_range: Tuple[float, float]
+    mean_energy: float
+    energy_std: float
+    speaking_rate: float  # syllables per second
+    pause_duration: float  # total pause time in seconds
+    
+    def to_tensor(self) -> Tensor:
+        """Convert to PyTorch tensor."""
+        return torch.tensor([
+            self.mean_pitch,
+            self.pitch_std,
+            self.pitch_range[0],
+            self.pitch_range[1],
+            self.mean_energy,
+            self.energy_std,
+            self.speaking_rate,
+            self.pause_duration,
+        ], dtype=torch.float32)
+    
+    def to_dict(self) -> Dict[str, float]:
+        """Convert to dictionary."""
+        return {
+            "mean_pitch": self.mean_pitch,
+            "pitch_std": self.pitch_std,
+            "pitch_min": self.pitch_range[0],
+            "pitch_max": self.pitch_range[1],
+            "mean_energy": self.mean_energy,
+            "energy_std": self.energy_std,
+            "speaking_rate": self.speaking_rate,
+            "pause_duration": self.pause_duration,
+        }
+
+
+@dataclass
+class TextFeatures:
+    """Text-based features for sentiment analysis."""
+    text_length: int
+    word_count: int
+    contains_intensifier: bool  # e.g., "အရမ်း", "များစွာ"
+    politeness_level: int  # 1-5 scale
+    formality: float  # 0-1 scale
+    
+    def to_tensor(self) -> Tensor:
+        """Convert to PyTorch tensor."""
+        return torch.tensor([
+            float(self.text_length),
+            float(self.word_count),
+            float(self.contains_intensifier),
+            float(self.politeness_level),
+            self.formality,
+        ], dtype=torch.float32)
+
+
+@dataclass
+class FusedFeatures:
+    """Combined multi-modal features."""
+    prosody: ProsodyFeatures
+    text: TextFeatures
+    sentiment_hint: Optional[SentimentClass] = None
+    
+    def concat_tensors(self) -> Tensor:
+        """Concatenate all features into single tensor."""
+        return torch.cat([
+            self.prosody.to_tensor(),
+            self.text.to_tensor(),
+        ])
+
+
+class ProsodyExtractor:
+    """Extract prosodic features from audio."""
+    
+    # Prosody patterns for different sentiments
+    GENUINE_PATTERN = {
+        "pitch_range": (50, 200),  # Hz
+        "speaking_rate": (2, 4),   # syllables/sec
+        "energy_std": (0.1, 0.3),
+    }
+    
+    SARCASTIC_PATTERN = {
+        "pitch_range": (200, 400),
+        "speaking_rate": (4, 8),
+        "energy_std": (0.3, 0.6),
+    }
+    
+    COMPLAINING_PATTERN = {
+        "pitch_range": (100, 250),
+        "speaking_rate": (3, 6),
+        "energy_std": (0.2, 0.5),
+    }
+    
+    def extract_from_audio(
+        self,
+        audio: np.ndarray,
+        sample_rate: int = 16000,
+    ) -> ProsodyFeatures:
+        """Extract prosodic features from audio signal."""
+        import librosa
+        
+        # Pitch tracking
+        pitches, magnitudes = librosa.piptrack(
+            y=audio,
+            sr=sample_rate,
+            n_fft=512,
+            hop_length=160,
+        )
+        
+        pitch_values = []
+        for i in range(pitches.shape[1]):
+            index = magnitudes[:, i].argmax()
+            pitch = pitches[index, i]
+            if pitch > 0:
+                pitch_values.append(pitch)
+        
+        # Energy
+        rms = librosa.feature.rms(y=audio, hop_length=160)[0]
+        
+        # Speaking rate (syllable detection)
+        onsets = librosa.onset.onset_detect(
+            y=audio,
+            sr=sample_rate,
+            hop_length=160,
+        )
+        
+        duration = len(audio) / sample_rate
+        speaking_rate = len(onsets) / duration if duration > 0 else 0
+        
+        # Pause detection
+        energy_threshold = np.percentile(rms, 25)
+        pauses = rms < energy_threshold
+        pause_duration = np.sum(pauses) * 160 / sample_rate
+        
+        return ProsodyFeatures(
+            mean_pitch=np.mean(pitch_values) if pitch_values else 0,
+            pitch_std=np.std(pitch_values) if pitch_values else 0,
+            pitch_range=(
+                np.min(pitch_values) if pitch_values else 0,
+                np.max(pitch_values) if pitch_values else 0,
+            ),
+            mean_energy=np.mean(rms),
+            energy_std=np.std(rms),
+            speaking_rate=speaking_rate,
+            pause_duration=pause_duration,
+        )
+    
+    def infer_sentiment(self, prosody: ProsodyFeatures) -> SentimentClass:
+        """Infer sentiment from prosodic features."""
+        patterns = [
+            (SentimentClass.GENUINE, self.GENUINE_PATTERN),
+            (SentimentClass.SARCASTIC, self.SARCASTIC_PATTERN),
+            (SentimentClass.COMPLAINING, self.COMPLAINING_PATTERN),
+        ]
+        
+        scores = {}
+        for sentiment, pattern in patterns:
+            score = 0
+            features = prosody.to_dict()
+            
+            for key, (low, high) in pattern.items():
+                if key in features:
+                    value = features[key]
+                    if low <= value <= high:
+                        score += 1
+            
+            scores[sentiment] = score
+        
+        return max(scores, key=scores.get)
+
+
+class TextFeatureExtractor:
+    """Extract text-based features."""
+    
+    INTENSIFIERS = {"အရမ်း", "များစွာ", "ပါး", "သိပ်", "အလွန်"}
+    POLITE_WORDS = {"ကျေးဇူး", "�心病", "ဂုဏ်", "အား", "ကြိုးစား", "ပင်ပန်း"}
+    
+    def extract_from_text(self, text: str) -> TextFeatures:
+        """Extract features from text."""
+        words = text.split()
+        
+        has_intensifier = any(
+            word in self.INTENSIFIERS for word in words
+        )
+        
+        politeness = self._estimate_politeness(text)
+        formality = self._estimate_formality(text)
+        
+        return TextFeatures(
+            text_length=len(text),
+            word_count=len(words),
+            contains_intensifier=has_intensifier,
+            politeness_level=politeness,
+            formality=formality,
+        )
+    
+    def _estimate_politeness(self, text: str) -> int:
+        """Estimate politeness level (1-5)."""
+        score = 3  # default neutral
+        polite_count = sum(1 for w in self.POLITE_WORDS if w in text)
+        if "ပါ" in text or "ပါး" in text:
+            score += 1
+        if "ကျေးဇူး" in text:
+            score += 1
+        if polite_count > 2:
+            score += 1
+        return min(5, max(1, score))
+    
+    def _estimate_formality(self, text: str) -> float:
+        """Estimate formality (0-1)."""
+        formal_markers = {"မှ", "သည်", "ကို", "ဖြင့်", "အား"}
+        informal_markers = {"နော်", "ဟုတ်", "မဟုတ်", "လား"}
+        
+        formal_count = sum(1 for m in formal_markers if m in text)
+        informal_count = sum(1 for m in informal_markers if m in text)
+        
+        if formal_count + informal_count == 0:
+            return 0.5
+        
+        return formal_count / (formal_count + informal_count + 1)
+
+
+class MultiModalFusion(nn.Module):
+    """Fuse audio and text modalities."""
+    
+    def __init__(
+        self,
+        prosody_dim: int = 8,
+        text_dim: int = 5,
+        hidden_dim: int = 64,
+        num_classes: int = 4,
+    ):
+        super().__init__()
+        
+        self.prosody_encoder = nn.Sequential(
+            nn.Linear(prosody_dim, hidden_dim),
+            nn.ReLU(),
+            nn.Dropout(0.2),
+        )
+        
+        self.text_encoder = nn.Sequential(
+            nn.Linear(text_dim, hidden_dim),
+            nn.ReLU(),
+            nn.Dropout(0.2),
+        )
+        
+        self.fusion = nn.Sequential(
+            nn.Linear(hidden_dim * 2, hidden_dim),
+            nn.ReLU(),
+            nn.Dropout(0.3),
+            nn.Linear(hidden_dim, num_classes),
+        )
+    
+    def forward(self, prosody: Tensor, text: Tensor) -> Tensor:
+        """Forward pass."""
+        p_encoded = self.prosody_encoder(prosody)
+        t_encoded = self.text_encoder(text)
+        
+        fused = torch.cat([p_encoded, t_encoded], dim=-1)
+        logits = self.fusion(fused)
+        
+        return logits
+    
+    def predict(self, prosody: Tensor, text: Tensor) -> Tuple[Tensor, Tensor]:
+        """Predict sentiment with probabilities."""
+        logits = self.forward(prosody, text)
+        probs = torch.softmax(logits, dim=-1)
+        return logits, probs
+
+
+class SentimentClassifier:
+    """High-level classifier for multi-modal sentiment."""
+    
+    def __init__(self, model: MultiModalFusion):
+        self.model = model
+        self.prosody_extractor = ProsodyExtractor()
+        self.text_extractor = TextFeatureExtractor()
+    
+    def classify(
+        self,
+        audio: np.ndarray,
+        text: str,
+        return_probs: bool = True,
+    ) -> Dict[str, Any]:
+        """Classify sentiment from audio and text."""
+        prosody_features = self.prosody_extractor.extract_from_audio(audio)
+        prosody_hint = self.prosody_extractor.infer_sentiment(prosody_features)
+        
+        text_features = self.text_extractor.extract_from_text(text)
+        
+        fused = FusedFeatures(
+            prosody=prosody_features,
+            text=text_features,
+            sentiment_hint=prosody_hint,
+        )
+        
+        prosody_tensor = fused.prosody.to_tensor().unsqueeze(0)
+        text_tensor = fused.text.to_tensor().unsqueeze(0)
+        
+        with torch.no_grad():
+            logits, probs = self.model.predict(prosody_tensor, text_tensor)
+        
+        result = {
+            "predicted_class": SentimentClass(probs.argmax().item()).value,
+            "prosody_hint": prosody_hint.value,
+            "text_features": text_features.to_dict(),
+            "prosody_features": prosody_features.to_dict(),
+        }
+        
+        if return_probs:
+            result["probabilities"] = {
+                c.value: probs[0, i].item()
+                for i, c in enumerate(SentimentClass)
+            }
+        
+        return result
+
+
+def create_fusion_model(
+    prosody_dim: int = 8,
+    text_dim: int = 5,
+    hidden_dim: int = 64,
+    num_classes: int = 4,
+) -> MultiModalFusion:
+    """Factory function to create fusion model."""
+    return MultiModalFusion(
+        prosody_dim=prosody_dim,
+        text_dim=text_dim,
+        hidden_dim=hidden_dim,
+        num_classes=num_classes,
+    )
+
+
+if __name__ == "__main__":
+    # Example usage
+    model = create_fusion_model()
+    prosody = torch.randn(1, 8)
+    text = torch.randn(1, 5)
+    
+    logits, probs = model.predict(prosody, text)
+    print(f"Predicted class: {SentimentClass(probs.argmax().item()).value}")
+    print(f"Probabilities: {probs}")

data_processing/text_normalizer.py

@@ -0,0 +1,210 @@
+"""Text normalization module for Myanmar language."""
+
+import re
+import unicodedata
+from pathlib import Path
+from typing import Dict, List, Optional
+
+import pandas as pd
+import yaml
+
+logger = __import__("loguru").logger
+
+
+class MyanmarTextNormalizer:
+    """Normalize Myanmar (Burmese) text for consistent processing."""
+
+    # Myanmar Unicode ranges
+    MYANMAR_CHARS = re.compile(
+        r"[\u1000-\u100F\u1010-\u101F\u1020-\u102A\u102C-\u1030\u1031\u1032\u1036-\u1038\u1039\u103A]"
+    )
+    
+    # Normalization rules
+    NORMALIZATION_RULES = {
+        # Zero-width characters
+        "\u200B": "",  # Zero-width space
+        "\u200C": "",  # Zero-width non-joiner
+        "\u200D": "",  # Zero-width joiner
+        "\u2060": "",  # Word joiner
+        # Myanmar-specific normalizations
+        "\u1031\u103B": "\u103B\u1031",  # medial order
+        "\u103D\u103E": "\u103E\u103D",  # stack order
+    }
+
+    def __init__(self, custom_rules_path: Optional[str] = None):
+        self.custom_rules = {}
+        if custom_rules_path and Path(custom_rules_path).exists():
+            with open(custom_rules_path, "r", encoding="utf-8") as f:
+                self.custom_rules = yaml.safe_load(f) or {}
+        
+        self.rules = {**self.NORMALIZATION_RULES, **self.custom_rules}
+
+    def normalize_unicode(self, text: str) -> str:
+        """Standardize Unicode representation (NFC normalization)."""
+        return unicodedata.normalize("NFC", text)
+
+    def remove_diacritics(self, text: str) -> str:
+        """Remove tone marks for simplified processing."""
+        diacritics = re.compile(
+            r"[\u102B-\u102D\u102F-\u1032\u1034\u1036\u1037\u1039]"
+        )
+        return diacritics.sub("", text)
+
+    def remove_whitespace(self, text: str) -> str:
+        """Remove excessive whitespace."""
+        text = re.sub(r"\s+", " ", text)
+        return text.strip()
+
+    def normalize_punctuation(self, text: str) -> str:
+        """Standardize punctuation marks."""
+        replacements = {
+            "၊": "။",  # Myanmar comma to full stop
+            "„": '"',
+            "‟": '"',
+            "'": "'",
+            "`": "'",
+            "—": "–",
+            "–": "-",
+        }
+        for old, new in replacements.items():
+            text = text.replace(old, new)
+        return text
+
+    def apply_custom_rules(self, text: str) -> str:
+        """Apply user-defined normalization rules."""
+        for pattern, replacement in self.rules.items():
+            text = text.replace(pattern, replacement)
+        return text
+
+    def expand_abbreviations(self, text: str, abbreviations: Dict[str, str] = None) -> str:
+        """Expand common abbreviations."""
+        if abbreviations is None:
+            abbreviations = {
+                "အ.ပ.ခ": "အငြိမ်းစားပြည်ထဲရေးဝန်ကြီး",
+                "ဒ.ပ.လ": "ဒုတိယသမ္မတ",
+                "ပ.ရ.မှူး": "ပြည်သူ့လွှတ်တော်ဥက္ကဋ္ဌ",
+            }
+        
+        for abbr, full in abbreviations.items():
+            text = re.sub(rf"\b{re.escape(abbr)}\b", full, text)
+        
+        return text
+
+    def normalize_numbers(self, text: str) -> str:
+        """Convert Myanmar numerals to Arabic (0-9)."""
+        myanmar_digits = "၀၁၂၃၄၅၆၇၈၉"
+        arabic_digits = "0123456789"
+        
+        trans_table = str.maketrans(
+            {myanmar_digits[i]: arabic_digits[i] for i in range(10)}
+        )
+        return text.translate(trans_table)
+
+    def filter_non_myanmar(self, text: str, keep_english: bool = True) -> str:
+        """Remove or keep non-Myanmar characters."""
+        if keep_english:
+            pattern = r"[^\u1000-\u109F\u0020-\u007E\u00A0-\u00FF]"
+        else:
+            pattern = r"[^\u1000-\u109F\s]"
+        
+        return re.sub(pattern, "", text)
+
+    def normalize_line(self, text: str) -> str:
+        """Apply all normalization steps to a single line."""
+        text = self.normalize_unicode(text)
+        text = self.apply_custom_rules(text)
+        text = self.remove_whitespace(text)
+        text = self.normalize_punctuation(text)
+        return text
+
+    def normalize_corpus(
+        self,
+        texts: List[str],
+        remove_non_myanmar: bool = False,
+    ) -> List[str]:
+        """Normalize a list of texts."""
+        normalized = []
+        for text in texts:
+            text = self.normalize_line(text)
+            if remove_non_myanmar:
+                text = self.filter_non_myanmar(text, keep_english=False)
+            normalized.append(text)
+        
+        logger.info(f"Normalized {len(normalized)} texts")
+        return normalized
+
+    def normalize_dataset(
+        self,
+        input_path: str,
+        output_path: str,
+        text_column: str = "text",
+    ) -> pd.DataFrame:
+        """Normalize a dataset and save to file."""
+        df = pd.read_csv(input_path)
+        
+        if text_column not in df.columns:
+            raise ValueError(f"Column '{text_column}' not found in dataset")
+        
+        df[f"{text_column}_normalized"] = self.normalize_corpus(
+            df[text_column].tolist()
+        )
+        
+        df.to_csv(output_path, index=False)
+        logger.info(f"Normalized dataset saved to {output_path}")
+        
+        return df
+
+
+class ProsodyNormalizer:
+    """Normalize prosodic features for consistent representation."""
+
+    def normalize_pitch(self, pitch_values: List[float]) -> List[float]:
+        """Normalize pitch values to semitones from mean."""
+        import numpy as np
+        pitch_arr = np.array(pitch_values)
+        mean_pitch = np.mean(pitch_arr[pitch_arr > 0])
+        if mean_pitch == 0:
+            return pitch_values
+        semitones = 12 * np.log2(pitch_arr / mean_pitch)
+        return semitones.tolist()
+
+    def normalize_energy(self, energy_values: List[float]) -> List[float]:
+        """Normalize energy values to 0-1 range."""
+        import numpy as np
+        energy_arr = np.array(energy_values)
+        min_e, max_e = energy_arr.min(), energy_arr.max()
+        if max_e - min_e == 0:
+            return [0.5] * len(energy_values)
+        return ((energy_arr - min_e) / (max_e - min_e)).tolist()
+
+    def quantize_prosody(
+        self,
+        prosody: dict,
+        num_levels: int = 5,
+    ) -> dict:
+        """Quantize prosodic features for categorical representation."""
+        quantized = {}
+        for key, value in prosody.items():
+            if isinstance(value, (int, float)) and key != "pitch_range":
+                normalized = max(0, min(1, (value + 100) / 200))
+                quantized[key] = int(normalized * (num_levels - 1))
+            else:
+                quantized[key] = value
+        return quantized
+
+
+def create_normalizer(config: dict = None) -> MyanmarTextNormalizer:
+    """Factory function to create normalizer from config."""
+    if config is None:
+        config = {}
+    
+    return MyanmarTextNormalizer(
+        custom_rules_path=config.get("custom_rules_path")
+    )
+
+
+if __name__ == "__main__":
+    normalizer = create_normalizer()
+    test_text = "  မင်္ဂလာပါ   ၊  ကျေးဇူးပါ  ပါ   သည်  "
+    print(f"Original: {test_text}")
+    print(f"Normalized: {normalizer.normalize_line(test_text)}")

federated/aggregator.py

@@ -0,0 +1,283 @@
+"""Weight aggregation algorithms for Federated Learning.
+
+Implements various aggregation methods beyond simple FedAvg:
+- FedProx (proximal term)
+- SCAFFOLD (variance reduction)
+- FedOpt (adaptive optimization)
+"""
+
+from abc import ABC, abstractmethod
+from typing import Any, Dict, List, Optional, Tuple
+
+import numpy as np
+import torch
+
+
+class Aggregator(ABC):
+    """Base class for federated aggregation algorithms."""
+    
+    @abstractmethod
+    def aggregate(
+        self,
+        parameters: List[List[np.ndarray]],
+        weights: List[float],
+        client_metrics: List[Dict],
+    ) -> List[np.ndarray]:
+        """Aggregate client parameters.
+        
+        Args:
+            parameters: List of client parameters
+            weights: Weight for each client (usually proportional to data size)
+            client_metrics: Metrics from each client
+        
+        Returns:
+            Aggregated parameters
+        """
+        pass
+
+
+class FedAvgAggregator(Aggregator):
+    """Federated Averaging (FedAvg) - standard weighted average."""
+    
+    def aggregate(
+        self,
+        parameters: List[List[np.ndarray]],
+        weights: List[float],
+        client_metrics: List[Dict],
+    ) -> List[np.ndarray]:
+        """Weighted average of client parameters."""
+        if not parameters:
+            raise ValueError("No parameters to aggregate")
+        
+        if len(parameters) == 1:
+            return parameters[0]
+        
+        # Normalize weights
+        total_weight = sum(weights)
+        normalized_weights = [w / total_weight for w in weights]
+        
+        # Weighted average
+        aggregated = None
+        
+        for params, weight in zip(parameters, normalized_weights):
+            if aggregated is None:
+                aggregated = [p * weight for p in params]
+            else:
+                aggregated = [
+                    a + p * weight
+                    for a, p in zip(aggregated, params)
+                ]
+        
+        return aggregated
+
+
+class FedProxAggregator(Aggregator):
+    """FedProx with proximal term for handling heterogeneity."""
+    
+    def __init__(self, mu: float = 0.01):
+        """
+        Args:
+            mu: Proximal term coefficient
+        """
+        self.mu = mu
+    
+    def aggregate(
+        self,
+        parameters: List[List[np.ndarray]],
+        weights: List[float],
+        client_metrics: List[Dict],
+        global_parameters: Optional[List[np.ndarray]] = None,
+    ) -> List[np.ndarray]:
+        """Aggregate with proximal term regularization."""
+        if global_parameters is None:
+            # Fall back to FedAvg if no global params
+            return FedAvgAggregator().aggregate(parameters, weights, client_metrics)
+        
+        # Weighted average with proximal correction
+        total_weight = sum(weights)
+        normalized_weights = [w / total_weight for w in weights]
+        
+        aggregated = None
+        
+        for params, weight, global_params in zip(
+            parameters, normalized_weights, global_parameters
+        ):
+            # Proximal term: add regularization toward global model
+            corrected_params = [
+                p + self.mu * (g - p)
+                for p, g in zip(params, global_params)
+            ]
+            
+            if aggregated is None:
+                aggregated = [p * weight for p in corrected_params]
+            else:
+                aggregated = [
+                    a + p * weight
+                    for a, p in zip(aggregated, corrected_params)
+                ]
+        
+        return aggregated
+
+
+class SCAFFOLDAggregator(Aggregator):
+    """SCAFFOLD: Stochastic Controlled Averaging for Federated Learning."""
+    
+    def __init__(self, global_control: Optional[List[np.ndarray]] = None):
+        self.global_control = global_control or None
+    
+    def set_global_control(self, control: List[np.ndarray]) -> None:
+        """Set global control variates."""
+        self.global_control = control
+    
+    def aggregate(
+        self,
+        parameters: List[List[np.ndarray]],
+        weights: List[float],
+        client_metrics: List[Dict],
+        client_controls: Optional[List[List[np.ndarray]]] = None,
+    ) -> List[np.ndarray]:
+        """Aggregate using SCAFFOLD algorithm."""
+        if client_controls is None or self.global_control is None:
+            return FedAvgAggregator().aggregate(parameters, weights, client_metrics)
+        
+        total_weight = sum(weights)
+        normalized_weights = [w / total_weight for w in weights]
+        
+        # Compute weight updates (gradient-like terms)
+        aggregated = None
+        
+        for params, weight, client_ctrl, global_ctrl in zip(
+            parameters, normalized_weights, client_controls, self.global_control
+        ):
+            # Direction: client_params - global_params + global_control - client_control
+            delta = [
+                p - g + gc - cc
+                for p, g, gc, cc in zip(params, parameters[0], self.global_control, client_ctrl)
+            ]
+            
+            if aggregated is None:
+                aggregated = [d * weight for d in delta]
+            else:
+                aggregated = [
+                    a + d * weight
+                    for a, d in zip(aggregated, delta)
+                ]
+        
+        # Add back global parameters
+        if aggregated:
+            aggregated = [
+                g + self.mu * a if hasattr(self, 'mu') else g + 0.001 * a
+                for g, a in zip(self.global_control if self.global_control else parameters[0], aggregated)
+            ]
+        
+        return aggregated
+    
+    @property
+    def mu(self) -> float:
+        """Learning rate for SCAFFOLD."""
+        return 0.001
+
+
+class FedOptAggregator(Aggregator):
+    """FedOpt: Adaptive Federated Optimization using server-side optimizer."""
+    
+    def __init__(
+        self,
+        server_lr: float = 1.0,
+        beta_1: float = 0.9,
+        beta_2: float = 0.99,
+        epsilon: float = 1e-4,
+    ):
+        self.server_lr = server_lr
+        self.beta_1 = beta_1
+        self.beta_2 = beta_2
+        self.epsilon = epsilon
+        
+        self.m_t: Optional[List[np.ndarray]] = None
+        self.v_t: Optional[List[np.ndarray]] = None
+        self.t = 0
+    
+    def aggregate(
+        self,
+        parameters: List[List[np.ndarray]],
+        weights: List[float],
+        client_metrics: List[Dict],
+    ) -> List[np.ndarray]:
+        """Aggregate using FedOpt (server-side Adam)."""
+        if not parameters:
+            raise ValueError("No parameters to aggregate")
+        
+        # FedAvg as base
+        base_aggregate = FedAvgAggregator().aggregate(parameters, weights, client_metrics)
+        
+        # Compute delta from base to weighted average
+        if self.m_t is None:
+            self.m_t = [np.zeros_like(p) for p in base_aggregate]
+            self.v_t = [np.zeros_like(p) for p in base_aggregate]
+        
+        delta = [
+            ba - p0
+            for ba, p0 in zip(base_aggregate, parameters[0])
+        ]
+        
+        self.t += 1
+        
+        # Update momentum and second moment
+        self.m_t = [
+            self.beta_1 * m + (1 - self.beta_1) * d
+            for m, d in zip(self.m_t, delta)
+        ]
+        
+        self.v_t = [
+            self.beta_2 * v + (1 - self.beta_2) * (d ** 2)
+            for v, d in zip(self.v_t, delta)
+        ]
+        
+        # Bias correction
+        m_hat = [m / (1 - self.beta_1 ** self.t) for m in self.m_t]
+        v_hat = [v / (1 - self.beta_2 ** self.t) for v in self.v_t]
+        
+        # Apply update
+        aggregated = [
+            p0 + self.server_lr * m / (np.sqrt(v) + self.epsilon)
+            for p0, m, v in zip(parameters[0], m_hat, v_hat)
+        ]
+        
+        return aggregated
+
+
+def create_aggregator(
+    method: str = "fedavg",
+    **kwargs,
+) -> Aggregator:
+    """Factory function to create aggregator.
+    
+    Args:
+        method: Aggregation method ("fedavg", "fedprox", "scaffold", "fedopt")
+        **kwargs: Additional arguments for the aggregator
+    
+    Returns:
+        Aggregator instance
+    """
+    if method == "fedavg":
+        return FedAvgAggregator()
+    elif method == "fedprox":
+        return FedProxAggregator(mu=kwargs.get("mu", 0.01))
+    elif method == "scaffold":
+        return SCAFFOLDAggregator()
+    elif method == "fedopt":
+        return FedOptAggregator(
+            server_lr=kwargs.get("server_lr", 1.0),
+            beta_1=kwargs.get("beta_1", 0.9),
+            beta_2=kwargs.get("beta_2", 0.99),
+        )
+    else:
+        raise ValueError(f"Unknown aggregation method: {method}")
+
+
+if __name__ == "__main__":
+    print("Available aggregators: fedavg, fedprox, scaffold, fedopt")
+    
+    # Example usage
+    agg = create_aggregator("fedavg")
+    print(f"Created aggregator: {type(agg).__name__}")

federated/client.py

@@ -0,0 +1,272 @@
+"""Federated Learning Client for Myanmar Ghost project.
+
+Enables training on distributed data (e.g., hospitals, restaurants)
+without centralizing sensitive data.
+"""
+
+import flwr as fl
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.optim as optim
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Tuple
+import yaml
+
+logger = __import__("loguru").logger
+
+
+class FederatedClient(fl.client.NumPyClient):
+    """Flower client for federated learning."""
+    
+    def __init__(
+        self,
+        model: nn.Module,
+        trainloader,
+        valloader,
+        device: str = "cuda" if torch.cuda.is_available() else "cpu",
+        client_id: str = "client_1",
+        output_dir: str = "outputs/federated",
+    ):
+        self.model = model.to(device)
+        self.trainloader = trainloader
+        self.valloader = valloader
+        self.device = device
+        self.client_id = client_id
+        self.output_dir = Path(output_dir)
+        self.output_dir.mkdir(parents=True, exist_ok=True)
+        
+        self.criterion = nn.CrossEntropyLoss()
+        self.optimizer = optim.AdamW(self.model.parameters(), lr=1e-4)
+    
+    def get_parameters(self) -> List[np.ndarray]:
+        """Get model parameters as numpy arrays."""
+        return [val.cpu().numpy() for _, val in self.model.state_dict().items()]
+    
+    def set_parameters(self, parameters: List[np.ndarray]) -> None:
+        """Set model parameters from numpy arrays."""
+        state_dict = dict(self.model.state_dict())
+        for i, (key, _) in enumerate(state_dict.items()):
+            state_dict[key] = torch.from_numpy(parameters[i])
+        self.model.load_state_dict(state_dict)
+    
+    def fit(
+        self,
+        parameters: List[np.ndarray],
+        config: Dict[str, Any],
+    ) -> Tuple[List[np.ndarray], int, Dict]:
+        """Train model on local data.
+        
+        Args:
+            parameters: Global model parameters
+            config: Training configuration
+        
+        Returns:
+            Updated parameters, number of samples, metrics
+        """
+        # Set global parameters
+        self.set_parameters(parameters)
+        
+        # Training configuration
+        epochs = config.get("local_epochs", 1)
+        batch_size = config.get("batch_size", 32)
+        learning_rate = config.get("learning_rate", 1e-4)
+        
+        self.optimizer = optim.AdamW(
+            self.model.parameters(),
+            lr=learning_rate,
+        )
+        
+        # Local training
+        self.model.train()
+        total_loss = 0.0
+        total_samples = 0
+        
+        for epoch in range(epochs):
+            epoch_loss = 0.0
+            epoch_samples = 0
+            
+            for batch_idx, (inputs, labels) in enumerate(self.trainloader):
+                inputs = inputs.to(self.device)
+                labels = labels.to(self.device)
+                
+                self.optimizer.zero_grad()
+                outputs = self.model(inputs)
+                loss = self.criterion(outputs, labels)
+                loss.backward()
+                self.optimizer.step()
+                
+                epoch_loss += loss.item() * inputs.size(0)
+                epoch_samples += inputs.size(0)
+            
+            total_loss += epoch_loss
+            total_samples += epoch_samples
+            
+            logger.info(
+                f"Client {self.client_id} - Epoch {epoch+1}/{epochs}: "
+                f"Loss={epoch_loss/epoch_samples:.4f}"
+            )
+        
+        # Save checkpoint
+        self._save_checkpoint(epochs)
+        
+        metrics = {
+            "loss": total_loss / total_samples,
+            "samples": total_samples,
+            "epochs": epochs,
+        }
+        
+        return self.get_parameters(), total_samples, metrics
+    
+    def evaluate(
+        self,
+        parameters: List[np.ndarray],
+        config: Dict[str, Any],
+    ) -> Tuple[float, int, Dict]:
+        """Evaluate model on local validation data.
+        
+        Args:
+            parameters: Model parameters
+            config: Evaluation configuration
+        
+        Returns:
+            Loss, number of samples, metrics
+        """
+        self.set_parameters(parameters)
+        
+        self.model.eval()
+        total_loss = 0.0
+        total_correct = 0
+        total_samples = 0
+        
+        with torch.no_grad():
+            for inputs, labels in self.valloader:
+                inputs = inputs.to(self.device)
+                labels = labels.to(self.device)
+                
+                outputs = self.model(inputs)
+                loss = self.criterion(outputs, labels)
+                
+                total_loss += loss.item() * inputs.size(0)
+                _, predicted = outputs.max(1)
+                total_correct += predicted.eq(labels).sum().item()
+                total_samples += inputs.size(0)
+        
+        accuracy = total_correct / total_samples if total_samples > 0 else 0.0
+        
+        logger.info(
+            f"Client {self.client_id} - Evaluation: "
+            f"Loss={total_loss/total_samples:.4f}, Accuracy={accuracy:.4f}"
+        )
+        
+        metrics = {
+            "loss": total_loss / total_samples,
+            "accuracy": accuracy,
+            "samples": total_samples,
+        }
+        
+        return total_loss / total_samples, total_samples, metrics
+    
+    def _save_checkpoint(self, epochs: int) -> None:
+        """Save model checkpoint."""
+        path = self.output_dir / f"{self.client_id}_checkpoint.pt"
+        torch.save({
+            "model_state_dict": self.model.state_dict(),
+            "optimizer_state_dict": self.optimizer.state_dict(),
+            "epochs": epochs,
+            "client_id": self.client_id,
+        }, path)
+        logger.info(f"Checkpoint saved to {path}")
+
+
+def load_client_config(config_path: str) -> Dict:
+    """Load client configuration from YAML."""
+    with open(config_path, "r", encoding="utf-8") as f:
+        return yaml.safe_load(f)
+
+
+class ClientFactory:
+    """Factory for creating federated clients."""
+    
+    def __init__(
+        self,
+        model_fn,
+        data_dir: str,
+        output_dir: str = "outputs/federated",
+    ):
+        self.model_fn = model_fn
+        self.data_dir = Path(data_dir)
+        self.output_dir = Path(output_dir)
+    
+    def create_client(
+        self,
+        client_id: str,
+        config: Dict,
+    ) -> FederatedClient:
+        """Create a client for the given configuration."""
+        from torch.utils.data import DataLoader
+        
+        # Load data
+        train_data = self._load_partition(
+            client_id,
+            config.get("train_file", f"{client_id}_train.pt"),
+        )
+        val_data = self._load_partition(
+            client_id,
+            config.get("val_file", f"{client_id}_val.pt"),
+        )
+        
+        trainloader = DataLoader(
+            train_data,
+            batch_size=config.get("batch_size", 32),
+            shuffle=True,
+        )
+        valloader = DataLoader(
+            val_data,
+            batch_size=config.get("batch_size", 32),
+            shuffle=False,
+        )
+        
+        model = self.model_fn()
+        
+        return FederatedClient(
+            model=model,
+            trainloader=trainloader,
+            valloader=valloader,
+            device=config.get("device", "cuda"),
+            client_id=client_id,
+            output_dir=str(self.output_dir),
+        )
+    
+    def _load_partition(self, client_id: str, filename: str):
+        """Load data partition for a client."""
+        path = self.data_dir / client_id / filename
+        if path.exists():
+            return torch.load(path)
+        raise FileNotFoundError(f"Data partition not found: {path}")
+
+
+def start_client(
+    model_fn,
+    data_dir: str,
+    client_id: str,
+    server_address: str = "localhost:8080",
+    config_path: Optional[str] = None,
+) -> None:
+    """Start a federated learning client."""
+    config = {}
+    if config_path:
+        config = load_client_config(config_path)
+    
+    factory = ClientFactory(model_fn, data_dir)
+    client = factory.create_client(client_id, config)
+    
+    app = fl.client.start_numpy_client(
+        server_address=server_address,
+        client=client,
+    )
+
+
+if __name__ == "__main__":
+    print("FederatedClient module loaded")
+    print("Use start_client() to start a federated learning client")

federated/server.py

@@ -0,0 +1,205 @@
+"""Federated Learning Server for Myanmar Ghost project.
+
+Coordinates model training across distributed clients
+using Flower framework.
+"""
+
+import flwr as fl
+import numpy as np
+import torch
+import torch.nn as nn
+import yaml
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Tuple
+
+logger = __import__("loguru").logger
+
+
+class FederatedServer:
+    """Flower server for federated learning."""
+    
+    def __init__(
+        self,
+        model: nn.Module,
+        strategy: Optional[fl.server.strategy.Strategy] = None,
+        output_dir: str = "outputs/federated",
+    ):
+        self.model = model
+        self.strategy = strategy or self._default_strategy()
+        self.output_dir = Path(output_dir)
+        self.output_dir.mkdir(parents=True, exist_ok=True)
+        
+        self.server = None
+        self.history = None
+    
+    def _default_strategy(self) -> fl.server.strategy.Strategy:
+        """Create default federated averaging strategy."""
+        return fl.server.strategy.FedAvg(
+            fraction_fit=0.5,
+            fraction_evaluate=0.5,
+            min_fit_clients=2,
+            min_evaluate_clients=2,
+            min_available_clients=2,
+        )
+    
+    def get_model_parameters(self) -> List[np.ndarray]:
+        """Get current model parameters."""
+        return [val.cpu().numpy() for _, val in self.model.state_dict().items()]
+    
+    def set_model_parameters(self, parameters: List[np.ndarray]) -> None:
+        """Set model parameters from numpy arrays."""
+        state_dict = dict(self.model.state_dict())
+        for i, (key, _) in enumerate(state_dict.items()):
+            state_dict[key] = torch.from_numpy(parameters[i])
+        self.model.load_state_dict(state_dict)
+    
+    def aggregate_results(
+        self,
+        results: List[Tuple[List[np.ndarray], int, Dict]],
+    ) -> Tuple[List[np.ndarray], Dict]:
+        """Aggregate client results using weighted averaging.
+        
+        Args:
+            results: List of (parameters, num_samples, metrics)
+        
+        Returns:
+            Aggregated parameters, aggregated metrics
+        """
+        total_samples = sum(r[1] for r in results)
+        
+        # Weighted average of parameters
+        weighted_params = None
+        
+        for params, n_samples, _ in results:
+            weight = n_samples / total_samples
+            if weighted_params is None:
+                weighted_params = [p * weight for p in params]
+            else:
+                weighted_params = [
+                    wp + p * weight
+                    for wp, p in zip(weighted_params, params)
+                ]
+        
+        # Aggregate metrics
+        aggregated_metrics = {}
+        for _, _, metrics in results:
+            for key, value in metrics.items():
+                if key not in aggregated_metrics:
+                    aggregated_metrics[key] = []
+                aggregated_metrics[key].append(value)
+        
+        # Average metrics
+        avg_metrics = {
+            key: np.mean(values)
+            for key, values in aggregated_metrics.items()
+        }
+        
+        logger.info(
+            f"Aggregated {len(results)} client results "
+            f"(total samples: {total_samples})"
+        )
+        
+        return weighted_params, avg_metrics
+    
+    def save_global_model(self, path: Optional[str] = None) -> str:
+        """Save the global model."""
+        if path is None:
+            from datetime import datetime
+            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+            path = self.output_dir / f"global_model_{timestamp}.pt"
+        
+        torch.save({
+            "model_state_dict": self.model.state_dict(),
+        }, path)
+        
+        logger.info(f"Global model saved to {path}")
+        return str(path)
+    
+    def load_global_model(self, path: str) -> None:
+        """Load a global model checkpoint."""
+        checkpoint = torch.load(path)
+        self.model.load_state_dict(checkpoint["model_state_dict"])
+        logger.info(f"Global model loaded from {path}")
+    
+    def start(
+        self,
+        server_address: str = "[::]:8080",
+        num_rounds: int = 5,
+    ) -> fl.server.Server:
+        """Start the federated learning server.
+        
+        Args:
+            server_address: Address to bind the server
+            num_rounds: Number of federated rounds
+        
+        Returns:
+            Server instance
+        """
+        from flwr.server import ServerConfig
+        
+        config = ServerConfig(num_rounds=num_rounds)
+        
+        def on_fit_config_fn(rnd: int) -> Dict[str, Any]:
+            """Generate config for each round."""
+            return {
+                "local_epochs": 1,
+                "batch_size": 32,
+                "learning_rate": 1e-4,
+                "round": rnd,
+            }
+        
+        def on_evaluate_config_fn(rnd: int) -> Dict[str, Any]:
+            """Generate config for evaluation."""
+            return {
+                "batch_size": 32,
+                "round": rnd,
+            }
+        
+        # Create strategy with callbacks
+        strategy = fl.server.strategy.FedAvg(
+            fraction_fit=0.5,
+            fraction_evaluate=0.5,
+            min_fit_clients=2,
+            min_evaluate_clients=2,
+            min_available_clients=2,
+            on_fit_config_fn=on_fit_config_fn,
+            on_evaluate_config_fn=on_evaluate_config_fn,
+        )
+        
+        self.server = fl.server.start_server(
+            server_address=server_address,
+            server=config,
+            strategy=strategy,
+            model=self.model,
+        )
+        
+        return self.server
+    
+    def get_history(self) -> Optional[List[Dict]]:
+        """Get training history from server."""
+        if self.server and hasattr(self.server, "history"):
+            return self.server.history
+        return None
+
+
+def load_server_config(config_path: str) -> Dict:
+    """Load server configuration from YAML."""
+    with open(config_path, "r", encoding="utf-8") as f:
+        return yaml.safe_load(f)
+
+
+def create_server(
+    model: nn.Module,
+    strategy: Optional[str] = "fedavg",
+    output_dir: str = "outputs/federated",
+) -> FederatedServer:
+    """Factory function to create federated server."""
+    return FederatedServer(
+        model=model,
+        output_dir=output_dir,
+    )
+
+
+if __name__ == "__main__":
+    print("FederatedServer module loaded")
+    print("Use create_server() to create a server and start() to begin")

models/base_model.py

@@ -0,0 +1,106 @@
+"""Base model class for Myanmar Ghost project."""
+
+import logging
+from abc import ABC, abstractmethod
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Tuple
+
+import torch
+import torch.nn as nn
+
+logger = logging.getLogger(__name__)
+
+
+class BaseModel(ABC, nn.Module):
+    """Abstract base class for all models."""
+    
+    def __init__(self, config: Optional[Dict] = None):
+        super().__init__()
+        self.config = config or {}
+        self.device = torch.device(
+            "cuda" if torch.cuda.is_available() else "cpu"
+        )
+    
+    @abstractmethod
+    def forward(self, *args, **kwargs) -> torch.Tensor:
+        """Forward pass."""
+        pass
+    
+    @abstractmethod
+    def predict(self, *args, **kwargs) -> Dict[str, Any]:
+        """Make predictions."""
+        pass
+    
+    def save(self, path: str) -> None:
+        """Save model checkpoint."""
+        Path(path).parent.mkdir(parents=True, exist_ok=True)
+        torch.save({
+            "model_state_dict": self.state_dict(),
+            "config": self.config,
+        }, path)
+        logger.info(f"Model saved to {path}")
+    
+    def load(self, path: str) -> None:
+        """Load model checkpoint."""
+        checkpoint = torch.load(path, map_location=self.device)
+        self.load_state_dict(checkpoint["model_state_dict"])
+        if "config" in checkpoint:
+            self.config = checkpoint["config"]
+        logger.info(f"Model loaded from {path}")
+    
+    def get_num_parameters(self) -> int:
+        """Get total number of parameters."""
+        return sum(p.numel() for p in self.parameters())
+    
+    def get_num_trainable_parameters(self) -> int:
+        """Get number of trainable parameters."""
+        return sum(p.numel() for p in self.parameters() if p.requires_grad)
+
+
+class SentimentClassifier(nn.Module):
+    """Base sentiment classifier."""
+    
+    def __init__(
+        self,
+        input_dim: int,
+        hidden_dim: int,
+        num_classes: int = 4,
+        dropout: float = 0.1,
+    ):
+        super().__init__()
+        self.fc1 = nn.Linear(input_dim, hidden_dim)
+        self.dropout = nn.Dropout(dropout)
+        self.fc2 = nn.Linear(hidden_dim, hidden_dim // 2)
+        self.fc3 = nn.Linear(hidden_dim // 2, num_classes)
+        self.relu = nn.ReLU()
+    
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        x = self.relu(self.fc1(x))
+        x = self.dropout(x)
+        x = self.relu(self.fc2(x))
+        x = self.dropout(x)
+        x = self.fc3(x)
+        return x
+
+
+def create_model(
+    model_type: str = "transformer",
+    **kwargs,
+) -> BaseModel:
+    """Factory function to create models."""
+    from .transformer_model import TransformerSentimentModel
+    from .multimodal_model import MultiModalSentimentModel
+    
+    if model_type == "transformer":
+        return TransformerSentimentModel(**kwargs)
+    elif model_type == "multimodal":
+        return MultiModalSentimentModel(**kwargs)
+    elif model_type == "base":
+        return SentimentClassifier(**kwargs)
+    else:
+        raise ValueError(f"Unknown model type: {model_type}")
+
+
+if __name__ == "__main__":
+    model = SentimentClassifier(input_dim=768, hidden_dim=256, num_classes=4)
+    print(f"Model parameters: {model.get_num_parameters():,}")

models/evaluate.py

@@ -0,0 +1,252 @@
+"""Evaluation script for Myanmar Ghost sentiment model."""
+
+import argparse
+import json
+import logging
+import sys
+from pathlib import Path
+from typing import Any, Dict
+
+import numpy as np
+import torch
+import torch.nn as nn
+from torch.utils.data import DataLoader, Dataset
+from tqdm import tqdm
+
+sys.path.insert(0, str(Path(__file__).parent.parent.parent))
+
+from src.utils.logger import setup_logger
+from src.utils.metrics import compute_metrics, compute_confusion_matrix
+
+logger = setup_logger("evaluate", log_dir="outputs/logs")
+
+
+class SentimentDataset(Dataset):
+    """Dataset for sentiment classification."""
+    
+    def __init__(
+        self,
+        data,
+        tokenizer,
+        max_length: int = 512,
+        label_mapping: dict = None,
+    ):
+        self.data = data
+        self.tokenizer = tokenizer
+        self.max_length = max_length
+        self.label_mapping = label_mapping or {
+            "negative": 0, "neutral": 1, "positive": 2, "sarcastic": 3
+        }
+    
+    def __len__(self) -> int:
+        return len(self.data)
+    
+    def __getitem__(self, idx: int):
+        item = self.data[idx]
+        
+        encoding = self.tokenizer(
+            item["text"],
+            truncation=True,
+            max_length=self.max_length,
+            padding="max_length",
+            return_tensors="pt",
+        )
+        
+        label = self.label_mapping.get(item.get("label", "neutral"), 1)
+        
+        return (
+            encoding["input_ids"].squeeze(0),
+            encoding["attention_mask"].squeeze(0),
+            torch.tensor(label, dtype=torch.long),
+            item.get("text", ""),
+        )
+
+
+def load_data(data_path: str):
+    """Load evaluation data."""
+    if data_path.endswith(".jsonl"):
+        data = []
+        with open(data_path, "r", encoding="utf-8") as f:
+            for line in f:
+                if line.strip():
+                    data.append(json.loads(line))
+    elif data_path.endswith(".json"):
+        with open(data_path, "r", encoding="utf-8") as f:
+            data = json.load(f)
+    else:
+        raise ValueError(f"Unsupported format: {data_path}")
+    
+    return data
+
+
+def evaluate(
+    model: nn.Module,
+    dataloader: DataLoader,
+    device: torch.device,
+    class_names: list = None,
+) -> Dict[str, Any]:
+    """Evaluate the model."""
+    if class_names is None:
+        class_names = ["negative", "neutral", "positive", "sarcastic"]
+    
+    model.eval()
+    
+    all_predictions = []
+    all_labels = []
+    all_texts = []
+    all_probabilities = []
+    
+    with torch.no_grad():
+        for input_ids, attention_mask, labels, texts in tqdm(dataloader, desc="Evaluating"):
+            input_ids = input_ids.to(device)
+            attention_mask = attention_mask.to(device)
+            
+            outputs = model(input_ids, attention_mask)
+            probs = torch.softmax(outputs, dim=-1)
+            
+            predictions = outputs.argmax(dim=-1).cpu().tolist()
+            all_predictions.extend(predictions)
+            all_labels.extend(labels.tolist())
+            all_texts.extend(texts)
+            all_probabilities.extend(probs.cpu().numpy().tolist())
+    
+    # Compute metrics
+    metrics = compute_metrics(all_predictions, all_labels, class_names)
+    
+    # Confusion matrix
+    cm = compute_confusion_matrix(all_predictions, all_labels)
+    
+    # Per-sample results
+    results = []
+    for i, (text, label, pred, probs) in enumerate(zip(
+        all_texts, all_labels, all_predictions, all_probabilities
+    )):
+        results.append({
+            "text": text,
+            "true_label": class_names[label],
+            "predicted_label": class_names[pred],
+            "correct": label == pred,
+            "probabilities": {
+                class_names[j]: probs[j] for j in range(len(class_names))
+            },
+        })
+    
+    return {
+        "metrics": metrics,
+        "confusion_matrix": cm.tolist(),
+        "class_names": class_names,
+        "results": results,
+    }
+
+
+def save_results(results: Dict, output_path: str) -> None:
+    """Save evaluation results to file."""
+    output_dir = Path(output_path).parent
+    output_dir.mkdir(parents=True, exist_ok=True)
+    
+    # Save full results
+    with open(output_path, "w", encoding="utf-8") as f:
+        json.dump(results, f, indent=2, ensure_ascii=False)
+    
+    # Save summary
+    summary_path = output_dir / "evaluation_summary.txt"
+    with open(summary_path, "w", encoding="utf-8") as f:
+        f.write("=" * 60 + "\n")
+        f.write("EVALUATION SUMMARY\n")
+        f.write("=" * 60 + "\n\n")
+        
+        metrics = results["metrics"]
+        f.write(f"Accuracy:      {metrics['accuracy']:.4f}\n")
+        f.write(f"F1 (weighted): {metrics['f1_weighted']:.4f}\n")
+        f.write(f"F1 (macro):    {metrics['f1_macro']:.4f}\n")
+        f.write(f"Precision:     {metrics['precision_weighted']:.4f}\n")
+        f.write(f"Recall:        {metrics['recall_weighted']:.4f}\n\n")
+        
+        f.write("Per-class F1:\n")
+        for name in results["class_names"]:
+            f1_key = f"f1_{name}"
+            if f1_key in metrics:
+                f.write(f"  {name}: {metrics[f1_key]:.4f}\n")
+        
+        f.write(f"\nConfusion Matrix:\n")
+        f.write(str(np.array(results["confusion_matrix"])) + "\n")
+    
+    logger.info(f"Results saved to {output_path}")
+    logger.info(f"Summary saved to {summary_path}")
+
+
+def main(args):
+    """Main evaluation function."""
+    logger.info("Starting evaluation...")
+    logger.info(f"Arguments: {vars(args)}")
+    
+    device = torch.device("cuda" if torch.cuda.is_available() and not args.cpu else "cpu")
+    logger.info(f"Using device: {device}")
+    
+    # Load tokenizer
+    from transformers import AutoTokenizer
+    tokenizer = AutoTokenizer.from_pretrained(args.model_name)
+    
+    # Load data
+    logger.info(f"Loading data from {args.data_path}")
+    data = load_data(args.data_path)
+    logger.info(f"Total samples: {len(data)}")
+    
+    # Create dataset and dataloader
+    dataset = SentimentDataset(data, tokenizer, args.max_length)
+    dataloader = DataLoader(
+        dataset,
+        batch_size=args.batch_size,
+        shuffle=False,
+        num_workers=2,
+    )
+    
+    # Load model
+    logger.info(f"Loading model from {args.model_path}")
+    from src.models.transformer_model import TransformerSentimentModel
+    
+    model = TransformerSentimentModel(
+        model_name=args.model_name,
+        num_labels=4,
+    )
+    
+    checkpoint = torch.load(args.model_path, map_location=device)
+    if "model_state_dict" in checkpoint:
+        model.load_state_dict(checkpoint["model_state_dict"])
+    else:
+        model.load_state_dict(checkpoint)
+    
+    model.to(device)
+    
+    # Evaluate
+    results = evaluate(model, dataloader, device)
+    
+    # Print summary
+    logger.info("\n" + "=" * 60)
+    logger.info("EVALUATION RESULTS")
+    logger.info("=" * 60)
+    logger.info(f"Accuracy:      {results['metrics']['accuracy']:.4f}")
+    logger.info(f"F1 (weighted): {results['metrics']['f1_weighted']:.4f}")
+    logger.info(f"Precision:     {results['metrics']['precision_weighted']:.4f}")
+    logger.info(f"Recall:        {results['metrics']['recall_weighted']:.4f}")
+    
+    # Save results
+    output_path = args.output_path or "outputs/results/evaluation_results.json"
+    save_results(results, output_path)
+    
+    return results["metrics"]
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Evaluate Myanmar Ghost model")
+    
+    parser.add_argument("--data_path", type=str, required=True, help="Test data file")
+    parser.add_argument("--model_path", type=str, required=True, help="Model checkpoint")
+    parser.add_argument("--model_name", type=str, default="bert-base-multilingual-cased")
+    parser.add_argument("--output_path", type=str, default=None, help="Output path")
+    parser.add_argument("--batch_size", type=int, default=32)
+    parser.add_argument("--max_length", type=int, default=512)
+    parser.add_argument("--cpu", action="store_true", help="Use CPU only")
+    
+    args = parser.parse_args()
+    main(args)

models/multimodal_model.py

@@ -0,0 +1,192 @@
+"""Multi-modal sentiment model combining audio and text."""
+
+import logging
+from typing import Any, Dict, Optional
+
+import torch
+import torch.nn as nn
+
+from .base_model import BaseModel
+
+logger = logging.getLogger(__name__)
+
+
+class MultiModalSentimentModel(BaseModel):
+    """Multi-modal model combining text and audio features."""
+    
+    def __init__(
+        self,
+        text_dim: int = 768,
+        audio_dim: int = 8,
+        hidden_dim: int = 256,
+        num_classes: int = 4,
+        dropout: float = 0.2,
+    ):
+        """
+        Args:
+            text_dim: Text embedding dimension
+            audio_dim: Audio feature dimension
+            hidden_dim: Hidden layer dimension
+            num_classes: Number of sentiment classes
+            dropout: Dropout rate
+        """
+        super().__init__()
+        
+        self.text_dim = text_dim
+        self.audio_dim = audio_dim
+        
+        # Text encoder
+        self.text_encoder = nn.Sequential(
+            nn.Linear(text_dim, hidden_dim),
+            nn.ReLU(),
+            nn.Dropout(dropout),
+        )
+        
+        # Audio encoder
+        self.audio_encoder = nn.Sequential(
+            nn.Linear(audio_dim, hidden_dim // 2),
+            nn.ReLU(),
+            nn.Dropout(dropout),
+        )
+        
+        # Fusion layer
+        self.fusion = nn.Sequential(
+            nn.Linear(hidden_dim + hidden_dim // 2, hidden_dim),
+            nn.ReLU(),
+            nn.Dropout(dropout),
+        )
+        
+        # Classification head
+        self.classifier = nn.Sequential(
+            nn.Linear(hidden_dim, hidden_dim // 2),
+            nn.ReLU(),
+            nn.Dropout(dropout),
+            nn.Linear(hidden_dim // 2, num_classes),
+        )
+    
+    def forward(
+        self,
+        text_features: torch.Tensor,
+        audio_features: torch.Tensor,
+    ) -> torch.Tensor:
+        """
+        Forward pass.
+        
+        Args:
+            text_features: Text embeddings (batch, text_dim)
+            audio_features: Audio features (batch, audio_dim)
+        
+        Returns:
+            Logits (batch, num_classes)
+        """
+        # Encode each modality
+        text_encoded = self.text_encoder(text_features)
+        audio_encoded = self.audio_encoder(audio_features)
+        
+        # Concatenate and fuse
+        fused = torch.cat([text_encoded, audio_encoded], dim=-1)
+        fused = self.fusion(fused)
+        
+        # Classify
+        logits = self.classifier(fused)
+        
+        return logits
+    
+    def predict(
+        self,
+        text_features: torch.Tensor,
+        audio_features: Optional[torch.Tensor] = None,
+    ) -> Dict[str, Any]:
+        """Make predictions."""
+        self.eval()
+        
+        if audio_features is None:
+            # Text-only mode
+            audio_features = torch.zeros(
+                text_features.size(0), self.audio_dim
+            ).to(text_features.device)
+        
+        with torch.no_grad():
+            logits = self.forward(text_features, audio_features)
+            probs = torch.softmax(logits, dim=-1)
+        
+        sentiment_labels = ["negative", "neutral", "positive", "sarcastic"]
+        
+        predictions = []
+        for i, probs_i in enumerate(probs):
+            pred_idx = probs_i.argmax().item()
+            predictions.append({
+                "sentiment": sentiment_labels[pred_idx],
+                "confidence": probs_i[pred_idx].item(),
+                "probabilities": {
+                    label: probs_i[j].item()
+                    for j, label in enumerate(sentiment_labels)
+                },
+            })
+        
+        return {"predictions": predictions}
+
+
+class CrossModalAttention(nn.Module):
+    """Cross-modal attention for audio-text fusion."""
+    
+    def __init__(
+        self,
+        query_dim: int,
+        key_dim: int,
+        hidden_dim: int,
+        num_heads: int = 4,
+    ):
+        super().__init__()
+        
+        self.num_heads = num_heads
+        self.head_dim = hidden_dim // num_heads
+        
+        self.query = nn.Linear(query_dim, hidden_dim)
+        self.key = nn.Linear(key_dim, hidden_dim)
+        self.value = nn.Linear(key_dim, hidden_dim)
+        
+        self.output = nn.Linear(hidden_dim, hidden_dim)
+        self.scale = self.head_dim ** -0.5
+    
+    def forward(
+        self,
+        query: torch.Tensor,
+        key_value: torch.Tensor,
+    ) -> torch.Tensor:
+        """Cross-attention forward pass."""
+        batch_size = query.size(0)
+        
+        # Linear projections
+        Q = self.query(query).view(batch_size, -1, self.num_heads, self.head_dim).transpose(1, 2)
+        K = self.key(key_value).view(batch_size, -1, self.num_heads, self.head_dim).transpose(1, 2)
+        V = self.value(key_value).view(batch_size, -1, self.num_heads, self.head_dim).transpose(1, 2)
+        
+        # Attention scores
+        scores = torch.matmul(Q, K.transpose(-2, -1)) * self.scale
+        attention = torch.softmax(scores, dim=-1)
+        
+        # Apply attention to values
+        context = torch.matmul(attention, V)
+        context = context.transpose(1, 2).contiguous().view(batch_size, -1, self.num_heads * self.head_dim)
+        
+        return self.output(context)
+
+
+if __name__ == "__main__":
+    print("Testing MultiModalSentimentModel...")
+    
+    model = MultiModalSentimentModel(
+        text_dim=768,
+        audio_dim=8,
+        hidden_dim=256,
+        num_classes=4,
+    )
+    
+    # Mock inputs
+    text_features = torch.randn(2, 768)
+    audio_features = torch.randn(2, 8)
+    
+    logits = model(text_features, audio_features)
+    print(f"Output shape: {logits.shape}")
+    print(f"Total parameters: {model.get_num_parameters():,}")

models/train.py

@@ -0,0 +1,318 @@
+"""Training script for Myanmar Ghost sentiment model."""
+
+import argparse
+import logging
+import sys
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+import torch
+import torch.nn as nn
+import torch.optim as optim
+from torch.utils.data import DataLoader, Dataset
+from tqdm import tqdm
+
+# Add parent directory to path
+sys.path.insert(0, str(Path(__file__).parent.parent.parent))
+
+from src.utils.logger import setup_logger
+from src.utils.metrics import compute_metrics, MetricsTracker
+
+logger = setup_logger("train", log_dir="outputs/logs")
+
+
+class SentimentDataset(Dataset):
+    """Dataset for sentiment classification."""
+    
+    def __init__(
+        self,
+        data: List[Dict],
+        tokenizer,
+        max_length: int = 512,
+        label_mapping: Dict[str, int] = None,
+    ):
+        self.data = data
+        self.tokenizer = tokenizer
+        self.max_length = max_length
+        self.label_mapping = label_mapping or {
+            "negative": 0,
+            "neutral": 1,
+            "positive": 2,
+            "sarcastic": 3,
+        }
+    
+    def __len__(self) -> int:
+        return len(self.data)
+    
+    def __getitem__(self, idx: int) -> tuple:
+        item = self.data[idx]
+        
+        encoding = self.tokenizer(
+            item["text"],
+            truncation=True,
+            max_length=self.max_length,
+            padding="max_length",
+            return_tensors="pt",
+        )
+        
+        label = self.label_mapping.get(item.get("label", "neutral"), 1)
+        
+        return (
+            encoding["input_ids"].squeeze(0),
+            encoding["attention_mask"].squeeze(0),
+            torch.tensor(label, dtype=torch.long),
+        )
+
+
+def train_epoch(
+    model: nn.Module,
+    dataloader: DataLoader,
+    criterion: nn.Module,
+    optimizer: optim.Optimizer,
+    device: torch.device,
+    scheduler: Optional[Any] = None,
+) -> Dict[str, float]:
+    """Train for one epoch."""
+    model.train()
+    
+    total_loss = 0.0
+    all_predictions = []
+    all_labels = []
+    
+    progress_bar = tqdm(dataloader, desc="Training")
+    
+    for batch_idx, (input_ids, attention_mask, labels) in enumerate(progress_bar):
+        input_ids = input_ids.to(device)
+        attention_mask = attention_mask.to(device)
+        labels = labels.to(device)
+        
+        optimizer.zero_grad()
+        
+        outputs = model(input_ids, attention_mask)
+        loss = criterion(outputs, labels)
+        
+        loss.backward()
+        torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)
+        optimizer.step()
+        
+        if scheduler:
+            scheduler.step()
+        
+        total_loss += loss.item()
+        
+        predictions = outputs.argmax(dim=-1).cpu().tolist()
+        all_predictions.extend(predictions)
+        all_labels.extend(labels.cpu().tolist())
+        
+        progress_bar.set_postfix({"loss": loss.item()})
+    
+    metrics = compute_metrics(all_predictions, all_labels)
+    metrics["loss"] = total_loss / len(dataloader)
+    
+    return metrics
+
+
+def evaluate(
+    model: nn.Module,
+    dataloader: DataLoader,
+    criterion: nn.Module,
+    device: torch.device,
+) -> Dict[str, float]:
+    """Evaluate the model."""
+    model.eval()
+    
+    total_loss = 0.0
+    all_predictions = []
+    all_labels = []
+    
+    with torch.no_grad():
+        for input_ids, attention_mask, labels in tqdm(dataloader, desc="Evaluating"):
+            input_ids = input_ids.to(device)
+            attention_mask = attention_mask.to(device)
+            labels = labels.to(device)
+            
+            outputs = model(input_ids, attention_mask)
+            loss = criterion(outputs, labels)
+            
+            total_loss += loss.item()
+            
+            predictions = outputs.argmax(dim=-1).cpu().tolist()
+            all_predictions.extend(predictions)
+            all_labels.extend(labels.cpu().tolist())
+    
+    metrics = compute_metrics(all_predictions, all_labels)
+    metrics["loss"] = total_loss / len(dataloader)
+    
+    return metrics
+
+
+def load_data(data_path: str) -> List[Dict]:
+    """Load training data from JSON or JSONL file."""
+    import json
+    
+    data = []
+    
+    if data_path.endswith(".jsonl"):
+        with open(data_path, "r", encoding="utf-8") as f:
+            for line in f:
+                if line.strip():
+                    data.append(json.loads(line))
+    elif data_path.endswith(".json"):
+        with open(data_path, "r", encoding="utf-8") as f:
+            data = json.load(f)
+    else:
+        raise ValueError(f"Unsupported file format: {data_path}")
+    
+    return data
+
+
+def main(args):
+    """Main training function."""
+    logger.info("Starting training...")
+    logger.info(f"Arguments: {vars(args)}")
+    
+    # Device
+    device = torch.device(
+        "cuda" if torch.cuda.is_available() and not args.cpu else "cpu"
+    )
+    logger.info(f"Using device: {device}")
+    
+    # Load tokenizer
+    logger.info(f"Loading tokenizer from {args.model_name}")
+    from transformers import AutoTokenizer
+    tokenizer = AutoTokenizer.from_pretrained(args.model_name)
+    
+    # Load data
+    logger.info(f"Loading data from {args.train_data}")
+    train_data = load_data(args.train_data)
+    val_data = load_data(args.val_data) if args.val_data else []
+    
+    logger.info(f"Train samples: {len(train_data)}, Val samples: {len(val_data)}")
+    
+    # Create datasets
+    train_dataset = SentimentDataset(train_data, tokenizer, args.max_length)
+    train_loader = DataLoader(
+        train_dataset,
+        batch_size=args.batch_size,
+        shuffle=True,
+        num_workers=2,
+    )
+    
+    val_loader = None
+    if val_data:
+        val_dataset = SentimentDataset(val_data, tokenizer, args.max_length)
+        val_loader = DataLoader(
+            val_dataset,
+            batch_size=args.batch_size,
+            shuffle=False,
+            num_workers=2,
+        )
+    
+    # Create model
+    logger.info("Creating model...")
+    from src.models.transformer_model import TransformerSentimentModel
+    
+    model = TransformerSentimentModel(
+        model_name=args.model_name,
+        num_labels=4,
+        dropout=args.dropout,
+        freeze_encoder=args.freeze_encoder,
+    )
+    model.to(device)
+    
+    logger.info(f"Model parameters: {model.get_num_parameters():,}")
+    logger.info(f"Trainable: {model.get_num_trainable_parameters():,}")
+    
+    # Loss and optimizer
+    criterion = nn.CrossEntropyLoss()
+    optimizer = optim.AdamW(
+        model.parameters(),
+        lr=args.learning_rate,
+        weight_decay=args.weight_decay,
+    )
+    
+    # Scheduler
+    total_steps = len(train_loader) * args.num_epochs
+    warmup_steps = int(total_steps * 0.1)
+    
+    scheduler = optim.lr_scheduler.LinearLR(
+        optimizer,
+        start_factor=0.1,
+        total_iters=warmup_steps,
+    )
+    
+    # Training loop
+    metrics_tracker = MetricsTracker(
+        metrics=["loss", "accuracy", "f1_weighted"],
+    )
+    
+    best_f1 = 0.0
+    best_model_path = Path(args.output_dir) / "best_model.pt"
+    
+    for epoch in range(args.num_epochs):
+        logger.info(f"\nEpoch {epoch + 1}/{args.num_epochs}")
+        
+        # Train
+        train_metrics = train_epoch(
+            model, train_loader, criterion, optimizer, device, scheduler
+        )
+        
+        logger.info(f"Train - Loss: {train_metrics['loss']:.4f}, "
+                   f"Acc: {train_metrics['accuracy']:.4f}, "
+                   f"F1: {train_metrics['f1_weighted']:.4f}")
+        
+        # Evaluate
+        if val_loader:
+            val_metrics = evaluate(model, val_loader, criterion, device)
+            
+            logger.info(f"Val   - Loss: {val_metrics['loss']:.4f}, "
+                        f"Acc: {val_metrics['accuracy']:.4f}, "
+                        f"F1: {val_metrics['f1_weighted']:.4f}")
+            
+            metrics = {"train_" + k: v for k, v in train_metrics.items()}
+            metrics.update({"val_" + k: v for k, v in val_metrics.items()})
+        else:
+            metrics = {"train_" + k: v for k, v in train_metrics.items()}
+        
+        metrics_tracker.update(metrics, epoch)
+        
+        # Save best model
+        current_f1 = train_metrics.get("f1_weighted", 0)
+        if current_f1 > best_f1:
+            best_f1 = current_f1
+            model.save(str(best_model_path))
+            logger.info(f"Saved best model (F1: {best_f1:.4f})")
+    
+    # Save final model
+    final_path = Path(args.output_dir) / "final_model.pt"
+    model.save(str(final_path))
+    
+    logger.info(f"\nTraining complete! Best F1: {best_f1:.4f}")
+    
+    return best_f1
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Train Myanmar Ghost model")
+    
+    # Data arguments
+    parser.add_argument("--train_data", type=str, required=True, help="Training data file")
+    parser.add_argument("--val_data", type=str, default=None, help="Validation data file")
+    parser.add_argument("--output_dir", type=str, default="outputs/models", help="Output directory")
+    
+    # Model arguments
+    parser.add_argument("--model_name", type=str, default="bert-base-multilingual-cased")
+    parser.add_argument("--max_length", type=int, default=512)
+    parser.add_argument("--dropout", type=float, default=0.1)
+    parser.add_argument("--freeze_encoder", action="store_true")
+    
+    # Training arguments
+    parser.add_argument("--batch_size", type=int, default=16)
+    parser.add_argument("--num_epochs", type=int, default=10)
+    parser.add_argument("--learning_rate", type=float, default=5e-5)
+    parser.add_argument("--weight_decay", type=float, default=0.01)
+    parser.add_argument("--cpu", action="store_true", help="Use CPU only")
+    
+    args = parser.parse_args()
+    
+    main(args)

models/transformer_model.py

@@ -0,0 +1,175 @@
+"""Transformer-based sentiment model for Myanmar text."""
+
+import logging
+from typing import Any, Dict, Optional
+
+import torch
+import torch.nn as nn
+from transformers import (
+    AutoConfig,
+    AutoModel,
+    AutoModelForSequenceClassification,
+    AutoTokenizer,
+)
+
+from .base_model import BaseModel
+
+logger = logging.getLogger(__name__)
+
+
+class TransformerSentimentModel(BaseModel):
+    """Transformer-based sentiment classification model."""
+    
+    def __init__(
+        self,
+        model_name: str = "bert-base-multilingual-cased",
+        num_labels: int = 4,
+        dropout: float = 0.1,
+        freeze_encoder: bool = False,
+    ):
+        """
+        Args:
+            model_name: Pretrained model name
+            num_labels: Number of sentiment labels
+            dropout: Dropout rate
+            freeze_encoder: Whether to freeze encoder weights
+        """
+        super().__init__()
+        
+        self.model_name = model_name
+        self.num_labels = num_labels
+        
+        # Load pretrained config
+        self.config = AutoConfig.from_pretrained(model_name)
+        
+        # Load pretrained model
+        self.transformer = AutoModel.from_pretrained(model_name)
+        
+        # Classification head
+        self.dropout = nn.Dropout(dropout)
+        self.classifier = nn.Linear(self.config.hidden_size, num_labels)
+        
+        # Freeze encoder if requested
+        if freeze_encoder:
+            for param in self.transformer.parameters():
+                param.requires_grad = False
+        
+        self.to(self.device)
+    
+    def forward(
+        self,
+        input_ids: torch.Tensor,
+        attention_mask: Optional[torch.Tensor] = None,
+        token_type_ids: Optional[torch.Tensor] = None,
+    ) -> torch.Tensor:
+        """Forward pass."""
+        outputs = self.transformer(
+            input_ids=input_ids,
+            attention_mask=attention_mask,
+            token_type_ids=token_type_ids,
+        )
+        
+        # Use [CLS] token representation
+        pooled_output = outputs.last_hidden_state[:, 0, :]
+        pooled_output = self.dropout(pooled_output)
+        logits = self.classifier(pooled_output)
+        
+        return logits
+    
+    def predict(
+        self,
+        texts: list,
+        tokenizer,
+        batch_size: int = 16,
+    ) -> Dict[str, Any]:
+        """Make predictions on texts."""
+        self.eval()
+        
+        all_probs = []
+        
+        with torch.no_grad():
+            for i in range(0, len(texts), batch_size):
+                batch_texts = texts[i:i + batch_size]
+                
+                encoding = tokenizer(
+                    batch_texts,
+                    padding=True,
+                    truncation=True,
+                    max_length=512,
+                    return_tensors="pt",
+                )
+                
+                input_ids = encoding["input_ids"].to(self.device)
+                attention_mask = encoding["attention_mask"].to(self.device)
+                
+                logits = self.forward(input_ids, attention_mask)
+                probs = torch.softmax(logits, dim=-1)
+                
+                all_probs.append(probs.cpu().numpy())
+        
+        import numpy as np
+        all_probs = np.vstack(all_probs)
+        
+        sentiment_labels = ["negative", "neutral", "positive", "sarcastic"]
+        
+        predictions = []
+        for i, probs in enumerate(all_probs):
+            pred_idx = probs.argmax()
+            predictions.append({
+                "text": texts[i],
+                "sentiment": sentiment_labels[pred_idx],
+                "confidence": probs[pred_idx],
+                "probabilities": {
+                    label: probs[j] for j, label in enumerate(sentiment_labels)
+                },
+            })
+        
+        return {"predictions": predictions}
+    
+    def extract_features(
+        self,
+        input_ids: torch.Tensor,
+        attention_mask: Optional[torch.Tensor] = None,
+    ) -> torch.Tensor:
+        """Extract hidden features."""
+        outputs = self.transformer(
+            input_ids=input_ids,
+            attention_mask=attention_mask,
+        )
+        return outputs.last_hidden_state
+
+
+def load_pretrained_model(
+    model_path: str,
+    num_labels: int = 4,
+) -> TransformerSentimentModel:
+    """Load a pretrained model from path or HuggingFace."""
+    # Check if it's a HuggingFace model
+    if "/" in model_path:
+        return TransformerSentimentModel(
+            model_name=model_path,
+            num_labels=num_labels,
+        )
+    
+    # Load from local checkpoint
+    model = TransformerSentimentModel(num_labels=num_labels)
+    checkpoint = torch.load(model_path, map_location="cpu")
+    
+    if "model_state_dict" in checkpoint:
+        model.load_state_dict(checkpoint["model_state_dict"])
+    elif "model" in checkpoint:
+        model.transformer = checkpoint["model"]
+    
+    return model
+
+
+if __name__ == "__main__":
+    print("Testing TransformerSentimentModel...")
+    
+    model = TransformerSentimentModel(
+        model_name="bert-base-multilingual-cased",
+        num_labels=4,
+    )
+    
+    print(f"Total parameters: {model.get_num_parameters():,}")
+    print(f"Trainable parameters: {model.get_num_trainable_parameters():,}")

pipelines/data_pipeline.py

@@ -0,0 +1,193 @@
+"""End-to-end data processing pipeline."""
+
+import argparse
+import logging
+import sys
+from pathlib import Path
+from typing import List
+
+sys.path.insert(0, str(Path(__file__).parent.parent.parent))
+
+from src.utils.logger import setup_logger
+
+logger = setup_logger("data_pipeline")
+
+
+def process_audio(input_dir: str, output_dir: str, config: dict = None) -> List[str]:
+    """Process audio files."""
+    from src.data_processing.audio_processor import AudioProcessor
+    
+    logger.info(f"Processing audio from {input_dir}")
+    
+    processor = AudioProcessor(
+        sample_rate=config.get("sample_rate", 16000),
+        n_fft=config.get("n_fft", 512),
+        hop_length=config.get("hop_length", 160),
+    )
+    
+    results = processor.batch_process(input_dir, output_dir)
+    
+    logger.info(f"Processed {len(results)} audio files")
+    return [r["output_path"] for r in results]
+
+
+def process_text(input_path: str, output_path: str, config: dict = None) -> str:
+    """Process text data."""
+    from src.data_processing.text_normalizer import MyanmarTextNormalizer
+    
+    logger.info(f"Processing text from {input_path}")
+    
+    normalizer = MyanmarTextNormalizer(
+        custom_rules_path=config.get("custom_rules") if config else None
+    )
+    
+    import pandas as pd
+    
+    if input_path.endswith(".csv"):
+        df = pd.read_csv(input_path)
+        texts = df["text"].tolist()
+    else:
+        raise ValueError(f"Unsupported input format: {input_path}")
+    
+    normalized_texts = normalizer.normalize_corpus(texts)
+    df["text_normalized"] = normalized_texts
+    
+    df.to_csv(output_path, index=False)
+    logger.info(f"Saved normalized text to {output_path}")
+    
+    return output_path
+
+
+def augment_data(input_path: str, output_path: str, config: dict = None) -> str:
+    """Augment training data."""
+    from src.augmentation.synonym_replacer import MyanmarSynonymReplacer
+    from src.augmentation.perturbator import TextPerturbator
+    
+    logger.info(f"Augmenting data from {input_path}")
+    
+    replacer = MyanmarSynonymReplacer()
+    perturbator = TextPerturbator()
+    
+    import pandas as pd
+    import json
+    
+    if input_path.endswith(".csv"):
+        df = pd.read_csv(input_path)
+        samples = df.to_dict("records")
+    elif input_path.endswith(".json"):
+        with open(input_path, "r") as f:
+            samples = json.load(f)
+    else:
+        raise ValueError(f"Unsupported format: {input_path}")
+    
+    augmented = []
+    
+    for sample in samples:
+        text = sample.get("text", "")
+        
+        # Synonym replacement
+        aug_text, replacements = replacer.augment_text(
+            text,
+            replace_prob=config.get("synonym_prob", 0.3) if config else 0.3,
+        )
+        if replacements:
+            aug_sample = sample.copy()
+            aug_sample["text"] = aug_text
+            aug_sample["augmentation_type"] = "synonym"
+            aug_sample["replacements"] = replacements
+            augmented.append(aug_sample)
+        
+        # Perturbation
+        aug_text, perturbations = perturbator.apply_random_perturbations(
+            text,
+            n_perturbations=config.get("n_perturbations", 2) if config else 2,
+        )
+        if perturbations:
+            aug_sample = sample.copy()
+            aug_sample["text"] = aug_text
+            aug_sample["augmentation_type"] = "perturbation"
+            aug_sample["perturbations"] = [p.value for p in perturbations]
+            augmented.append(aug_sample)
+    
+    # Save augmented data
+    output_df = pd.DataFrame(augmented)
+    output_df.to_csv(output_path, index=False)
+    
+    logger.info(f"Generated {len(augmented)} augmented samples")
+    return output_path
+
+
+def split_data(input_path: str, output_dir: str, config: dict = None) -> dict:
+    """Split data into train/val/test."""
+    from sklearn.model_selection import train_test_split
+    
+    logger.info(f"Splitting data from {input_path}")
+    
+    import pandas as pd
+    
+    df = pd.read_csv(input_path)
+    
+    train_ratio = config.get("train_ratio", 0.8) if config else 0.8
+    val_ratio = config.get("val_ratio", 0.1) if config else 0.1
+    
+    # First split: train vs rest
+    train_df, temp_df = train_test_split(
+        df, train_size=train_ratio, random_state=42
+    )
+    
+    # Second split: val vs test
+    val_size = val_ratio / (1 - train_ratio)
+    val_df, test_df = train_test_split(
+        temp_df, train_size=val_size, random_state=42
+    )
+    
+    # Save splits
+    Path(output_dir).mkdir(parents=True, exist_ok=True)
+    
+    train_df.to_csv(f"{output_dir}/train.csv", index=False)
+    val_df.to_csv(f"{output_dir}/val.csv", index=False)
+    test_df.to_csv(f"{output_dir}/test.csv", index=False)
+    
+    logger.info(f"Saved: train={len(train_df)}, val={len(val_df)}, test={len(test_df)}")
+    
+    return {
+        "train": f"{output_dir}/train.csv",
+        "val": f"{output_dir}/val.csv",
+        "test": f"{output_dir}/test.csv",
+    }
+
+
+def run_pipeline(input_path: str, output_dir: str, config: dict = None) -> dict:
+    """Run the full data processing pipeline."""
+    logger.info("Starting data processing pipeline...")
+    
+    Path(output_dir).mkdir(parents=True, exist_ok=True)
+    
+    # Step 1: Normalize text
+    normalized_path = f"{output_dir}/normalized.csv"
+    process_text(input_path, normalized_path, config)
+    
+    # Step 2: Augment data
+    augmented_path = f"{output_dir}/augmented.csv"
+    augment_data(normalized_path, augmented_path, config)
+    
+    # Step 3: Split data
+    splits = split_data(augmented_path, f"{output_dir}/splits", config)
+    
+    logger.info("Pipeline complete!")
+    
+    return {
+        "normalized": normalized_path,
+        "augmented": augmented_path,
+        "splits": splits,
+    }
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Data processing pipeline")
+    parser.add_argument("--input", type=str, required=True, help="Input data file")
+    parser.add_argument("--output", type=str, default="data/processed", help="Output directory")
+    
+    args = parser.parse_args()
+    
+    run_pipeline(args.input, args.output)

pipelines/deployment_pipeline.py

@@ -0,0 +1,234 @@
+"""Model deployment pipeline for Myanmar Ghost."""
+
+import argparse
+import logging
+import shutil
+import sys
+from pathlib import Path
+from datetime import datetime
+
+sys.path.insert(0, str(Path(__file__).parent.parent.parent))
+
+from src.utils.logger import setup_logger
+
+logger = setup_logger("deployment_pipeline")
+
+
+def export_model(model_path: str, output_dir: str, format: str = "pytorch") -> str:
+    """Export model for deployment."""
+    logger.info(f"Exporting model from {model_path}")
+    
+    import torch
+    
+    # Load model
+    checkpoint = torch.load(model_path, map_location="cpu")
+    
+    # Create output directory
+    export_path = Path(output_dir) / "exported_model"
+    export_path.mkdir(parents=True, exist_ok=True)
+    
+    if format == "pytorch":
+        # Save as PyTorch model
+        torch.save(checkpoint, export_path / "model.pt")
+        logger.info(f"Exported to {export_path}")
+    
+    elif format == "onnx":
+        # Export to ONNX (requires model forward pass)
+        logger.warning("ONNX export not implemented yet")
+    
+    elif format == "safetensors":
+        # Save as safetensors
+        try:
+            from safetensors.torch import save_file
+            if "model_state_dict" in checkpoint:
+                state_dict = checkpoint["model_state_dict"]
+            else:
+                state_dict = checkpoint
+            
+            save_file(state_dict, export_path / "model.safetensors")
+            logger.info(f"Exported safetensors to {export_path}")
+        except ImportError:
+            logger.warning("safetensors not installed, using PyTorch format")
+            torch.save(checkpoint, export_path / "model.pt")
+    
+    # Save metadata
+    metadata = {
+        "exported_at": datetime.now().isoformat(),
+        "format": format,
+        "original_path": model_path,
+    }
+    
+    import json
+    with open(export_path / "metadata.json", "w") as f:
+        json.dump(metadata, f, indent=2)
+    
+    return str(export_path)
+
+
+def create_docker_image(model_path: str, output_dir: str, tag: str = "myanmar-ghost") -> str:
+    """Create Docker image for deployment."""
+    logger.info(f"Creating Docker image: {tag}")
+    
+    import subprocess
+    
+    # Create deployment directory
+    deploy_dir = Path(output_dir) / "docker"
+    deploy_dir.mkdir(parents=True, exist_ok=True)
+    
+    # Copy model
+    shutil.copytree(model_path, deploy_dir / "model", dirs_exist_ok=True)
+    
+    # Create Dockerfile
+    dockerfile = f"""
+FROM python:3.10-slim
+
+WORKDIR /app
+
+COPY model/ /app/model/
+COPY deployment/api/ /app/api/
+
+RUN pip install --no-cache-dir \\
+    torch \\
+    transformers \\
+    fastapi \\
+    uvicorn
+
+EXPOSE 8000
+
+CMD ["uvicorn", "api.app:app", "--host", "0.0.0.0", "--port", "8000"]
+"""
+    
+    with open(deploy_dir / "Dockerfile", "w") as f:
+        f.write(dockerfile)
+    
+    # Build image
+    result = subprocess.run(
+        ["docker", "build", "-t", tag, "."],
+        cwd=deploy_dir,
+        capture_output=True,
+        text=True,
+    )
+    
+    if result.returncode != 0:
+        logger.error(f"Docker build failed: {result.stderr}")
+        raise RuntimeError("Docker build failed")
+    
+    logger.info(f"Docker image created: {tag}")
+    return tag
+
+
+def push_to_huggingface(model_path: str, repo_id: str) -> str:
+    """Push model to HuggingFace Hub."""
+    logger.info(f"Pushing model to HuggingFace: {repo_id}")
+    
+    import subprocess
+    
+    try:
+        # Use huggingface_hub
+        from huggingface_hub import HfApi, create_repo
+        
+        api = HfApi()
+        
+        # Create repo if doesn't exist
+        try:
+            create_repo(repo_id, repo_type="model", exist_ok=True)
+        except Exception:
+            pass
+        
+        # Upload folder
+        api.upload_folder(
+            folder_path=model_path,
+            repo_id=repo_id,
+            repo_type="model",
+        )
+        
+        logger.info(f"Pushed to https://huggingface.co/{repo_id}")
+        return f"https://huggingface.co/{repo_id}"
+    
+    except ImportError:
+        logger.warning("huggingface_hub not installed, using CLI")
+        
+        result = subprocess.run([
+            "huggingface-cli", "upload",
+            repo_id,
+            model_path,
+        ], capture_output=True, text=True)
+        
+        if result.returncode != 0:
+            logger.error(f"HuggingFace upload failed: {result.stderr}")
+            raise RuntimeError("HuggingFace upload failed")
+        
+        return f"https://huggingface.co/{repo_id}"
+
+
+def deploy_to_render(api_token: str, service_name: str, model_path: str) -> str:
+    """Deploy to Render using blueprint."""
+    logger.info(f"Deploying to Render: {service_name}")
+    
+    import subprocess
+    
+    # Create render.yaml
+    render_yaml = f"""
+services:
+  - type: web
+    name: {service_name}
+    env: docker
+    repo: {model_path}
+    envVars:
+      - key: MODEL_PATH
+        value: /app/model
+"""
+    
+    with open("render.yaml", "w") as f:
+        f.write(render_yaml)
+    
+    logger.info("Created render.yaml - deploy manually via Render dashboard")
+    
+    return "render.yaml created"
+
+
+def run_deployment_pipeline(
+    model_path: str,
+    output_dir: str = "outputs/deployment",
+    format: str = "safetensors",
+    push_hub: bool = False,
+    repo_id: str = None,
+) -> dict:
+    """Run full deployment pipeline."""
+    logger.info("Starting deployment pipeline...")
+    
+    Path(output_dir).mkdir(parents=True, exist_ok=True)
+    
+    # Export model
+    export_path = export_model(model_path, output_dir, format)
+    
+    results = {
+        "export_path": export_path,
+    }
+    
+    # Push to HuggingFace if requested
+    if push_hub and repo_id:
+        hub_url = push_to_huggingface(export_path, repo_id)
+        results["hub_url"] = hub_url
+    
+    logger.info("Deployment pipeline complete!")
+    
+    return results
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Model deployment pipeline")
+    parser.add_argument("--model_path", type=str, required=True)
+    parser.add_argument("--output_dir", type=str, default="outputs/deployment")
+    parser.add_argument("--format", type=str, default="safetensors")
+    parser.add_argument("--push_hub", action="store_true")
+    parser.add_argument("--repo_id", type=str, default=None)
+    
+    args = parser.parse_args()
+    run_deployment_pipeline(
+        args.model_path,
+        args.output_dir,
+        args.format,
+        args.push_hub,
+        args.repo_id,
+    )

pipelines/evaluation_pipeline.py

@@ -0,0 +1,197 @@
+"""End-to-end model evaluation pipeline."""
+
+import argparse
+import json
+import logging
+import sys
+from datetime import datetime
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent.parent.parent))
+
+from src.utils.logger import setup_logger
+from src.utils.visualization import (
+    plot_confusion_matrix,
+    plot_label_distribution,
+)
+
+logger = setup_logger("evaluation_pipeline")
+
+
+def evaluate_model(model_path: str, test_data: str, output_dir: str) -> dict:
+    """Evaluate a trained model."""
+    logger.info(f"Evaluating model: {model_path}")
+    
+    # Run evaluation
+    import subprocess
+    
+    result = subprocess.run([
+        "python", "src/models/evaluate.py",
+        "--model_path", model_path,
+        "--data_path", test_data,
+        "--output_path", f"{output_dir}/results.json",
+    ], capture_output=True, text=True)
+    
+    logger.info(result.stdout)
+    if result.returncode != 0:
+        logger.error(result.stderr)
+        raise RuntimeError("Evaluation failed")
+    
+    # Load results
+    with open(f"{output_dir}/results.json", "r") as f:
+        results = json.load(f)
+    
+    return results
+
+
+def generate_report(results: dict, output_dir: str) -> str:
+    """Generate evaluation report with visualizations."""
+    logger.info("Generating evaluation report...")
+    
+    report_path = f"{output_dir}/report.html"
+    
+    # Create visualizations
+    cm = results["confusion_matrix"]
+    class_names = results["class_names"]
+    
+    plot_confusion_matrix(
+        cm, class_names,
+        title="Sentiment Classification Confusion Matrix",
+        output_path=f"{output_dir}/confusion_matrix.png",
+    )
+    
+    # Create HTML report
+    html = f"""
+    <!DOCTYPE html>
+    <html>
+    <head>
+        <title>Myanmar Ghost Evaluation Report</title>
+        <style>
+            body {{ font-family: Arial, sans-serif; margin: 40px; }}
+            .metric {{ display: inline-block; margin: 10px; padding: 20px; 
+                       background: #f0f0f0; border-radius: 8px; }}
+            .metric-value {{ font-size: 32px; font-weight: bold; color: #2196F3; }}
+            .metric-label {{ font-size: 14px; color: #666; }}
+            img {{ max-width: 600px; margin: 20px 0; }}
+        </style>
+    </head>
+    <body>
+        <h1>🇲🇲 Myanmar Ghost Evaluation Report</h1>
+        <p>Generated: {datetime.now().strftime("%Y-%m-%d %H:%M:%S")}</p>
+        
+        <h2>📊 Metrics</h2>
+        <div class="metrics">
+            <div class="metric">
+                <div class="metric-value">{results['metrics']['accuracy']:.2%}</div>
+                <div class="metric-label">Accuracy</div>
+            </div>
+            <div class="metric">
+                <div class="metric-value">{results['metrics']['f1_weighted']:.2%}</div>
+                <div class="metric-label">F1 (weighted)</div>
+            </div>
+            <div class="metric">
+                <div class="metric-value">{results['metrics']['precision_weighted']:.2%}</div>
+                <div class="metric-label">Precision</div>
+            </div>
+            <div class="metric">
+                <div class="metric-value">{results['metrics']['recall_weighted']:.2%}</div>
+                <div class="metric-label">Recall</div>
+            </div>
+        </div>
+        
+        <h2>📈 Confusion Matrix</h2>
+        <img src="confusion_matrix.png" alt="Confusion Matrix">
+        
+        <h2>📋 Per-Class Performance</h2>
+        <table border="1" cellpadding="10">
+            <tr>
+                <th>Class</th>
+                <th>F1 Score</th>
+            </tr>
+    """
+    
+    for name in class_names:
+        f1_key = f"f1_{name}"
+        if f1_key in results["metrics"]:
+            html += f"""
+            <tr>
+                <td>{name}</td>
+                <td>{results['metrics'][f1_key]:.4f}</td>
+            </tr>
+            """
+    
+    html += """
+        </table>
+    </body>
+    </html>
+    """
+    
+    with open(report_path, "w", encoding="utf-8") as f:
+        f.write(html)
+    
+    logger.info(f"Report saved to {report_path}")
+    return report_path
+
+
+def compare_models(model_results: list, output_dir: str) -> dict:
+    """Compare multiple model evaluations."""
+    logger.info(f"Comparing {len(model_results)} models...")
+    
+    comparison = {
+        "models": [],
+        "best_model": None,
+        "best_f1": 0,
+    }
+    
+    for result in model_results:
+        model_name = result.get("model_name", "unknown")
+        metrics = result.get("metrics", {})
+        
+        comparison["models"].append({
+            "name": model_name,
+            "accuracy": metrics.get("accuracy", 0),
+            "f1_weighted": metrics.get("f1_weighted", 0),
+            "f1_macro": metrics.get("f1_macro", 0),
+        })
+        
+        if metrics.get("f1_weighted", 0) > comparison["best_f1"]:
+            comparison["best_f1"] = metrics.get("f1_weighted", 0)
+            comparison["best_model"] = model_name
+    
+    # Save comparison
+    with open(f"{output_dir}/model_comparison.json", "w") as f:
+        json.dump(comparison, f, indent=2)
+    
+    logger.info(f"Best model: {comparison['best_model']} (F1: {comparison['best_f1']:.4f})")
+    
+    return comparison
+
+
+def run_evaluation_pipeline(
+    model_path: str,
+    test_data: str,
+    output_dir: str = "outputs/evaluation",
+) -> dict:
+    """Run full evaluation pipeline."""
+    Path(output_dir).mkdir(parents=True, exist_ok=True)
+    
+    # Evaluate model
+    results = evaluate_model(model_path, test_data, output_dir)
+    
+    # Generate report
+    report_path = generate_report(results, output_dir)
+    
+    return {
+        "results": results,
+        "report": report_path,
+    }
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--model_path", type=str, required=True)
+    parser.add_argument("--test_data", type=str, required=True)
+    parser.add_argument("--output_dir", type=str, default="outputs/evaluation")
+    
+    args = parser.parse_args()
+    run_evaluation_pipeline(args.model_path, args.test_data, args.output_dir)

utils/__init__.py

@@ -0,0 +1,19 @@
+"""Utility modules for Myanmar Ghost project."""
+
+from .logger import setup_logger, get_logger
+from .metrics import compute_metrics, MetricsTracker
+from .file_utils import load_json, save_json, load_yaml, save_yaml
+from .visualization import plot_training_curves, plot_confusion_matrix
+
+__all__ = [
+    "setup_logger",
+    "get_logger",
+    "compute_metrics",
+    "MetricsTracker",
+    "load_json",
+    "save_json",
+    "load_yaml",
+    "save_yaml",
+    "plot_training_curves",
+    "plot_confusion_matrix",
+]

utils/file_utils.py

@@ -0,0 +1,142 @@
+"""File I/O utilities for Myanmar Ghost project."""
+
+import json
+import os
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+import pandas as pd
+import yaml
+
+
+def load_json(path: str) -> Any:
+    """Load JSON file."""
+    with open(path, "r", encoding="utf-8") as f:
+        return json.load(f)
+
+
+def save_json(data: Any, path: str, indent: int = 2) -> None:
+    """Save data to JSON file."""
+    Path(path).parent.mkdir(parents=True, exist_ok=True)
+    with open(path, "w", encoding="utf-8") as f:
+        json.dump(data, f, indent=indent, ensure_ascii=False)
+
+
+def load_yaml(path: str) -> Dict:
+    """Load YAML file."""
+    with open(path, "r", encoding="utf-8") as f:
+        return yaml.safe_load(f)
+
+
+def save_yaml(data: Dict, path: str) -> None:
+    """Save data to YAML file."""
+    Path(path).parent.mkdir(parents=True, exist_ok=True)
+    with open(path, "w", encoding="utf-8") as f:
+        yaml.dump(data, f, allow_unicode=True, default_flow_style=False)
+
+
+def load_jsonl(path: str) -> List[Dict]:
+    """Load JSONL file (one JSON object per line)."""
+    data = []
+    with open(path, "r", encoding="utf-8") as f:
+        for line in f:
+            if line.strip():
+                data.append(json.loads(line))
+    return data
+
+
+def save_jsonl(data: List[Dict], path: str) -> None:
+    """Save data to JSONL file."""
+    Path(path).parent.mkdir(parents=True, exist_ok=True)
+    with open(path, "w", encoding="utf-8") as f:
+        for item in data:
+            f.write(json.dumps(item, ensure_ascii=False) + "\n")
+
+
+def load_csv(path: str) -> pd.DataFrame:
+    """Load CSV file as DataFrame."""
+    return pd.read_csv(path)
+
+
+def save_csv(df: pd.DataFrame, path: str, index: bool = False) -> None:
+    """Save DataFrame to CSV file."""
+    Path(path).parent.mkdir(parents=True, exist_ok=True)
+    df.to_csv(path, index=index)
+
+
+def ensure_dir(path: str) -> Path:
+    """Ensure directory exists."""
+    p = Path(path)
+    p.mkdir(parents=True, exist_ok=True)
+    return p
+
+
+def list_files(
+    directory: str,
+    pattern: str = "*",
+    recursive: bool = False,
+) -> List[Path]:
+    """List files in directory matching pattern."""
+    p = Path(directory)
+    if recursive:
+        return list(p.rglob(pattern))
+    return list(p.glob(pattern))
+
+
+def get_file_size(path: str) -> int:
+    """Get file size in bytes."""
+    return os.path.getsize(path)
+
+
+def copy_file(src: str, dst: str) -> None:
+    """Copy file from src to dst."""
+    import shutil
+    Path(dst).parent.mkdir(parents=True, exist_ok=True)
+    shutil.copy2(src, dst)
+
+
+def move_file(src: str, dst: str) -> None:
+    """Move file from src to dst."""
+    import shutil
+    Path(dst).parent.mkdir(parents=True, exist_ok=True)
+    shutil.move(src, dst)
+
+
+def delete_file(path: str) -> None:
+    """Delete file."""
+    Path(path).unlink(missing_ok=True)
+
+
+class ConfigManager:
+    """Manage configuration files."""
+    
+    def __init__(self, config_dir: str = "configs"):
+        self.config_dir = Path(config_dir)
+    
+    def load(self, name: str, config_type: str = "yaml") -> Dict:
+        """Load configuration by name."""
+        path = self.config_dir / f"{name}.{config_type}"
+        
+        if config_type == "yaml":
+            return load_yaml(str(path))
+        elif config_type == "json":
+            return load_json(str(path))
+        else:
+            raise ValueError(f"Unsupported config type: {config_type}")
+    
+    def save(self, name: str, config: Dict, config_type: str = "yaml") -> None:
+        """Save configuration by name."""
+        path = self.config_dir / f"{name}.{config_type}"
+        
+        if config_type == "yaml":
+            save_yaml(config, str(path))
+        elif config_type == "json":
+            save_json(config, str(path))
+        else:
+            raise ValueError(f"Unsupported config type: {config_type}")
+
+
+if __name__ == "__main__":
+    # Test file utilities
+    print("File utilities loaded")
+    print(f"Current directory: {Path.cwd()}")

utils/logger.py

@@ -0,0 +1,139 @@
+"""Logging utilities for Myanmar Ghost project."""
+
+import logging
+import sys
+from datetime import datetime
+from pathlib import Path
+from typing import Optional
+
+from loguru import logger as _logger
+
+
+def setup_logger(
+    name: str = "myanmar_ghost",
+    log_dir: Optional[str] = None,
+    level: str = "INFO",
+    format: str = None,
+) -> logging.Logger:
+    """Set up logger with file and console output.
+    
+    Args:
+        name: Logger name
+        log_dir: Directory for log files
+        level: Logging level
+        format: Custom log format
+    
+    Returns:
+        Configured logger
+    """
+    if format is None:
+        format = (
+            "<green>{time:YYYY-MM-DD HH:mm:ss}</green> | "
+            "<level>{level: <8}</level> | "
+            "<cyan>{name}</cyan>:<cyan>{function}</cyan>:<cyan>{line}</cyan> | "
+            "<level>{message}</level>"
+        )
+    
+    # Remove default handler
+    _logger.remove()
+    
+    # Console output
+    _logger.add(
+        sys.stdout,
+        format=format,
+        level=level,
+        colorize=True,
+    )
+    
+    # File output
+    if log_dir:
+        log_path = Path(log_dir)
+        log_path.mkdir(parents=True, exist_ok=True)
+        
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+        log_file = log_path / f"{name}_{timestamp}.log"
+        
+        _logger.add(
+            log_file,
+            format=format,
+            level=level,
+            rotation="100 MB",
+            retention="30 days",
+            compression="zip",
+        )
+    
+    return _logger
+
+
+def get_logger(name: str = None) -> logging.Logger:
+    """Get logger instance.
+    
+    Args:
+        name: Logger name (optional)
+    
+    Returns:
+        Logger instance
+    """
+    return _logger
+
+
+class TrainingLogger:
+    """Logger for training metrics and progress."""
+    
+    def __init__(
+        self,
+        log_dir: str = "outputs/logs",
+        experiment_name: str = "experiment",
+    ):
+        self.log_dir = Path(log_dir)
+        self.log_dir.mkdir(parents=True, exist_ok=True)
+        
+        self.experiment_name = experiment_name
+        self.metrics_history = []
+        
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+        self.log_file = self.log_dir / f"{experiment_name}_{timestamp}.log"
+    
+    def log_metrics(self, metrics: dict, step: int) -> None:
+        """Log metrics at a specific step."""
+        entry = {
+            "step": step,
+            "timestamp": datetime.now().isoformat(),
+            **metrics,
+        }
+        self.metrics_history.append(entry)
+        
+        log_line = f"Step {step}: " + ", ".join(
+            f"{k}={v:.4f}" if isinstance(v, float) else f"{k}={v}"
+            for k, v in metrics.items()
+        )
+        _logger.info(log_line)
+    
+    def log_epoch(self, epoch: int, metrics: dict) -> None:
+        """Log metrics at epoch end."""
+        entry = {
+            "epoch": epoch,
+            "timestamp": datetime.now().isoformat(),
+            **metrics,
+        }
+        self.metrics_history.append(entry)
+        
+        log_line = f"Epoch {epoch}: " + ", ".join(
+            f"{k}={v:.4f}" if isinstance(v, float) else f"{k}={v}"
+            for k, v in metrics.items()
+        )
+        _logger.info(log_line)
+    
+    def save_history(self) -> str:
+        """Save metrics history to file."""
+        import json
+        
+        with open(self.log_file, "w", encoding="utf-8") as f:
+            json.dump(self.metrics_history, f, indent=2)
+        
+        return str(self.log_file)
+
+
+if __name__ == "__main__":
+    logger = setup_logger("test_logger", "outputs/logs")
+    logger.info("Logger initialized successfully")

utils/metrics.py

@@ -0,0 +1,176 @@
+"""Metrics computation utilities for Myanmar Ghost project."""
+
+from dataclasses import dataclass
+from typing import Any, Dict, List, Optional
+
+import numpy as np
+import torch
+from sklearn.metrics import (
+    accuracy_score,
+    f1_score,
+    precision_score,
+    recall_score,
+    confusion_matrix,
+    classification_report,
+)
+
+
+@dataclass
+class MetricResult:
+    """Result of metric computation."""
+    name: str
+    value: float
+    std: Optional[float] = None
+
+
+def compute_accuracy(predictions: List[int], targets: List[int]) -> float:
+    """Compute accuracy."""
+    return accuracy_score(targets, predictions)
+
+
+def compute_f1(
+    predictions: List[int],
+    targets: List[int],
+    average: str = "weighted",
+) -> float:
+    """Compute F1 score."""
+    return f1_score(targets, predictions, average=average, zero_division=0)
+
+
+def compute_precision(
+    predictions: List[int],
+    targets: List[int],
+    average: str = "weighted",
+) -> float:
+    """Compute precision score."""
+    return precision_score(targets, predictions, average=average, zero_division=0)
+
+
+def compute_recall(
+    predictions: List[int],
+    targets: List[int],
+    average: str = "weighted",
+) -> float:
+    """Compute recall score."""
+    return recall_score(targets, predictions, average=average, zero_division=0)
+
+
+def compute_confusion_matrix(
+    predictions: List[int],
+    targets: List[int],
+) -> np.ndarray:
+    """Compute confusion matrix."""
+    return confusion_matrix(targets, predictions)
+
+
+def compute_metrics(
+    predictions: List[int],
+    targets: List[int],
+    class_names: Optional[List[str]] = None,
+) -> Dict[str, Any]:
+    """Compute all metrics.
+    
+    Args:
+        predictions: List of predicted labels
+        targets: List of ground truth labels
+        class_names: Optional list of class names
+    
+    Returns:
+        Dictionary of metrics
+    """
+    metrics = {
+        "accuracy": compute_accuracy(predictions, targets),
+        "f1_weighted": compute_f1(predictions, targets, "weighted"),
+        "f1_macro": compute_f1(predictions, targets, "macro"),
+        "f1_micro": compute_f1(predictions, targets, "micro"),
+        "precision_weighted": compute_precision(predictions, targets, "weighted"),
+        "precision_macro": compute_precision(predictions, targets, "macro"),
+        "recall_weighted": compute_recall(predictions, targets, "weighted"),
+        "recall_macro": compute_recall(predictions, targets, "macro"),
+    }
+    
+    # Per-class metrics
+    labels = list(range(len(class_names))) if class_names else None
+    per_class_f1 = f1_score(targets, predictions, labels=labels, average=None, zero_division=0)
+    
+    if class_names:
+        for i, name in enumerate(class_names):
+            metrics[f"f1_{name}"] = per_class_f1[i]
+    
+    return metrics
+
+
+class MetricsTracker:
+    """Track metrics during training."""
+    
+    def __init__(
+        self,
+        metrics: List[str] = None,
+        class_names: Optional[List[str]] = None,
+    ):
+        self.metrics = metrics or ["loss", "accuracy", "f1"]
+        self.class_names = class_names or ["negative", "neutral", "positive", "sarcastic"]
+        
+        self.history = {m: [] for m in self.metrics}
+        self.best_values = {m: float("-inf") for m in self.metrics}
+        self.best_epochs = {m: 0 for m in self.metrics}
+    
+    def update(self, metrics: Dict[str, float], step: int) -> None:
+        """Update metrics at current step."""
+        for name, value in metrics.items():
+            if name in self.metrics:
+                self.history[name].append((step, value))
+                
+                # Track best
+                if value > self.best_values[name]:
+                    self.best_values[name] = value
+                    self.best_epochs[name] = step
+    
+    def get_current(self, metric_name: str) -> float:
+        """Get current value of a metric."""
+        if metric_name in self.history and self.history[metric_name]:
+            return self.history[metric_name][-1][1]
+        return 0.0
+    
+    def get_best(self, metric_name: str) -> tuple:
+        """Get best value and epoch of a metric."""
+        return self.best_values.get(metric_name, 0), self.best_epochs.get(metric_name, 0)
+    
+    def get_summary(self) -> Dict[str, Any]:
+        """Get summary of all metrics."""
+        return {
+            "best": self.best_values,
+            "best_epochs": self.best_epochs,
+            "current": {m: self.get_current(m) for m in self.metrics},
+        }
+
+
+def compute_bleu(
+    predictions: List[str],
+    references: List[str],
+) -> float:
+    """Compute BLEU score for text generation."""
+    from sacrebleu import sentence_bleu
+    
+    scores = []
+    for pred, ref in zip(predictions, references):
+        score = sentence_bleu(pred, [ref])
+        scores.append(score.score)
+    
+    return np.mean(scores)
+
+
+def compute_perplexity(
+    loss: float,
+) -> float:
+    """Compute perplexity from cross-entropy loss."""
+    return np.exp(loss)
+
+
+if __name__ == "__main__":
+    # Test metrics computation
+    predictions = [0, 1, 2, 0, 1, 2, 0, 1, 2]
+    targets = [0, 1, 2, 0, 1, 1, 0, 0, 2]
+    
+    metrics = compute_metrics(predictions, targets)
+    print("Metrics:", metrics)

utils/visualization.py

@@ -0,0 +1,270 @@
+"""Visualization utilities for Myanmar Ghost project."""
+
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+import matplotlib.pyplot as plt
+import numpy as np
+import seaborn as sns
+
+
+def plot_training_curves(
+    history: Dict[str, List[float]],
+    metrics: List[str] = None,
+    title: str = "Training Curves",
+    output_path: Optional[str] = None,
+    figsize: tuple = (12, 8),
+) -> plt.Figure:
+    """Plot training curves for multiple metrics.
+    
+    Args:
+        history: Dictionary mapping metric names to lists of values
+        metrics: List of metrics to plot (default: all)
+        title: Plot title
+        output_path: Path to save figure
+        figsize: Figure size
+    
+    Returns:
+        Matplotlib figure
+    """
+    if metrics is None:
+        metrics = list(history.keys())
+    
+    n_metrics = len(metrics)
+    n_cols = min(2, n_metrics)
+    n_rows = (n_metrics + n_cols - 1) // n_cols
+    
+    fig, axes = plt.subplots(n_rows, n_cols, figsize=figsize)
+    fig.suptitle(title, fontsize=16)
+    
+    if n_metrics == 1:
+        axes = [axes]
+    else:
+        axes = axes.flatten() if hasattr(axes, 'flatten') else axes
+    
+    for i, metric in enumerate(metrics):
+        ax = axes[i] if i < len(axes) else axes[0]
+        
+        if metric in history:
+            values = history[metric]
+            steps = list(range(len(values)))
+            
+            ax.plot(steps, values, marker='o', markersize=3)
+            ax.set_xlabel('Step/Epoch')
+            ax.set_ylabel(metric.capitalize())
+            ax.set_title(metric.capitalize())
+            ax.grid(True, alpha=0.3)
+    
+    # Hide unused subplots
+    for i in range(n_metrics, len(axes)):
+        axes[i].set_visible(False)
+    
+    plt.tight_layout()
+    
+    if output_path:
+        Path(output_path).parent.mkdir(parents=True, exist_ok=True)
+        plt.savefig(output_path, dpi=150, bbox_inches='tight')
+    
+    return fig
+
+
+def plot_confusion_matrix(
+    cm: np.ndarray,
+    class_names: List[str],
+    title: str = "Confusion Matrix",
+    output_path: Optional[str] = None,
+    figsize: tuple = (10, 8),
+    normalize: bool = False,
+) -> plt.Figure:
+    """Plot confusion matrix.
+    
+    Args:
+        cm: Confusion matrix
+        class_names: Names of classes
+        title: Plot title
+        output_path: Path to save figure
+        figsize: Figure size
+        normalize: Whether to normalize
+    
+    Returns:
+        Matplotlib figure
+    """
+    if normalize:
+        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
+    
+    fig, ax = plt.subplots(figsize=figsize)
+    
+    sns.heatmap(
+        cm,
+        annot=True,
+        fmt='.2f' if normalize else 'd',
+        cmap='Blues',
+        xticklabels=class_names,
+        yticklabels=class_names,
+        ax=ax,
+    )
+    
+    ax.set_xlabel('Predicted')
+    ax.set_ylabel('True')
+    ax.set_title(title)
+    
+    plt.tight_layout()
+    
+    if output_path:
+        Path(output_path).parent.mkdir(parents=True, exist_ok=True)
+        plt.savefig(output_path, dpi=150, bbox_inches='tight')
+    
+    return fig
+
+
+def plot_label_distribution(
+    labels: List[Any],
+    class_names: Optional[List[str]] = None,
+    title: str = "Label Distribution",
+    output_path: Optional[str] = None,
+    figsize: tuple = (10, 6),
+) -> plt.Figure:
+    """Plot distribution of labels.
+    
+    Args:
+        labels: List of labels
+        class_names: Names of classes
+        title: Plot title
+        output_path: Path to save figure
+        figsize: Figure size
+    
+    Returns:
+        Matplotlib figure
+    """
+    from collections import Counter
+    
+    counts = Counter(labels)
+    
+    if class_names:
+        labels_order = class_names
+        values = [counts.get(l, 0) for l in labels_order]
+    else:
+        labels_order = list(counts.keys())
+        values = list(counts.values())
+    
+    fig, ax = plt.subplots(figsize=figsize)
+    
+    bars = ax.bar(labels_order, values, color='steelblue', alpha=0.7)
+    
+    # Add count labels on bars
+    for bar, count in zip(bars, values):
+        height = bar.get_height()
+        ax.text(
+            bar.get_x() + bar.get_width() / 2.,
+            height,
+            f'{int(count)}',
+            ha='center',
+            va='bottom',
+        )
+    
+    ax.set_xlabel('Class')
+    ax.set_ylabel('Count')
+    ax.set_title(title)
+    ax.grid(True, alpha=0.3, axis='y')
+    
+    plt.tight_layout()
+    
+    if output_path:
+        Path(output_path).parent.mkdir(parents=True, exist_ok=True)
+        plt.savefig(output_path, dpi=150, bbox_inches='tight')
+    
+    return fig
+
+
+def plot_attention_weights(
+    attention_weights: np.ndarray,
+    tokens: List[str],
+    title: str = "Attention Weights",
+    output_path: Optional[str] = None,
+    figsize: tuple = (12, 10),
+) -> plt.Figure:
+    """Plot attention weights heatmap.
+    
+    Args:
+        attention_weights: Attention weight matrix
+        tokens: List of tokens
+        title: Plot title
+        output_path: Path to save figure
+        figsize: Figure size
+    
+    Returns:
+        Matplotlib figure
+    """
+    fig, ax = plt.subplots(figsize=figsize)
+    
+    sns.heatmap(
+        attention_weights,
+        xticklabels=tokens,
+        yticklabels=tokens,
+        cmap='viridis',
+        ax=ax,
+        cbar_kw={'label': 'Attention Weight'},
+    )
+    
+    ax.set_xlabel('Key Tokens')
+    ax.set_ylabel('Query Tokens')
+    ax.set_title(title)
+    
+    plt.tight_layout()
+    
+    if output_path:
+        Path(output_path).parent.mkdir(parents=True, exist_ok=True)
+        plt.savefig(output_path, dpi=150, bbox_inches='tight')
+    
+    return fig
+
+
+def plot_loss_landscape(
+    losses: np.ndarray,
+    xlabel: str = "x",
+    ylabel: str = "y",
+    title: str = "Loss Landscape",
+    output_path: Optional[str] = None,
+    figsize: tuple = (10, 6),
+) -> plt.Figure:
+    """Plot loss landscape.
+    
+    Args:
+        losses: 2D array of loss values
+        xlabel: Label for x-axis
+        ylabel: Label for y-axis
+        title: Plot title
+        output_path: Path to save figure
+        figsize: Figure size
+    
+    Returns:
+        Matplotlib figure
+    """
+    fig, ax = plt.subplots(figsize=figsize)
+    
+    if losses.ndim == 1:
+        ax.plot(losses)
+    else:
+        sns.heatmap(losses, ax=ax, cmap='viridis')
+    
+    ax.set_xlabel(xlabel)
+    ax.set_ylabel(ylabel)
+    ax.set_title(title)
+    
+    plt.tight_layout()
+    
+    if output_path:
+        Path(output_path).parent.mkdir(parents=True, exist_ok=True)
+        plt.savefig(output_path, dpi=150, bbox_inches='tight')
+    
+    return fig
+
+
+if __name__ == "__main__":
+    print("Visualization utilities loaded")
+    print("Available functions:")
+    print("  - plot_training_curves")
+    print("  - plot_confusion_matrix")
+    print("  - plot_label_distribution")
+    print("  - plot_attention_weights")
+    print("  - plot_loss_landscape")

xai/lime_explainer.py

@@ -0,0 +1,313 @@
+"""LIME explainer for Myanmar Ghost model.
+
+Uses LIME (Local Interpretable Model-agnostic Explanations)
+to explain individual predictions.
+"""
+
+import logging
+from typing import Any, Callable, Dict, List, Optional, Tuple
+
+import numpy as np
+import torch
+from lime.lime_text import LimeTextExplainer
+
+logger = logging.getLogger(__name__)
+
+
+class ThankingLIMEExplainer:
+    """LIME-based explainer for Myanmar text classification."""
+    
+    def __init__(
+        self,
+        model,
+        tokenizer,
+        class_names: Optional[List[str]] = None,
+        kernel_width: float = 25.0,
+    ):
+        """
+        Args:
+            model: PyTorch model
+            tokenizer: Tokenizer
+            class_names: Names for output classes
+            kernel_width: Kernel width for LIME
+        """
+        self.model = model
+        self.tokenizer = tokenizer
+        self.class_names = class_names or [
+            "negative", "neutral", "positive", "sarcastic"
+        ]
+        
+        self.model.eval()
+        
+        # Create LIME explainer
+        self.explainer = LimeTextExplainer(
+            class_names=self.class_names,
+            kernel_width=kernel_width,
+            verbose=False,
+        )
+    
+    def _predict_proba(self, texts: List[str]) -> np.ndarray:
+        """Prediction function for LIME.
+        
+        Args:
+            texts: List of text strings
+        
+        Returns:
+            Probability array (n_samples, n_classes)
+        """
+        # Tokenize
+        encoding = self.tokenizer(
+            texts,
+            padding=True,
+            truncation=True,
+            max_length=128,
+            return_tensors="pt",
+        )
+        
+        input_ids = encoding["input_ids"]
+        attention_mask = encoding["attention_mask"]
+        
+        with torch.no_grad():
+            outputs = self.model(input_ids, attention_mask)
+            
+            if hasattr(outputs, "logits"):
+                logits = outputs.logits
+            else:
+                logits = outputs
+            
+            probs = torch.softmax(logits, dim=-1)
+        
+        return probs.cpu().numpy()
+    
+    def explain(
+        self,
+        text: str,
+        num_features: int = 10,
+        num_samples: int = 5000,
+        top_labels: int = 4,
+    ) -> Any:
+        """Explain a single text prediction.
+        
+        Args:
+            text: Myanmar text to explain
+            num_features: Number of features to show
+            num_samples: Number of samples for LIME
+            top_labels: Number of top labels to explain
+        
+        Returns:
+            LIME Explanation object
+        """
+        logger.info(f"Explaining with LIME: {text[:50]}...")
+        
+        explanation = self.explainer.explain_instance(
+            text,
+            self._predict_proba,
+            num_features=num_features,
+            num_samples=num_samples,
+            top_labels=top_labels,
+        )
+        
+        return explanation
+    
+    def get_word_importance(
+        self,
+        text: str,
+        class_index: Optional[int] = None,
+        num_features: int = 10,
+    ) -> List[Tuple[str, float]]:
+        """Get word importance for a specific class.
+        
+        Args:
+            text: Myanmar text
+            class_index: Class index (None = predicted class)
+            num_features: Number of top features
+        
+        Returns:
+            List of (word, importance) tuples
+        """
+        explanation = self.explain(text, num_features=num_features)
+        
+        if class_index is None:
+            # Use predicted class
+            class_index = explanation.available_labels()[0]
+        
+        exp_list = explanation.as_list(label=class_index)
+        
+        return exp_list
+    
+    def visualize(
+        self,
+        explanation: Any,
+        output_path: Optional[str] = None,
+    ) -> str:
+        """Generate text visualization of explanation.
+        
+        Args:
+            explanation: LIME Explanation object
+            output_path: Optional path to save
+        
+        Returns:
+            Visualization text
+        """
+        output = "\n" + "=" * 60 + "\n"
+        output += "LIME EXPLANATION\n"
+        output += "=" * 60 + "\n"
+        
+        for label in explanation.available_labels()[:3]:
+            label_name = self.class_names[label]
+            output += f"\n{label_name.upper()}:\n"
+            output += "-" * 40 + "\n"
+            
+            for word, weight in explanation.as_list(label=label):
+                sign = "+" if weight > 0 else ""
+                output += f"  {word}: {sign}{weight:.4f}\n"
+        
+        print(output)
+        
+        if output_path:
+            with open(output_path, "w", encoding="utf-8") as f:
+                f.write(output)
+            logger.info(f"Visualization saved to {output_path}")
+        
+        return output
+    
+    def batch_explain(
+        self,
+        texts: List[str],
+        num_features: int = 10,
+    ) -> List[Dict[str, Any]]:
+        """Explain multiple texts.
+        
+        Args:
+            texts: List of texts
+            num_features: Number of features per explanation
+        
+        Returns:
+            List of explanation dictionaries
+        """
+        results = []
+        
+        for text in texts:
+            explanation = self.explain(text, num_features=num_features)
+            
+            result = {
+                "text": text,
+                "predicted_class": self.class_names[
+                    explanation.available_labels()[0]
+                ],
+                "explanations": {},
+            }
+            
+            for label in explanation.available_labels():
+                result["explanations"][self.class_names[label]] = {
+                    word: weight
+                    for word, weight in explanation.as_list(label=label)
+                }
+            
+            results.append(result)
+        
+        return results
+
+
+class SegmentLevelLIME:
+    """LIME with Myanmar-specific segmentation."""
+    
+    def __init__(
+        self,
+        model,
+        tokenizer,
+        segment_syllables: bool = True,
+    ):
+        """
+        Args:
+            model: PyTorch model
+            tokenizer: Tokenizer
+            segment_syllables: Segment by Myanmar syllables
+        """
+        self.model = model
+        self.tokenizer = tokenizer
+        self.segment_syllables = segment_syllables
+    
+    def _segment_text(self, text: str) -> List[str]:
+        """Segment text into interpretable units.
+        
+        For Myanmar, this can be syllable or word level.
+        """
+        if self.segment_syllables:
+            # Simple syllable segmentation
+            # Myanmar syllables end with vowel markers or consonants
+            segments = []
+            current = ""
+            
+            for char in text:
+                current += char
+                # Check for syllable boundary (simplified)
+                if char in "း့်ှင်း":
+                    segments.append(current)
+                    current = ""
+            
+            if current:
+                segments.append(current)
+            
+            return segments if segments else [text]
+        else:
+            return text.split()
+    
+    def _join_segments(self, segments: List[str]) -> str:
+        """Join segments back to text."""
+        return "".join(segments)
+    
+    def explain(
+        self,
+        text: str,
+        num_samples: int = 1000,
+    ) -> Dict[str, Any]:
+        """Explain text with syllable-level segmentation.
+        
+        Args:
+            text: Myanmar text
+            num_samples: Number of LIME samples
+        
+        Returns:
+            Explanation dictionary
+        """
+        segments = self._segment_text(text)
+        
+        logger.info(f"Segmented into {len(segments)} units")
+        
+        # Use standard LIME with segmented text
+        base_explainer = ThankingLIMEExplainer(
+            self.model,
+            self.tokenizer,
+        )
+        
+        explanation = base_explainer.explain(
+            text,
+            num_features=len(segments),
+            num_samples=num_samples,
+        )
+        
+        return {
+            "text": text,
+            "segments": segments,
+            "explanation": explanation,
+            "word_importance": base_explainer.get_word_importance(text),
+        }
+
+
+def create_lime_explainer(
+    model,
+    tokenizer,
+    class_names: Optional[List[str]] = None,
+) -> ThankingLIMEExplainer:
+    """Factory function to create LIME explainer."""
+    return ThankingLIMEExplainer(
+        model=model,
+        tokenizer=tokenizer,
+        class_names=class_names,
+    )
+
+
+if __name__ == "__main__":
+    print("ThankingLIMEExplainer loaded")
+    print("Use create_lime_explainer() to create an explainer")

xai/shap_explainer.py

@@ -0,0 +1,410 @@
+"""SHAP explainer for Myanmar Ghost model.
+
+Uses SHAP (SHapley Additive exPlanations) to explain
+individual predictions and word importance.
+"""
+
+import logging
+from pathlib import Path
+from typing import Any, Callable, Dict, List, Optional, Tuple, Union
+
+import numpy as np
+import shap
+import torch
+from tqdm import tqdm
+
+logger = logging.getLogger(__name__)
+
+
+class ThankingSHAPExplainer:
+    """SHAP-based explainer for Myanmar text classification."""
+    
+    def __init__(
+        self,
+        model,
+        tokenizer,
+        background_size: int = 100,
+        device: str = "cuda" if torch.cuda.is_available() else "cpu",
+    ):
+        """
+        Args:
+            model: PyTorch model or HuggingFace model
+            tokenizer: Tokenizer for the model
+            background_size: Number of background samples for SHAP
+            device: Device to run on
+        """
+        self.model = model
+        self.tokenizer = tokenizer
+        self.background_size = background_size
+        self.device = device
+        
+        self.model.to(device)
+        self.model.eval()
+        
+        self.explainer = None
+        self.background_data = None
+    
+    def _get_tokenizer(self):
+        """Get the tokenizer, handling both HF and custom tokenizers."""
+        if hasattr(self.tokenizer, "__call__"):
+            return self.tokenizer
+        return self.tokenizer.encode
+    
+    def _predict(self, texts: Union[List[str], np.ndarray]) -> np.ndarray:
+        """Model prediction function for SHAP."""
+        if isinstance(texts, np.ndarray):
+            texts = texts.tolist()
+        
+        # Tokenize
+        if hasattr(self.tokenizer, "batch_encode_plus"):
+            encoding = self.tokenizer.batch_encode_plus(
+                texts,
+                padding=True,
+                truncation=True,
+                max_length=128,
+                return_tensors="pt",
+            )
+            input_ids = encoding["input_ids"].to(self.device)
+            attention_mask = encoding["attention_mask"].to(self.device)
+        else:
+            input_ids = torch.tensor(
+                [self.tokenizer.encode(t) for t in texts]
+            ).to(self.device)
+            attention_mask = (input_ids != 0).long().to(self.device)
+        
+        with torch.no_grad():
+            outputs = self.model(input_ids, attention_mask)
+            
+            if hasattr(outputs, "logits"):
+                logits = outputs.logits
+            else:
+                logits = outputs
+            
+            probs = torch.softmax(logits, dim=-1).cpu().numpy()
+        
+        return probs
+    
+    def fit_background(
+        self,
+        background_texts: List[str],
+    ) -> None:
+        """Fit background distribution for SHAP.
+        
+        Args:
+            background_texts: List of texts to use as background
+        """
+        logger.info(f"Fitting SHAP background with {len(background_texts)} samples")
+        
+        # Sample background if too large
+        if len(background_texts) > self.background_size:
+            indices = np.random.choice(
+                len(background_texts),
+                self.background_size,
+                replace=False,
+            )
+            background_texts = [background_texts[i] for i in indices]
+        
+        self.background_data = background_texts
+        
+        # Create SHAP explainer
+        self.explainer = shap.Explainer(
+            self._predict,
+            self.tokenizer,
+            output_names=["negative", "neutral", "positive", "sarcastic"],
+        )
+        
+        # Calculate background values
+        logger.info("Computing SHAP values for background...")
+        self.explainer(background_texts[:min(10, len(background_texts))])
+        
+        logger.info("Background fitting complete")
+    
+    def explain(
+        self,
+        text: str,
+        num_samples: int = 100,
+        output_names: Optional[List[str]] = None,
+    ) -> shap.Explanation:
+        """Explain a single text.
+        
+        Args:
+            text: Myanmar text to explain
+            num_samples: Number of Monte Carlo samples
+            output_names: Names for output classes
+        
+        Returns:
+            SHAP Explanation object
+        """
+        if self.explainer is None:
+            logger.warning("No background data. Using default explainer.")
+            self.explainer = shap.Explainer(
+                self._predict,
+                self.tokenizer,
+                output_names=output_names or ["negative", "neutral", "positive", "sarcastic"],
+            )
+        
+        logger.info(f"Explaining text: {text[:50]}...")
+        
+        explainer = shap.Explainer(
+            self._predict,
+            self.tokenizer,
+            output_names=output_names,
+        )
+        
+        shap_values = explainer([text])
+        
+        return shap_values
+    
+    def explain_batch(
+        self,
+        texts: List[str],
+        output_names: Optional[List[str]] = None,
+    ) -> List[shap.Explanation]:
+        """Explain multiple texts.
+        
+        Args:
+            texts: List of Myanmar texts
+            output_names: Names for output classes
+        
+        Returns:
+            List of SHAP Explanation objects
+        """
+        if output_names is None:
+            output_names = ["negative", "neutral", "positive", "sarcastic"]
+        
+        explainer = shap.Explainer(
+            self._predict,
+            self.tokenizer,
+            output_names=output_names,
+        )
+        
+        explanations = []
+        for text in tqdm(texts, desc="Explaining texts"):
+            exp = explainer([text])
+            explanations.append(exp)
+        
+        return explanations
+    
+    def get_word_importance(
+        self,
+        text: str,
+        class_index: int = 2,  # positive by default
+    ) -> List[Tuple[str, float]]:
+        """Get word importance scores for a specific class.
+        
+        Args:
+            text: Myanmar text
+            class_index: Class index to explain
+        
+        Returns:
+            List of (word, importance) tuples
+        """
+        explanation = self.explain(text)
+        
+        # Get tokens and their SHAP values
+        tokens = self.tokenizer.tokenize(text)
+        shap_vals = explanation.values[0, :, class_index]
+        
+        # Handle tokenization differences
+        if len(shap_vals) < len(tokens):
+            # Pad if needed
+            shap_vals = np.pad(
+                shap_vals,
+                (0, len(tokens) - len(shap_vals)),
+                constant_values=0,
+            )
+        elif len(shap_vals) > len(tokens):
+            tokens = tokens + ["[PAD]"] * (len(shap_vals) - len(tokens))
+        
+        # Create word-score pairs
+        word_importance = list(zip(tokens, shap_vals.tolist()))
+        
+        # Sort by absolute importance
+        word_importance.sort(key=lambda x: abs(x[1]), reverse=True)
+        
+        return word_importance
+    
+    def visualize_text(
+        self,
+        explanation: shap.Explanation,
+        output_path: Optional[str] = None,
+    ) -> None:
+        """Visualize SHAP explanation as text.
+        
+        Args:
+            explanation: SHAP explanation
+            output_path: Optional path to save visualization
+        """
+        text = explanation.data[0] if hasattr(explanation, "data") else ""
+        
+        output = f"\n{'='*60}\n"
+        output += f"Text: {text}\n"
+        output += f"{'='*60}\n"
+        
+        # Get top features for each class
+        for i, class_name in enumerate(explanation.output_names):
+            values = explanation.values[0, :, i]
+            
+            # Get top 5 words
+            top_indices = np.argsort(np.abs(values))[-5:][::-1]
+            
+            output += f"\nClass: {class_name}\n"
+            output += "-" * 40 + "\n"
+            
+            for idx in top_indices:
+                if idx < len(text.split()):
+                    word = text.split()[idx]
+                    output += f"  {word}: {values[idx]:.4f}\n"
+        
+        print(output)
+        
+        if output_path:
+            with open(output_path, "w", encoding="utf-8") as f:
+                f.write(output)
+            logger.info(f"Visualization saved to {output_path}")
+
+
+class ThankingSHAPValues:
+    """Compute SHAP values for thanking expression analysis."""
+    
+    def __init__(
+        self,
+        model,
+        tokenizer,
+        class_names: List[str] = None,
+    ):
+        self.model = model
+        self.tokenizer = tokenizer
+        self.class_names = class_names or [
+            "genuine", "sarcastic", "complaining", "neutral"
+        ]
+    
+    def compute_feature_importance(
+        self,
+        texts: List[str],
+        feature_names: List[str],
+    ) -> Dict[str, float]:
+        """Compute SHAP-based feature importance.
+        
+        Args:
+            texts: List of texts
+            feature_names: Names of features to analyze
+        
+        Returns:
+            Dictionary of feature importance scores
+        """
+        import shap
+        
+        def predict_proba(texts: List[str]) -> np.ndarray:
+            encoding = self.tokenizer.batch_encode_plus(
+                texts,
+                padding=True,
+                truncation=True,
+                max_length=128,
+                return_tensors="pt",
+            )
+            
+            with torch.no_grad():
+                outputs = self.model(
+                    encoding["input_ids"],
+                    encoding["attention_mask"],
+                )
+                probs = torch.softmax(outputs.logits, dim=-1)
+                return probs.numpy()
+        
+        # Create simple background
+        background = texts[:min(20, len(texts))]
+        
+        explainer = shap.Explainer(predict_proba, background)
+        shap_values = explainer(texts[:5])  # Sample for speed
+        
+        # Aggregate importance
+        importance = {}
+        for i, feature in enumerate(feature_names):
+            importance[feature] = np.mean(
+                np.abs(shap_values.values[:, :, i])
+            )
+        
+        return importance
+    
+    def analyze_sentence(
+        self,
+        text: str,
+    ) -> Dict[str, Any]:
+        """Analyze a single sentence with SHAP.
+        
+        Args:
+            text: Myanmar text
+        
+        Returns:
+            Analysis results including word importance
+        """
+        import shap
+        
+        encoding = self.tokenizer(
+            text,
+            padding=True,
+            truncation=True,
+            max_length=128,
+            return_tensors="pt",
+        )
+        
+        explainer = shap.Explainer(
+            lambda x: self._predict_batch(x),
+            self.tokenizer,
+        )
+        
+        explanation = explainer([text])
+        
+        # Extract word-level importance
+        tokens = self.tokenizer.convert_ids_to_tokens(
+            encoding["input_ids"][0]
+        )
+        
+        result = {
+            "text": text,
+            "tokens": tokens,
+            "prediction": explanation.output_names[
+                np.argmax(explanation.values[0].mean(axis=1))
+            ],
+            "shap_values": explanation.values[0].tolist(),
+        }
+        
+        return result
+    
+    def _predict_batch(self, texts: List[str]) -> np.ndarray:
+        """Batch prediction for SHAP."""
+        encoding = self.tokenizer(
+            texts,
+            padding=True,
+            truncation=True,
+            max_length=128,
+            return_tensors="pt",
+        )
+        
+        with torch.no_grad():
+            outputs = self.model(
+                encoding["input_ids"],
+                encoding["attention_mask"],
+            )
+            probs = torch.softmax(outputs.logits, dim=-1)
+            return probs.numpy()
+
+
+def create_shap_explainer(
+    model,
+    tokenizer,
+    background_texts: Optional[List[str]] = None,
+) -> ThankingSHAPExplainer:
+    """Factory function to create SHAP explainer."""
+    explainer = ThankingSHAPExplainer(model, tokenizer)
+    
+    if background_texts:
+        explainer.fit_background(background_texts)
+    
+    return explainer
+
+
+if __name__ == "__main__":
+    print("ThankingSHAPExplainer loaded")
+    print("Use create_shap_explainer() to create an explainer")

xai/visualization.py

@@ -0,0 +1,360 @@
+"""Visualization utilities for XAI results."""
+
+import json
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Union
+
+import matplotlib.pyplot as plt
+import numpy as np
+
+try:
+    import shap
+    HAS_SHAP = True
+except ImportError:
+    HAS_SHAP = False
+
+try:
+    from IPython.display import HTML, display
+    HAS_IPYTHON = True
+except ImportError:
+    HAS_IPYTHON = False
+
+
+class XAIVisualizer:
+    """Visualize XAI results."""
+    
+    def __init__(
+        self,
+        class_names: Optional[List[str]] = None,
+        output_dir: str = "outputs/xai",
+    ):
+        self.class_names = class_names or [
+            "negative", "neutral", "positive", "sarcastic"
+        ]
+        self.output_dir = Path(output_dir)
+        self.output_dir.mkdir(parents=True, exist_ok=True)
+    
+    def plot_word_importance(
+        self,
+        words: List[str],
+        importance: List[float],
+        title: str = "Word Importance",
+        output_path: Optional[str] = None,
+        figsize: tuple = (10, 6),
+    ) -> plt.Figure:
+        """Plot word importance as horizontal bar chart.
+        
+        Args:
+            words: List of words/tokens
+            importance: Importance scores
+            title: Plot title
+            output_path: Path to save figure
+            figsize: Figure size
+        
+        Returns:
+            Matplotlib figure
+        """
+        fig, ax = plt.subplots(figsize=figsize)
+        
+        # Sort by absolute importance
+        sorted_pairs = sorted(
+            zip(words, importance),
+            key=lambda x: abs(x[1]),
+            reverse=True,
+        )
+        
+        sorted_words = [p[0] for p in sorted_pairs]
+        sorted_importance = [p[1] for p in sorted_pairs]
+        
+        # Color based on positive/negative
+        colors = ["green" if v > 0 else "red" for v in sorted_importance]
+        
+        ax.barh(sorted_words, sorted_importance, color=colors, alpha=0.7)
+        ax.axvline(x=0, color="black", linestyle="-", linewidth=0.5)
+        ax.set_xlabel("SHAP Value")
+        ax.set_title(title)
+        ax.invert_yaxis()
+        
+        plt.tight_layout()
+        
+        if output_path:
+            plt.savefig(output_path, dpi=150, bbox_inches="tight")
+        
+        return fig
+    
+    def plot_feature_importance(
+        self,
+        features: List[str],
+        importance: List[float],
+        title: str = "Feature Importance",
+        output_path: Optional[str] = None,
+        figsize: tuple = (10, 6),
+    ) -> plt.Figure:
+        """Plot feature importance bar chart.
+        
+        Args:
+            features: List of feature names
+            importance: Importance scores
+            title: Plot title
+            output_path: Path to save figure
+            figsize: Figure size
+        
+        Returns:
+            Matplotlib figure
+        """
+        fig, ax = plt.subplots(figsize=figsize)
+        
+        # Sort by importance
+        sorted_pairs = sorted(
+            zip(features, importance),
+            key=lambda x: x[1],
+            reverse=True,
+        )
+        
+        sorted_features = [p[0] for p in sorted_pairs]
+        sorted_importance = [p[1] for p in sorted_pairs]
+        
+        ax.barh(sorted_features, sorted_importance, color="steelblue", alpha=0.7)
+        ax.set_xlabel("Importance Score")
+        ax.set_title(title)
+        ax.invert_yaxis()
+        
+        plt.tight_layout()
+        
+        if output_path:
+            plt.savefig(output_path, dpi=150, bbox_inches="tight")
+        
+        return fig
+    
+    def plot_confidence_distribution(
+        self,
+        predictions: List[str],
+        confidences: List[float],
+        output_path: Optional[str] = None,
+        figsize: tuple = (10, 6),
+    ) -> plt.Figure:
+        """Plot distribution of prediction confidences.
+        
+        Args:
+            predictions: Predicted classes
+            confidences: Confidence scores
+            output_path: Path to save figure
+            figsize: Figure size
+        
+        Returns:
+            Matplotlib figure
+        """
+        fig, (ax1, ax2) = plt.subplots(1, 2, figsize=figsize)
+        
+        # Histogram
+        ax1.hist(confidences, bins=20, alpha=0.7, color="steelblue")
+        ax1.axvline(np.mean(confidences), color="red", linestyle="--", 
+                    label=f"Mean: {np.mean(confidences):.3f}")
+        ax1.set_xlabel("Confidence")
+        ax1.set_ylabel("Count")
+        ax1.set_title("Confidence Distribution")
+        ax1.legend()
+        
+        # By class
+        unique_classes = list(set(predictions))
+        class_confidences = {c: [] for c in unique_classes}
+        
+        for pred, conf in zip(predictions, confidences):
+            class_confidences[pred].append(conf)
+        
+        ax2.boxplot(
+            [class_confidences[c] for c in unique_classes],
+            labels=unique_classes,
+        )
+        ax2.set_xlabel("Class")
+        ax2.set_ylabel("Confidence")
+        ax2.set_title("Confidence by Class")
+        
+        plt.tight_layout()
+        
+        if output_path:
+            plt.savefig(output_path, dpi=150, bbox_inches="tight")
+        
+        return fig
+    
+    def plot_shap_summary(
+        self,
+        shap_values: np.ndarray,
+        features: np.ndarray,
+        feature_names: List[str],
+        output_path: Optional[str] = None,
+        figsize: tuple = (12, 8),
+    ) -> plt.Figure:
+        """Plot SHAP summary (beeswarm) plot.
+        
+        Args:
+            shap_values: SHAP values array
+            features: Feature values array
+            feature_names: Names of features
+            output_path: Path to save figure
+            figsize: Figure size
+        
+        Returns:
+            Matplotlib figure
+        """
+        if not HAS_SHAP:
+            raise ImportError("shap library required for this visualization")
+        
+        fig, ax = plt.subplots(figsize=figsize)
+        
+        shap.summary_plot(
+            shap_values,
+            features,
+            feature_names=feature_names,
+            show=False,
+        )
+        
+        plt.tight_layout()
+        
+        if output_path:
+            plt.savefig(output_path, dpi=150, bbox_inches="tight")
+        
+        return fig
+    
+    def plot_comparison(
+        self,
+        explanations: Dict[str, List[Tuple[str, float]]],
+        output_path: Optional[str] = None,
+        figsize: tuple = (12, 8),
+    ) -> plt.Figure:
+        """Compare explanations across different methods or samples.
+        
+        Args:
+            explanations: Dict mapping sample IDs to explanation tuples
+            output_path: Path to save figure
+            figsize: Figure size
+        
+        Returns:
+            Matplotlib figure
+        """
+        fig, ax = plt.subplots(figsize=figsize)
+        
+        # Get all unique words
+        all_words = set()
+        for exp in explanations.values():
+            for word, _ in exp:
+                all_words.add(word)
+        
+        all_words = list(all_words)[:20]  # Limit to top 20
+        
+        # Create matrix
+        matrix = []
+        for sample_id, exp in explanations.items():
+            exp_dict = dict(exp)
+            row = [exp_dict.get(w, 0) for w in all_words]
+            matrix.append(row)
+        
+        matrix = np.array(matrix)
+        
+        # Plot heatmap
+        im = ax.imshow(matrix, cmap="RdBu_r", aspect="auto")
+        ax.set_xticks(range(len(all_words)))
+        ax.set_xticklabels(all_words, rotation=45, ha="right")
+        ax.set_yticks(range(len(explanations)))
+        ax.set_yticklabels(list(explanations.keys()))
+        ax.set_title("Explanation Comparison")
+        
+        plt.colorbar(im, ax=ax, label="Importance")
+        
+        plt.tight_layout()
+        
+        if output_path:
+            plt.savefig(output_path, dpi=150, bbox_inches="tight")
+        
+        return fig
+    
+    def save_explanations(
+        self,
+        explanations: List[Dict[str, Any]],
+        output_path: str,
+    ) -> None:
+        """Save explanations to JSON file.
+        
+        Args:
+            explanations: List of explanation dictionaries
+            output_path: Path to save JSON
+        """
+        with open(output_path, "w", encoding="utf-8") as f:
+            json.dump(explanations, f, indent=2, ensure_ascii=False)
+    
+    def generate_html_report(
+        self,
+        explanations: List[Dict[str, Any]],
+        output_path: str,
+    ) -> None:
+        """Generate HTML report of explanations.
+        
+        Args:
+            explanations: List of explanation dictionaries
+            output_path: Path to save HTML
+        """
+        html = """
+        <!DOCTYPE html>
+        <html>
+        <head>
+            <title>XAI Explanation Report</title>
+            <style>
+                body { font-family: Arial, sans-serif; margin: 20px; }
+                .explanation { border: 1px solid #ccc; padding: 15px; margin: 10px 0; border-radius: 5px; }
+                .text { font-size: 18px; margin-bottom: 10px; }
+                .prediction { font-weight: bold; color: #2196F3; }
+                .word { display: inline-block; padding: 2px 5px; margin: 2px; border-radius: 3px; }
+                .positive { background-color: #c8e6c9; }
+                .negative { background-color: #ffcdd2; }
+                .neutral { background-color: #e0e0e0; }
+                table { border-collapse: collapse; width: 100%; }
+                th, td { border: 1px solid #ddd; padding: 8px; text-align: left; }
+                th { background-color: #f5f5f5; }
+            </style>
+        </head>
+        <body>
+            <h1>XAI Explanation Report</h1>
+            <p>Total explanations: """ + str(len(explanations)) + """</p>
+        """
+        
+        for i, exp in enumerate(explanations):
+            html += f"""
+            <div class="explanation">
+                <div class="text">{exp.get('text', 'N/A')}</div>
+                <div class="prediction">Predicted: {exp.get('predicted_class', 'N/A')}</div>
+                <div>
+            """
+            
+            for word, weight in exp.get("word_importance", []):
+                color_class = "positive" if weight > 0 else "negative"
+                html += f'<span class="word {color_class}">{word}: {weight:.3f}</span>'
+            
+            html += """
+                </div>
+            </div>
+            """
+        
+        html += """
+        </body>
+        </html>
+        """
+        
+        with open(output_path, "w", encoding="utf-8") as f:
+            f.write(html)
+
+
+def create_visualizer(
+    class_names: Optional[List[str]] = None,
+    output_dir: str = "outputs/xai",
+) -> XAIVisualizer:
+    """Factory function to create XAI visualizer."""
+    return XAIVisualizer(
+        class_names=class_names,
+        output_dir=output_dir,
+    )
+
+
+if __name__ == "__main__":
+    visualizer = create_visualizer()
+    print("XAIVisualizer loaded")
+    print(f"Available methods: plot_word_importance, plot_feature_importance, etc.")