train.py

import torch
from sklearn.svm import LinearSVC
from sklearn.pipeline import make_pipeline
from sklearn.feature_extraction.text import TfidfVectorizer
import joblib
from anchors import HINGLISH_EMOJI_ANCHORS
import json
import re

def preprocess(text):
    text = text.lower()
    text = re.sub(r'[^a-z\s]', '', text)
    return text

def train_classifier():
    X = []
    y = []
    
    print("Extracting training data...")
    for emoji, phrases in HINGLISH_EMOJI_ANCHORS.items():
        for phrase in phrases:
            X.append(preprocess(phrase))
            y.append(emoji)
                
    print(f"Extracted {len(X)} samples. Training TF-IDF + Linear SVC...")
    
    clf = make_pipeline(
        TfidfVectorizer(ngram_range=(1, 5), analyzer='char_wb', min_df=1, max_df=0.9), 
        LinearSVC(C=0.5, class_weight='balanced', max_iter=3000, random_state=42)
    )
    clf.fit(X, y)
    
    print(f"Training accuracy on anchor phrases: {clf.score(X, y) * 100:.2f}%")
    
    joblib.dump(clf, "classifier.joblib")
    print("Classifier saved to classifier.joblib")

if __name__ == "__main__":
    train_classifier()