import os, re, json, time, warnings, subprocess, signal warnings.filterwarnings("ignore") import numpy as np import gradio as gr import matplotlib matplotlib.use("Agg") import matplotlib.pyplot as plt import torch import torch.nn.functional as F from transformers import AutoTokenizer, AutoModelForSequenceClassification from huggingface_hub import hf_hub_download print("APP STARTED") # ── Config ──────────────────────────────────────────────────────────────────── MODEL_PATH = "" HF_MODEL_REPO = "Jaykumardas/Multilingual_News_Model" # ── Load model ──────────────────────────────────────────────────────────────── def load_model_and_labels(): model_source = HF_MODEL_REPO if HF_MODEL_REPO else MODEL_PATH print(f"[INFO] Loading from: {model_source}") try: tokenizer = AutoTokenizer.from_pretrained(model_source) print("[INFO] Tokenizer loaded OK") except Exception as e: raise RuntimeError(f"Tokenizer load failed: {e}") id2label = None lmap = os.path.join(model_source, "label_map.json") try: lmap_path = hf_hub_download( repo_id=model_source, filename="label_map.json" ) with open(lmap_path, encoding="utf-8") as f: lm = json.load(f) id2label = {int(k): v for k, v in lm["id2label"].items()} print(f"[INFO] id2label loaded from HF: {id2label}") except Exception as e: print(f"[WARN] label_map.json not found in HF repo: {e}") if id2label is None: cfg_path = os.path.join(model_source, "config.json") if os.path.isfile(cfg_path): with open(cfg_path, encoding="utf-8") as f: cfg = json.load(f) if cfg.get("id2label"): id2label = {int(k): v for k, v in cfg["id2label"].items()} print(f"[INFO] id2label from config.json: {id2label}") if id2label is None: raise RuntimeError("label_map.json not found. Re-run your save cell in Kaggle.") try: model = AutoModelForSequenceClassification.from_pretrained( model_source, num_labels=len(id2label), ignore_mismatched_sizes=True) device = "cuda" if torch.cuda.is_available() else "cpu" model.to(device).eval() print(f"[INFO] Model OK — {len(id2label)} classes — {device.upper()}") except Exception as e: raise RuntimeError(f"Model load failed: {e}") return model, tokenizer, id2label, device try: MODEL, TOKENIZER, ID2LABEL, DEVICE = load_model_and_labels() CLASS_NAMES = [ID2LABEL[i] for i in sorted(ID2LABEL)] NUM_CLASSES = len(CLASS_NAMES) MODEL_LOADED = True print(f"[INFO] Classes: {CLASS_NAMES}") except Exception as e: print(f"[ERROR] {e}") MODEL_LOADED = False CLASS_NAMES = ["Model not loaded"] NUM_CLASSES = 1 ID2LABEL = {0: "Model not loaded"} DEVICE = "cpu" # ── Icons / metrics / samples ───────────────────────────────────────────────── ICONS = { "entertainment":"🎬","sports":"🏏","state":"🗺️","national":"🇮🇳", "international":"🌏","business":"📈","technology":"💻","science":"🔬", "health":"🏥","politics":"🏛️", } ICONS.update({k.title(): v for k, v in list(ICONS.items())}) def get_icon(label): return ICONS.get(label, "📰") REAL_METRICS = { "TF-IDF + LR": {"test_acc":83.84,"test_f1":77.85,"color":"#3b82f6","train_time":"< 2 min"}, "BiLSTM": {"test_acc":79.36,"test_f1":67.16,"color":"#8b5cf6","train_time":"~14 min"}, "XLM-RoBERTa": {"test_acc":86.12,"test_f1":78.75,"color":"#10b981","train_time":"~45 min"}, } SAMPLES = { "Telugu": "హైదరాబాద్లో క్రికెట్ టోర్నమెంట్ ప్రారంభమైంది; జిల్లా స్థాయి జట్లు పాల్గొంటున్నాయి.", "Malayalam":"కേരളത്തിൽ ഇന്ന് കനത്ത മഴ; ഒൻപത് ജില്ലകളിൽ യെല്ലോ അലർട്ട് പ്രഖ്യാപിച്ചു.", "Marathi": "मुंबई शेअर बाजारात आज मोठी तेजी; सेन्सेक्स ५०० अंकांनी वधारला.", "Tamil": "தமிழ்நாட்டில் புதிய தொழில்நுட்ப பூங்கா திறப்பு; ஆயிரக்கணக்கான வேலை வாய்ப்புகள்.", "Gujarati": "ગુજરાત ટીમ સ્ટેટ ક્રિકેટ ચેમ્પિયનશિપ જીતી; ખેલાડીઓ ઉત્સાહિત.", } # ── Preprocessing ───────────────────────────────────────────────────────────── def clean_text(text): if not isinstance(text, str): return "" text = re.sub(r"https?://\S+|www\.\S+", " ", text) text = re.sub(r"<[^>]+>", " ", text) text = re.sub(r"[\u200b\u200c\u200d\ufeff\u00ad]", "", text) text = re.sub( r"[^\w\s\u0900-\u097F\u0C00-\u0C7F\u0D00-\u0D7F\u0B80-\u0BFF\u0A80-\u0AFF]", " ", text) return re.sub(r"\s+", " ", text).strip() # ── Inference ───────────────────────────────────────────────────────────────── def predict_text(text): if not MODEL_LOADED: return {c: 0.0 for c in CLASS_NAMES}, "Model not loaded", 0.0, 0 t_clean = clean_text(text) if not t_clean: return {c: 0.0 for c in CLASS_NAMES}, "Empty input", 0.0, 0 enc = TOKENIZER(t_clean, max_length=128, padding="max_length", truncation=True, return_tensors="pt") enc = {k: v.to(DEVICE) for k, v in enc.items()} t0 = time.time() with torch.no_grad(): logits = MODEL(**enc).logits ms = int((time.time() - t0) * 1000) probs = F.softmax(logits, dim=-1).squeeze().cpu().numpy() idx = int(np.argmax(probs)) label = ID2LABEL.get(idx, f"class_{idx}") return ({ID2LABEL.get(i, f"class_{i}"): float(probs[i]) for i in range(len(probs))}, label, float(probs[idx]), ms) # ── Charts ──────────────────────────────────────────────────────────────────── def conf_chart(probs_dict, pred_label): paired = sorted(zip(probs_dict.values(), probs_dict.keys()), reverse=True) vals = [p[0]*100 for p in paired] labs = [p[1] for p in paired] colors = ["#10b981" if l == pred_label else "#6366f1" if v > 10 else "#334155" for l, v in zip(labs, vals)] fig, ax = plt.subplots(figsize=(9, max(4, len(labs)*0.5+1))) fig.patch.set_facecolor("#0f172a"); ax.set_facecolor("#0f172a") bars = ax.barh(labs[::-1], vals[::-1], color=colors[::-1], height=0.55, edgecolor="none") for bar, v in zip(bars, vals[::-1]): ax.text(bar.get_width()+0.5, bar.get_y()+bar.get_height()/2, f"{v:.1f}%", va="center", ha="left", color="#e2e8f0", fontsize=10, fontweight="bold") ax.set_xlim(0, 115) ax.set_xlabel("Confidence (%)", color="#94a3b8", fontsize=11) ax.set_title("Prediction Confidence", color="#f1f5f9", fontsize=13, fontweight="bold", pad=12) ax.tick_params(colors="#94a3b8", labelsize=10) for s in ax.spines.values(): s.set_visible(False) ax.grid(axis="x", color="#1e293b", linewidth=0.8) plt.tight_layout(pad=1.5) return fig def metrics_chart(): models = list(REAL_METRICS.keys()) accs = [REAL_METRICS[m]["test_acc"] for m in models] f1s = [REAL_METRICS[m]["test_f1"] for m in models] cols = [REAL_METRICS[m]["color"] for m in models] x, w = np.arange(len(models)), 0.32 fig, ax = plt.subplots(figsize=(10, 5)) fig.patch.set_facecolor("#0f172a"); ax.set_facecolor("#0f172a") b1 = ax.bar(x-w/2, accs, w, label="Test Accuracy (%)", color=[c+"cc" for c in cols], edgecolor="none") b2 = ax.bar(x+w/2, f1s, w, label="Test F1 Macro (%)", color=cols, edgecolor="none", alpha=0.75) for bars in [b1, b2]: for bar in bars: h = bar.get_height() ax.text(bar.get_x()+bar.get_width()/2, h+0.5, f"{h:.1f}", ha="center", va="bottom", color="#e2e8f0", fontsize=10, fontweight="bold") ax.set_xticks(x); ax.set_xticklabels(models, color="#94a3b8", fontsize=11) ax.set_ylim(0, 105); ax.set_ylabel("Score (%)", color="#94a3b8", fontsize=11) ax.set_title("Model Comparison — Test Results", color="#f1f5f9", fontsize=13, fontweight="bold", pad=14) ax.tick_params(colors="#94a3b8") ax.legend(facecolor="#1e293b", edgecolor="none", labelcolor="#e2e8f0") for s in ax.spines.values(): s.set_visible(False) ax.grid(axis="y", color="#1e293b", linewidth=0.8) plt.tight_layout(pad=1.5) return fig _METRICS_FIG = metrics_chart() # pre-render once # ── Gradio handlers ─────────────────────────────────────────────────────────── def classify_single(text): if not text or not text.strip(): return '
Please enter a headline.
', None, None pd, label, conf, ms = predict_text(text) icon = get_icon(label) pct = conf * 100 cc = "#10b981" if pct >= 70 else "#f59e0b" if pct >= 40 else "#ef4444" html = f"""Enter at least one headline.
', None lines = [l.strip() for l in batch_text.strip().split("\n") if l.strip()][:50] rows = "" labels_list = [] for i, line in enumerate(lines, 1): pd, label, conf, _ = predict_text(line) icon = get_icon(label); pct = conf*100 cc = "#10b981" if pct >= 70 else "#f59e0b" if pct >= 40 else "#ef4444" prev = (line[:80]+"…") if len(line) > 80 else line labels_list.append(label) rows += f"""| # | Headline | Category | Conf. |
|---|
Chaitanya Bharathi Institute of Technology · Dept. of AI & ML
A single unified model that reads Telugu, Malayalam, Marathi, Tamil, and Gujarati natively, classifying news headlines into up to 10 categories — no translation required.
iNLTK Headlines subsets — 37,069 labeled headlines across 5 languages.
Split: Train 25,945 · Val 3,707 · Test 7,414
TF-IDF + LR: 83.84% · F1 77.85%
BiLSTM: 79.36% · F1 67.16%
XLM-RoBERTa: 86.% · F1 78.75%
IndicGLUE data loading, Unicode-safe preprocessing, TF-IDF baseline (84.95%), EDA.
BiLSTM design (60k vocab, GlobalMaxPool), training curves, per-class evaluation (79.36%).
XLM-RoBERTa fine-tuning, full evaluation, Gradio UI deployment (86.12%).