Update app.py

app.py

@@ -10,8 +10,8 @@ import pandas as pd
 import plotly.express as px
 import time
 import difflib
-
 from typing import List, Union
+
 from langdetect import detect, LangDetectException
 from transformers import (
     AutoTokenizer,
@@ -20,7 +20,10 @@ from transformers import (
     BitsAndBytesConfig,
 )
 import evaluate
-from sacrebleu import corpus_bleu, sentence_bleu  # Doc vs. segment BLEU
+from sacrebleu import corpus_bleu, sentence_bleu
+
+# ────────── Page Config (MUST be first) ──────────
+st.set_page_config(page_title="🔤 Translate→Eval+", layout="wide")
 
 # ────────── Global CSS ──────────
 st.markdown("""
@@ -46,7 +49,6 @@ def bootstrap(
     fn, predictions: List[str], references: List[str], sources: List[str]=None,
     n_resamples: int = 200, seed: int = 42
 ) -> List[float]:
-    """Bootstrap metric fn over (predictions, references, [sources])."""
     random.seed(seed)
     scores = []
     N = len(predictions)
@@ -63,16 +65,12 @@ def bootstrap(
 
 # ────────── Model Manager ──────────
 class ModelManager:
-    """
-    Loads the best translation model (NLLB‐200 or M2M100), 
-    8-bit if GPU available; auto-detects src_lang; dynamic tgt_lang.
-    """
     def __init__(self, candidates=None, quantize=True, default_tgt=None):
         if quantize and not torch.cuda.is_available():
             logger.warning("CUDA unavailable; disabling 8-bit quantization")
             quantize = False
-        self.quantize   = quantize
-        self.candidates = candidates or [
+        self.quantize    = quantize
+        self.candidates  = candidates or [
             "facebook/nllb-200-distilled-600M",
             "facebook/m2m100_418M",
         ]
@@ -93,16 +91,13 @@ class ModelManager:
                         name, device_map="auto", quantization_config=bnb
                     )
                 else:
-                    mdl = AutoModelForSeq2SeqLM.from_pretrained(
-                        name, device_map="auto"
-                    )
+                    mdl = AutoModelForSeq2SeqLM.from_pretrained(name, device_map="auto")
                 pipe = pipeline("translation", model=mdl, tokenizer=tok)
                 self.model_name = name
                 self.tokenizer  = tok
                 self.model      = mdl
                 self.pipeline   = pipe
                 self.lang_codes = list(tok.lang_code_to_id.keys())
-                # pick default target if none
                 if not self.default_tgt:
                     tur = [c for c in self.lang_codes if c.lower().startswith("tr")]
                     if not tur:
@@ -115,12 +110,8 @@ class ModelManager:
                 last_err = e
         raise RuntimeError(f"No model loaded: {last_err}")
 
-    def translate(
-        self, text: Union[str, List[str]],
-        src_lang: str = None, tgt_lang: str = None
-    ):
+    def translate(self, text: Union[str, List[str]], src_lang: str=None, tgt_lang: str=None):
         tgt = tgt_lang or self.default_tgt
-        # auto-detect src
         if not src_lang:
             sample = text[0] if isinstance(text, list) else text
             try:
@@ -153,161 +144,57 @@ class ModelManager:
 
 # ────────── Evaluator ──────────
 class TranslationEvaluator:
-    """
-    Wraps BLEU (corpus), ChrF, TER, BERTScore, COMET (ref & ref-free), and provides CIs.
-    """
     def __init__(self):
-        # BLEU (corpus)
-        self.bleu = evaluate.load("bleu")
-        # ChrF :contentReference[oaicite:0]{index=0}
-        self.chrf = evaluate.load("chrf")
-        # TER :contentReference[oaicite:1]{index=1}
-        self.ter  = evaluate.load("ter")
-        # BERTScore
+        self.bleu      = evaluate.load("bleu")
+        self.chrf      = evaluate.load("chrf")
+        self.ter       = evaluate.load("ter")
         self.bertscore = evaluate.load("bertscore")
-        # COMET (ref-based)
         self.comet_ref = evaluate.load("comet", model_id="unbabel/comet-mqm-qe-da")
-        # COMET QE (ref-free) :contentReference[oaicite:2]{index=2}
         self.comet_qe  = evaluate.load("comet", model_id="unbabel/wmt20-comet-qe-da")
-        logger.info("Loaded BLEU, ChrF, TER, BERTScore, COMET (ref & QE)")
+        logger.info("Loaded BLEU, ChrF, TER, BERTScore, COMET")
 
-    def compute_metrics(
-        self,
-        sources: List[str],
-        references: List[str],
-        predictions: List[str],
-        metrics: List[str],
-        ci: bool = True
-    ) -> dict:
+    def compute_metrics(self, srcs, refs, hyps, metrics, ci=True):
         out = {}
-
-        # -- BLEU (document-level)
         if "BLEU_doc" in metrics:
-            doc_bleu = self.bleu.compute(
-                predictions=predictions,
-                references=[[r] for r in references]
-            )["bleu"]
-            out["BLEU_doc"] = float(doc_bleu)
-
-        # -- BLEU (segment-level avg)
+            out["BLEU_doc"] = float(self.bleu.compute(predictions=hyps, references=[[r] for r in refs])["bleu"])
         if "BLEU_seg" in metrics:
-            seg_scores = [
-                sentence_bleu([r], p).score
-                for p, r in zip(predictions, references)
-            ]
-            out["BLEU_seg"] = float(sum(seg_scores) / len(seg_scores))
-
-        # -- ChrF
+            segs = [sentence_bleu([r], p).score for p,r in zip(hyps, refs)]
+            out["BLEU_seg"] = float(sum(segs)/len(segs))
         if "ChrF" in metrics:
-            cf = self.chrf.compute(
-                predictions=predictions,
-                references=[[r] for r in references]
-            )["score"]
-            out["ChrF"] = float(cf)
-
-        # -- TER
+            out["ChrF"] = float(self.chrf.compute(predictions=hyps, references=[[r] for r in refs])["score"])
         if "TER" in metrics:
-            tr = self.ter.compute(
-                predictions=predictions,
-                references=[[r] for r in references],
-                normalized=True
-            )["score"]
-            out["TER"] = float(tr)
-
-        # -- BERTScore
+            out["TER"] = float(self.ter.compute(predictions=hyps, references=[[r] for r in refs], normalized=True)["score"])
         if "BERTScore" in metrics:
-            bs = self.bertscore.compute(
-                predictions=predictions,
-                references=references,
-                lang="xx"
-            )["f1"]
-            out["BERTScore"] = float(sum(bs) / len(bs)) if bs else 0.0
-
-        # -- BERTurk
+            bs = self.bertscore.compute(predictions=hyps, references=refs, lang="xx")["f1"]
+            out["BERTScore"] = float(sum(bs)/len(bs)) if bs else 0.0
         if "BERTurk" in metrics:
-            bt = self.bertscore.compute(
-                predictions=predictions,
-                references=references,
-                lang="tr"
-            )["f1"]
-            out["BERTurk"] = float(sum(bt) / len(bt)) if bt else 0.0
-
-        # -- COMET (ref-based)
+            bt = self.bertscore.compute(predictions=hyps, references=refs, lang="tr")["f1"]
+            out["BERTurk"] = float(sum(bt)/len(bt)) if bt else 0.0
         if "COMET" in metrics:
-            cr = self.comet_ref.compute(
-                srcs=sources, hyps=predictions, refs=references
-            ).get("scores", 0.0)
-            out["COMET"] = float(cr[0] if isinstance(cr, list) else cr)
-
-        # -- QE (ref-free)
+            sc = self.comet_ref.compute(srcs=srcs, hyps=hyps, refs=refs).get("scores",0.0)
+            out["COMET"] = float(sc[0] if isinstance(sc,list) else sc)
         if "QE" in metrics:
-            cq = self.comet_qe.compute(
-                srcs=sources, hyps=predictions
-            ).get("scores", 0.0)
-            out["QE"] = float(cq[0] if isinstance(cq, list) else cq)
-
-        # -- Bootstrap CIs
+            q = self.comet_qe.compute(srcs=srcs, hyps=hyps).get("scores",0.0)
+            out["QE"] = float(q[0] if isinstance(q,list) else q)
         if ci:
-            # BLEU_doc CI
             if "CI_BLEU_doc" in metrics:
-                bsamp = bootstrap(
-                    lambda ps, rs: self.bleu.compute(
-                        predictions=ps,
-                        references=[[r] for r in rs]
-                    )["bleu"],
-                    predictions, references
-                )
-                out["CI_BLEU_doc"] = (
-                    float(np.percentile(bsamp, 2.5)),
-                    float(np.percentile(bsamp, 97.5))
-                )
-            # BERTScore CI
+                bsamp = bootstrap(lambda ps,rs: self.bleu.compute(predictions=ps,references=[[r] for r in rs])["bleu"], hyps, refs)
+                out["CI_BLEU_doc"] = (float(np.percentile(bsamp,2.5)), float(np.percentile(bsamp,97.5)))
             if "CI_BERTScore" in metrics:
-                bsamp = bootstrap(
-                    lambda ps, rs: sum(
-                        self.bertscore.compute(
-                            predictions=ps, references=rs, lang="xx"
-                        )["f1"]
-                    ) / len(ps),
-                    predictions, references
-                )
-                out["CI_BERTScore"] = (
-                    float(np.percentile(bsamp, 2.5)),
-                    float(np.percentile(bsamp, 97.5))
-                )
-            # COMET CI
+                bsamp = bootstrap(lambda ps,rs: sum(self.bertscore.compute(predictions=ps,references=rs,lang="xx")["f1"])/len(ps), hyps, refs)
+                out["CI_BERTScore"] = (float(np.percentile(bsamp,2.5)), float(np.percentile(bsamp,97.5)))
             if "CI_COMET" in metrics:
-                bsamp = bootstrap(
-                    lambda ps, rs, ss: float(
-                        self.comet_ref.compute(
-                            srcs=ss, hyps=ps, refs=rs
-                        ).get("scores", [0.0])[0]
-                    ),
-                    predictions, references, sources
-                )
-                out["CI_COMET"] = (
-                    float(np.percentile(bsamp, 2.5)),
-                    float(np.percentile(bsamp, 97.5))
-                )
-
+                bsamp = bootstrap(lambda ps,rs,ss: float(self.comet_ref.compute(srcs=ss,hyps=ps,refs=rs).get("scores",[0.0])[0]), hyps, refs, srcs)
+                out["CI_COMET"] = (float(np.percentile(bsamp,2.5)), float(np.percentile(bsamp,97.5)))
         return out
 
 # ────────── Error Categorizer ──────────
 class ErrorCategorizer:
-    """
-    Optional: classify error types via a fine-tuned text-classification model.
-    Supply your own HF model name for real categories.
-    """
-    def __init__(self, model_name: str = None):
-        if model_name:
-            self.pipe = pipeline("text-classification", model=model_name, device=0 if torch.cuda.is_available() else -1)
-        else:
-            self.pipe = None
-
-    def categorize(self, src: str, hyp: str):
-        if not self.pipe: 
-            return []
-        inp = f"SRC: {src}\nHYP: {hyp}\nError types (pick from taxonomy):"
+    def __init__(self, model_name=None):
+        self.pipe = pipeline("text-classification", model=model_name, device=0 if torch.cuda.is_available() else -1) if model_name else None
+    def categorize(self, src, hyp):
+        if not self.pipe: return []
+        inp = f"SRC: {src}\nHYP: {hyp}\nError types:"
         return self.pipe(inp, top_k=None)
 
 # ────────── Streamlit App ──────────
@@ -315,8 +202,7 @@ class ErrorCategorizer:
 def load_resources():
     mgr = ModelManager(quantize=True)
     ev  = TranslationEvaluator()
-    # set your error-classifier HF model here, or None to disable
-    err = ErrorCategorizer(model_name="your-org/translation-error-categorizer")
+    err = ErrorCategorizer(model_name=None)  # set your HF model here
     return mgr, ev, err
 
 def display_model_info(info: dict):
@@ -326,43 +212,29 @@ def display_model_info(info: dict):
     st.sidebar.write(f"• **Device:** {info['device']}")
     st.sidebar.write(f"• **Default tgt:** {info['default_tgt']}")
 
-def show_diff(ref: str, hyp: str):
+def show_diff(ref, hyp):
     differ = difflib.HtmlDiff(tabsize=4, wrapcolumn=60)
-    html = differ.make_table(
-        ref.split(), hyp.split(),
-        fromdesc="Reference", todesc="Hypothesis",
-        context=True, numlines=1
-    )
+    html = differ.make_table(ref.split(), hyp.split(), fromdesc="Reference", todesc="Hypothesis", context=True, numlines=1)
     components.html(html, height=200, scrolling=True)
 
 def main():
-    st.set_page_config(page_title="🔤 Translate→Eval+", layout="wide")
-    st.title("🌐 Translate → 🔠 Evaluate & Analyze")
-    st.write("Translate from any language, choose target, eval with advanced metrics, and inspect errors.")
+    # Note: set_page_config has been moved to the top!
+    st.title("🌐 Translate → Evaluate & Analyze")
+    st.write("Translate any language, choose target, eval with advanced metrics, and inspect errors.")
 
-    # Sidebar
     with st.sidebar:
         st.header("Settings")
         mgr, ev, err = load_resources()
         info = mgr.get_info()
         display_model_info(info)
 
-        tgt = st.selectbox(
-            "Target language", info["langs"],
-            index=info["langs"].index(info["default_tgt"])
-        )
-
-        metric_opts = [
-            "BLEU_doc","BLEU_seg","ChrF","TER",
-            "BERTScore","BERTurk","COMET","QE",
-            "CI_BLEU_doc","CI_BERTScore","CI_COMET"
-        ]
+        tgt = st.selectbox("Target language", info["langs"], index=info["langs"].index(info["default_tgt"]))
+        metric_opts = ["BLEU_doc","BLEU_seg","ChrF","TER","BERTScore","BERTurk","COMET","QE","CI_BLEU_doc","CI_BERTScore","CI_COMET"]
         metrics = st.multiselect("Metrics & CIs", metric_opts, default=["BLEU_doc","BERTScore","COMET"])
         batch_size = st.slider("Batch size", 1, 32, 8)
 
     tab1, tab2 = st.tabs(["Single","Batch CSV"])
 
-    # ────────── Single Sentence ──────────
     with tab1:
         src = st.text_area("Source text:", height=120)
         ref = st.text_area("Gold reference (optional):", height=80)
@@ -372,32 +244,25 @@ def main():
                 hyp = out[0]["translation_text"]
             st.markdown(f"**Hypothesis ({tgt}):**  {hyp}")
 
-            # metrics
             scores = ev.compute_metrics([src],[ref or ""],[hyp], metrics)
-            # display
             sd = {}
             for m in metrics:
                 v = scores.get(m)
-                if m.startswith("CI_"):
-                    low, high = v
-                    sd[m] = f"{low:.3f} – {high:.3f}"
+                if m.startswith("CI_") and v:
+                    sd[m] = f"{v[0]:.3f} – {v[1]:.3f}"
                 else:
                     sd[m] = f"{v:.4f}" if v is not None else "N/A"
             st.markdown("### Scores")
             st.table(pd.DataFrame([sd]))
 
-            # diff
             if ref.strip():
                 st.markdown("### Diff View")
                 show_diff(ref, hyp)
-
-            # error categories
             cats = err.categorize(src, hyp)
             if cats:
                 st.markdown("### Error Categories")
                 st.json(cats)
 
-    # ────────── Batch CSV ──────────
     with tab2:
         uploaded = st.file_uploader("Upload CSV with `src`,`ref_tr`", type=["csv"])
         if uploaded:
@@ -410,36 +275,34 @@ def main():
                     prog = st.progress(0)
                     N = len(df)
                     for i in range(0, N, batch_size):
-                        batch = df.iloc[i : i+batch_size]
+                        batch = df.iloc[i:i+batch_size]
                         srcs, refs = batch["src"].tolist(), batch["ref_tr"].tolist()
                         outs = mgr.translate(srcs, tgt_lang=tgt)
                         hyps = [o["translation_text"] for o in outs]
-                        for s, r, h in zip(srcs, refs, hyps):
-                            base = {"src":s, "ref_tr":r, "hyp_tr":h}
+                        for s,r,h in zip(srcs,refs,hyps):
+                            base = {"src":s,"ref_tr":r,"hyp_tr":h}
                             if r.strip():
                                 sc = ev.compute_metrics([s],[r],[h], metrics)
                                 for m in metrics:
-                                    if m.startswith("CI_"):
+                                    if m.startswith("CI_") and sc.get(m):
                                         low, high = sc[m]
                                         base[m] = f"{low:.3f}–{high:.3f}"
                                     else:
-                                        base[m] = sc[m]
+                                        base[m] = sc.get(m)
                             else:
                                 for m in metrics:
                                     base[m] = None
                             all_rows.append(base)
-                        prog.progress(min(i+batch_size, N)/N)
+                        prog.progress(min(i+batch_size,N)/N)
                     res_df = pd.DataFrame(all_rows)
 
                 st.markdown("### Results")
                 st.dataframe(res_df, use_container_width=True)
-
-                # histograms
                 for m in metrics:
                     st.markdown(f"#### {m} Distribution")
                     col = pd.to_numeric(res_df[m], errors="coerce").dropna()
                     if col.empty:
-                        st.write("No valid data for this metric.")
+                        st.write("No valid data.")
                     else:
                         fig = px.histogram(col, x=col)
                         st.plotly_chart(fig, use_container_width=True)