Update app.py

app.py

@@ -7,6 +7,9 @@ import numpy as np
 import pandas as pd
 from transformers import AutoTokenizer, AutoModelForTokenClassification
 
+# ----------------------------
+# Config
+# ----------------------------
 MODEL_ID = "Setur/BRAGD"
 TAGS_FILEPATH = "Sosialurin-BRAGD_tags.csv"   # must match model labels
 LABELS_FILEPATH = "tag_labels.json"           # add to repo root (FO+EN labels)
@@ -28,18 +31,12 @@ GROUP_ORDER = ["subcategory","gender","number","case","article","proper","degree
 # You said Subcategory B doesn't exist and will be deleted from the CSV:
 HIDE_CODES = {"subcategory": {"B"}}
 
-GROUP_TITLES = {
-    "en": {"subcategory":"Subcategory","gender":"Gender","number":"Number","case":"Case","article":"Article suffix","proper":"Proper noun",
-           "degree":"Degree","declension":"Declension","mood":"Mood","voice":"Voice","tense":"Tense","person":"Person","definiteness":"Definiteness"},
-    "fo": {"subcategory":"Undirflokkur","gender":"Kyn","number":"Tal","case":"Fall","article":"Bundið eftirlið","proper":"Sernavn",
-           "degree":"Stig","declension":"Bending","mood":"Háttur","voice":"Søgn","tense":"Tíð","person":"Persónur","definiteness":"Bundni/óbundni"},
-}
-
 UI = {
-    "fo": {"w":"Orð","t":"Mark","s":"Vís sum","m":"Merking","def":"bundið","ind":"óbundið"},
-    "en": {"w":"Word","t":"Tag","s":"Show as","m":"Meaning","def":"definite","ind":"indefinite"},
+    "fo": {"w":"Orð", "t":"Mark", "s":"Vís sum", "m":"Merking"},
+    "en": {"w":"Word","t":"Tag", "s":"Show as", "m":"Meaning"},
 }
 
+# Theme color: #89AFA9 (+ close shades)
 CSS = """
 :root{
   --primary-500:#89AFA9; --primary-600:#6F9992; --primary-700:#5B7F79;
@@ -92,7 +89,9 @@ def process_tag_features(tag_to_features: dict, intervals):
     wt_masks = {wt:[a for a in arrs if a[wt]==1] for wt in range(15)}
     out = {}
     for wt,labels in wt_masks.items():
-        if not labels: out[wt]=[]; continue
+        if not labels:
+            out[wt]=[]
+            continue
         sum_labels = np.sum(np.array(labels), axis=0)
         out[wt] = [iv for iv in intervals if np.sum(sum_labels[iv[0]:iv[1]+1]) != 0]
     return out
@@ -118,7 +117,9 @@ def predict_vectors(logits, attention_mask, begin_tokens, dict_intervals, vec_le
         vectors.append(vec)
     return vectors
 
+# ----------------------------
 # Load labels (extracted from your XLSX)
+# ----------------------------
 with open(LABELS_FILEPATH, "r", encoding="utf-8") as f:
     LABELS = json.load(f)
 
@@ -130,10 +131,14 @@ def label_for(lang: str, group: str, wc: str, code: str) -> str:
         return by_wc[wc][group][code]
     return glob.get(group, {}).get(code, "")
 
+# ----------------------------
 # Load CSV mappings (authoritative)
+# ----------------------------
 tag_to_features, features_to_tag, VEC_LEN, FEATURE_COLS = load_tag_mappings(TAGS_FILEPATH)
 
+# ----------------------------
 # Load model
+# ----------------------------
 tokenizer = AutoTokenizer.from_pretrained(MODEL_ID, token=HF_TOKEN)
 model = AutoModelForTokenClassification.from_pretrained(MODEL_ID, token=HF_TOKEN)
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
@@ -170,38 +175,65 @@ def group_code(vec: torch.Tensor, group: str) -> str:
             return code
     return ""
 
-def describe(vec: torch.Tensor, lang: str) -> str:
+def clean_label(s: str) -> str:
+    s = (s or "").strip()
+    s = re.sub(r"\s+", " ", s)
+    s = s.strip(" -;,:")
+    return s
+
+def visible_summary(vec: torch.Tensor, lang: str) -> str:
+    """
+    Vís sum / Show as:
+    - ONLY words/labels, no letters, no hyphens like "X –"
+    - all selected features (stable order)
+    """
+    lang = "fo" if lang=="fo" else "en"
+    raw_tag = vector_to_tag(vec)
     wc = wc_code(vec)
-    parts = []
-    if wc:
-        lbl = label_for(lang, "word_class", wc, wc)
-        parts.append(f"{wc} – {lbl}" if lbl else wc)
+
+    wc_lbl = label_for(lang, "word_class", wc, wc) or wc
+
+    # Exact override requested earlier:
+    if raw_tag == "DGd":
+        wc_lbl = "Fyriseting" if lang=="fo" else "Preposition"
+
+    labels = [clean_label(wc_lbl)]
+
     for g in GROUP_ORDER:
         c = group_code(vec, g)
         if not c:
             continue
-        lbl = label_for(lang, g, wc, c) or label_for(lang, g, "", c)
-        parts.append(f"{c} – {lbl}" if lbl else c)
-    return "; ".join(parts)
 
-def show_as(vec: torch.Tensor, lang: str) -> str:
+        lbl = label_for(lang, g, wc, c) or label_for(lang, g, "", c) or ""
+        lbl = clean_label(lbl)
+        if not lbl:
+            continue
+
+        if lbl not in labels:
+            labels.append(lbl)
+
+    return ", ".join([l for l in labels if l])
+
+def meaning_detail(vec: torch.Tensor, lang: str) -> str:
+    """
+    Merking / Meaning:
+    keeps codes + labels (useful for debugging and linguists)
+    """
     lang = "fo" if lang=="fo" else "en"
     wc = wc_code(vec)
-    wc_lbl = label_for(lang, "word_class", wc, wc) or wc
-    raw = vector_to_tag(vec)
+    parts = []
 
-    # Exact override you requested:
-    if raw == "DGd":
-        return "Fyriseting" if lang=="fo" else "Preposition"
+    wc_lbl = label_for(lang, "word_class", wc, wc)
+    parts.append(f"{wc} – {wc_lbl}" if wc_lbl else wc)
 
-    # S...a. / S...A. mapping (nouns): show definite/indefinite by Article (A/a)
-    if wc == "S":
-        art = group_code(vec, "article")  # A or a
-        if art == "A": return f"{wc_lbl} — {UI[lang]['def']}"
-        if art == "a": return f"{wc_lbl} — {UI[lang]['ind']}"
-        return wc_lbl
+    for g in GROUP_ORDER:
+        c = group_code(vec, g)
+        if not c:
+            continue
+        lbl = label_for(lang, g, wc, c) or label_for(lang, g, "", c)
+        parts.append(f"{c} – {lbl}" if lbl else c)
 
-    return wc_lbl
+    return "; ".join([p for p in parts if p])
 
 def compute_codes_by_wc():
     codes = defaultdict(lambda: defaultdict(set))  # wc -> group -> set(code)
@@ -229,8 +261,15 @@ def compute_codes_by_wc():
 CODES_BY_WC = compute_codes_by_wc()
 
 def build_legend(lang: str) -> str:
+    """
+    Elaborate overview:
+    Under each word class, show the letter codes actually used in the CURRENT CSV,
+    with labels from tag_labels.json (fallback to code if missing).
+    """
     lang = "fo" if lang=="fo" else "en"
-    lines = ["### Markingaryvirlit / Legend", ""]
+    title = "### Markingaryvirlit" if lang=="fo" else "### Tag legend"
+    lines = [title, ""]
+
     for wc in sorted(CODES_BY_WC.keys()):
         wcl = label_for(lang, "word_class", wc, wc) or ""
         lines.append(f"#### {wc} — {wcl}" if wcl else f"#### {wc}")
@@ -239,12 +278,48 @@ def build_legend(lang: str) -> str:
             cs = sorted(CODES_BY_WC[wc].get(g, set()))
             if not cs:
                 continue
-            lines.append(f"**{GROUP_TITLES[lang].get(g, g)}**")
+
+            if lang=="fo":
+                group_name = {
+                    "subcategory":"Undirflokkur",
+                    "gender":"Kyn",
+                    "number":"Tal",
+                    "case":"Fall",
+                    "article":"Bundni/óbundni",
+                    "proper":"Sernavn",
+                    "degree":"Stig",
+                    "declension":"Bending",
+                    "mood":"Háttur",
+                    "voice":"Søgn",
+                    "tense":"Tíð",
+                    "person":"Persónur",
+                    "definiteness":"Bundni/óbundni",
+                }.get(g, g)
+            else:
+                group_name = {
+                    "subcategory":"Subcategory",
+                    "gender":"Gender",
+                    "number":"Number",
+                    "case":"Case",
+                    "article":"Definite suffix",
+                    "proper":"Proper noun",
+                    "degree":"Degree",
+                    "declension":"Declension",
+                    "mood":"Mood",
+                    "voice":"Voice",
+                    "tense":"Tense",
+                    "person":"Person",
+                    "definiteness":"Definiteness",
+                }.get(g, g)
+
+            lines.append(f"**{group_name}**")
             for c in cs:
                 lbl = label_for(lang, g, wc, c) or label_for(lang, g, "", c)
                 lines.append(f"- `{c}` — {lbl}" if lbl else f"- `{c}`")
             lines.append("")
+
         lines.append("")
+
     return "\n".join(lines).strip()
 
 def run_model(sentence: str):
@@ -255,8 +330,16 @@ def run_model(sentence: str):
     if not tokens:
         return []
 
-    enc = tokenizer(tokens, is_split_into_words=True, add_special_tokens=True, max_length=128,
-                    padding="max_length", truncation=True, return_attention_mask=True, return_tensors="pt")
+    enc = tokenizer(
+        tokens,
+        is_split_into_words=True,
+        add_special_tokens=True,
+        max_length=128,
+        padding="max_length",
+        truncation=True,
+        return_attention_mask=True,
+        return_tensors="pt",
+    )
 
     input_ids = enc["input_ids"].to(device)
     attention_mask = enc["attention_mask"].to(device)
@@ -265,9 +348,12 @@ def run_model(sentence: str):
     begin = []
     last = None
     for wid in word_ids:
-        if wid is None: begin.append(0)
-        elif wid != last: begin.append(1)
-        else: begin.append(0)
+        if wid is None:
+            begin.append(0)
+        elif wid != last:
+            begin.append(1)
+        else:
+            begin.append(0)
         last = wid
 
     with torch.no_grad():
@@ -288,18 +374,27 @@ def run_model(sentence: str):
         vec_i += 1
     return rows
 
-def render(rows_state, lang: str):
+def render_main(rows_state, lang: str):
     lang = "fo" if lang=="fo" else "en"
-    cols = [UI[lang]["w"], UI[lang]["t"], UI[lang]["s"], UI[lang]["m"]]
+    cols = [UI[lang]["w"], UI[lang]["t"], UI[lang]["s"]]
     if not rows_state:
-        return pd.DataFrame(columns=cols), build_legend(lang)
+        return pd.DataFrame(columns=cols), build_legend(lang), pd.DataFrame(columns=[UI[lang]["w"], UI[lang]["t"], UI[lang]["m"]])
 
-    out = []
+    out_main = []
+    out_mean = []
     for r in rows_state:
         vec = torch.tensor(r["vec"])
-        out.append([r["word"], vector_to_tag(vec), show_as(vec, lang), describe(vec, lang)])
-    return pd.DataFrame(out, columns=cols), build_legend(lang)
+        tag = vector_to_tag(vec)
+        out_main.append([r["word"], tag, visible_summary(vec, lang)])
+        out_mean.append([r["word"], tag, meaning_detail(vec, lang)])
 
+    df_main = pd.DataFrame(out_main, columns=cols)
+    df_mean = pd.DataFrame(out_mean, columns=[UI[lang]["w"], UI[lang]["t"], UI[lang]["m"]])
+    return df_main, build_legend(lang), df_mean
+
+# ----------------------------
+# Gradio UI
+# ----------------------------
 theme = gr.themes.Soft()
 
 with gr.Blocks(theme=theme, css=CSS, title="BRAGD-markarin") as demo:
@@ -309,25 +404,29 @@ with gr.Blocks(theme=theme, css=CSS, title="BRAGD-markarin") as demo:
     btn = gr.Button("Marka / Tag", variant="primary")
 
     state = gr.State([])
+
     out_df = gr.Dataframe(wrap=True, interactive=False, label="Úrslit / Results")
 
     # Under results + can be changed AFTER tagging (no rerun; just re-render)
     lang = gr.Dropdown(choices=[("Føroyskt","fo"), ("English","en")], value="fo", label="Mál / Language")
 
+    with gr.Accordion("Merking / Meaning", open=False):
+        out_mean_df = gr.Dataframe(wrap=True, interactive=False, label="")
+
     with gr.Accordion("Markingaryvirlit / Legend", open=False):
         legend_md = gr.Markdown(build_legend("fo"))
 
     def on_tag(sentence, lang_choice):
         rows = run_model(sentence)
-        df, legend = render(rows, lang_choice)
-        return rows, df, legend
+        df_main, legend, df_mean = render_main(rows, lang_choice)
+        return rows, df_main, legend, df_mean
 
     def on_lang(rows, lang_choice):
-        df, legend = render(rows, lang_choice)
-        return df, legend
+        df_main, legend, df_mean = render_main(rows, lang_choice)
+        return df_main, legend, df_mean
 
-    btn.click(on_tag, inputs=[inp, lang], outputs=[state, out_df, legend_md])
-    lang.change(on_lang, inputs=[state, lang], outputs=[out_df, legend_md])
+    btn.click(on_tag, inputs=[inp, lang], outputs=[state, out_df, legend_md, out_mean_df])
+    lang.change(on_lang, inputs=[state, lang], outputs=[out_df, legend_md, out_mean_df])
 
 if __name__ == "__main__":
     demo.launch()