Spaces:

Setur
/

Marka

Running

App Files Files Community

unijoh commited on Jan 21

Commit

6e15cef

verified ·

1 Parent(s): 10e1ea8

Update app.py

Browse files

Files changed (1) hide show

app.py +606 -330

app.py CHANGED Viewed

@@ -1,427 +1,703 @@
-import os
-import json
-import re
-import pandas as pd
 import gradio as gr
-from huggingface_hub import InferenceClient
 # ----------------------------
 # Config
 # ----------------------------
-MODEL_REPO = "Setur/BRAGD"
-TAG_LABELS_PATH = "tag_labels.json"
-TAG_OVERVIEW_CSV = "Sosialurin-BRAGD_tags.csv"
-# HF Inference API token should be set as a Space secret:
-# Settings -> Secrets -> BRAGD_API_TOKEN
-HF_TOKEN = os.getenv("BRAGD_API_TOKEN", "")
-client = InferenceClient(model=MODEL_REPO, token=HF_TOKEN)
-# ----------------------------
-# Styling
-# ----------------------------
-CSS = """
-:root{
-  --primary-500: #89AFA9;   /* active + hover */
-  --primary-200: #CFE1DD;   /* inactive */
-  --primary-600: #6f948e;
-  --page-bg: #f6f7f8;
-  --panel-bg: transparent;
-  --text: #111;
 }
-body, .gradio-container{
   background: var(--page-bg) !important;
-  color: var(--text);
 }
-/* Kill random panel backgrounds */
-.gradio-container .block, .gradio-container .wrap, .gradio-container .gr-panel{
-  background: var(--panel-bg) !important;
 }
-/* Textbox: DO NOT TOUCH VISUALLY (keep white, clean, consistent) */
 #input_box textarea{
-  background: #fff !important;
-  border: 1px solid rgba(0,0,0,0.10) !important;
-  border-radius: 8px !important;
-  box-shadow: 0 2px 6px rgba(0,0,0,0.06) !important;
-  font-size: 18px !important;
-  line-height: 1.4 !important;
-  padding: 16px !important;
-}
-/* Big Marka button */
-#tag_btn button{
-  background: var(--primary-500) !important;
-  color: #0b1b19 !important;
-  border: 1px solid var(--primary-600) !important;
-  border-radius: 8px !important;
-  font-weight: 700 !important;
-  font-size: 18px !important;
-  padding: 12px 16px !important;
-  box-shadow: 0 2px 8px rgba(0,0,0,0.10) !important;
-}
-#tag_btn button:hover{
-  filter: brightness(0.98);
-}
-/* Results header row */
 #results_hdr{
-  margin-top: 8px;
-  align-items: center;
 }
-/* Language switch (Radio styled as buttons) */
-#lang_col { display:flex; justify-content:flex-end; }
-#lang_radio { display:flex; justify-content:flex-end; gap:0.6rem; background:transparent !important; }
-#lang_radio fieldset, #lang_radio .wrap, #lang_radio .gr-form{ background:transparent !important; border:none !important; padding:0 !important; margin:0 !important; }
-#lang_radio input[type="radio"]{ display:none !important; }
-#lang_radio label{
   cursor:pointer;
-  padding:0.38rem 1.05rem;
-  border-radius:0.65rem;
-  background:var(--primary-200);
   border:1px solid var(--primary-600);
   color:#0b1b19;
-  font-weight:600;
-  box-shadow: 0 1px 2px rgba(0,0,0,0.06);
   margin:0 !important;
 }
-#lang_radio label:hover{ background:var(--primary-500); }
-#lang_radio label:has(input:checked){
-  background:var(--primary-500);
-  border-color:var(--primary-600);
 }
-/* Tables */
-.gr-dataframe, .gr-dataframe table{
-  background: #fff !important;
 }
-"""
-# ----------------------------
-# Tag label loading
-# ----------------------------
-def load_tag_labels(path: str):
-    with open(path, "r", encoding="utf-8") as f:
-        data = json.load(f)
-    return data
-LABELS = load_tag_labels(TAG_LABELS_PATH)
 # ----------------------------
-# Tag overview CSV loading (word class -> codes)
 # ----------------------------
-def load_tag_overview_csv(path: str):
-    """
-    Expects columns: 'word_class', 'tag_code'
-    """
-    try:
-        df = pd.read_csv(path)
-    except Exception:
-        return {}
-    # normalize column names
-    cols = {c.lower().strip(): c for c in df.columns}
-    wc_col = cols.get("word_class")
-    code_col = cols.get("tag_code")
-    if not wc_col or not code_col:
-        return {}
     out = {}
-    for wc, g in df.groupby(wc_col):
-        out[str(wc)] = sorted(set(str(x) for x in g[code_col].dropna().tolist()))
     return out
-CODES_BY_WC = load_tag_overview_csv(TAG_OVERVIEW_CSV)
 # ----------------------------
-# Model call
 # ----------------------------
-def run_model(sentence: str):
-    """
-    Calls HF Inference API, returns list of dict rows:
-    [{"word":..., "tag":..., "analysis":...}, ...]
-    """
-    sentence = (sentence or "").strip()
-    if not sentence:
-        return []
-    # The model returns token-level predictions; we assume BRAGD returns full tokens + tag string.
-    # We'll call text-generation or token-classification style; adjust if needed.
-    # Using InferenceClient.text_generation is safest for many Spaces, but we keep a robust fallback.
-    try:
-        # If your endpoint is a custom pipeline, you may need .post with raw JSON.
-        # Here we assume a simple text_generation that returns a tagged output.
-        # --- Replace this part if your Space already had a working call ---
-        out = client.text_generation(sentence, max_new_tokens=256)
-        # If your existing app already parses output, keep that logic below.
-    except Exception as e:
-        raise gr.Error(f"Model call failed: {e}")
-    # Try to parse rows from output if it's already JSON-like; otherwise fallback to line parsing.
-    rows = []
-    if isinstance(out, (list, dict)):
-        # If API returns structured rows, normalize
-        data = out
-        if isinstance(out, dict) and "rows" in out:
-            data = out["rows"]
-        if isinstance(data, list):
-            for r in data:
-                if isinstance(r, dict) and {"word", "tag"} <= set(r.keys()):
-                    rows.append({"word": r.get("word", ""), "tag": r.get("tag", ""), "analysis": r.get("analysis", "")})
-            return rows
-    text = str(out)
-    # Fallback: accept formats like "word<TAB>tag" per line or "word tag" etc.
-    for line in text.splitlines():
-        line = line.strip()
-        if not line:
-            continue
-        if "\t" in line:
-            w, t = line.split("\t", 1)
-        else:
-            parts = line.split()
-            if len(parts) < 2:
-                continue
-            w, t = parts[0], parts[1]
-        rows.append({"word": w, "tag": t, "analysis": ""})
-    return rows
 # ----------------------------
-# Tag explanation logic
 # ----------------------------
-def label_for(lang: str, group: str, key: str, default: str = ""):
-    try:
-        return LABELS[lang][group][key]
-    except Exception:
-        return default
-def analysis_text(tag: str, lang: str):
-    """
-    Build the readable analysis string from a BRAGD tag.
-    Keeps your earlier “rules” (no random punctuation analysis, supine-only for luttøkuháttur, etc.)
-    """
-    tag = (tag or "").strip()
-    if not tag:
-        return ""
-    # Punctuation tags: keep short
-    if tag == "KE":
-        return "teksetting, setningsendi" if lang == "fo" else "punctuation, end of sentence"
-    if tag in {"KC"}:
-        return "teksetting, komma" if lang == "fo" else "punctuation, comma"
-    # Pull out word class (first char)
-    wc = tag[0]
-    wc_label = label_for(lang, "word_class", wc, wc)
-    # If DGd (preposition) in Faroese, don’t show “eingin stigbending”
-    parts = [wc_label]
-    # Helpers: add label only if it’s not the “none” type for some categories
-    def add(group, k, skip_if=None):
-        val = label_for(lang, group, k, "")
-        if not val:
-            return
-        if skip_if and val == skip_if:
-            return
-        parts.append(val)
-    # Very lightweight heuristic parsing:
-    # This assumes your tag labels cover these keys.
-    # If your previous app had more detailed parsing, keep it and just keep the UI fixes in this file.
-    # Here we preserve the visible output style.
-    # Gender / number / case / etc are typically subsequent chars.
-    # We’ll attempt common positions, but safely ignore unknowns.
-    # Example mapping by position is model-specific; keep safe:
-    # gender (2nd char), number (3rd), case (4th), etc.
-    if len(tag) >= 2:
-        add("gender", tag[1], skip_if=("eingin kyn" if lang == "fo" else "no gender"))
-    if len(tag) >= 3:
-        add("number", tag[2])
-    if len(tag) >= 4:
-        add("case", tag[3])
-    # Degree / definiteness / declension etc can vary; try a few more chars without forcing nonsense.
-    for i, grp in [(4, "definiteness"), (5, "degree"), (6, "declension"), (7, "person"), (8, "tense"), (9, "mood"), (10, "voice")]:
-        if len(tag) > i:
-            # Special rule: Faroese luttøkuháttur (participle/supine) should only show supine + voice
-            # If the word class is participle (L), we avoid adding mood/tense/person noise.
-            if wc == "L" and grp in {"person", "mood", "tense"}:
-                continue
-            add(grp, tag[i])
-    # DGd special: suppress “no degree”
-    if wc == "D" and tag.startswith("DGd") and lang == "fo":
-        parts = [p for p in parts if p != "eingin stigbending"]
-    return ", ".join([p for p in parts if p])
-# ----------------------------
-# Rendering
-# ----------------------------
-def render(rows, lang: str):
-    """
-    Returns:
-      df_main: Word/Tag/Analysis table
-      df_mean: Expanded tags table (optional)
-      overview_md: overview markdown
-    """
-    # Main table
-    if lang == "fo":
-        cols = ["Orð", "Mark", "Útgreining"]
-    else:
-        cols = ["Word", "Tag", "Analysis"]
-    data = []
-    for r in rows:
-        w = r.get("word", "")
-        t = r.get("tag", "")
-        a = analysis_text(t, lang)
-        data.append([w, t, a])
-    df_main = pd.DataFrame(data, columns=cols)
-    # Expanded tags: keep simple but useful (word class + raw tag)
-    df_mean = pd.DataFrame(
-        [{"tag": r.get("tag", ""), "analysis": analysis_text(r.get("tag", ""), lang)} for r in rows],
-        columns=["tag", "analysis"],
-    )
-    return df_main, df_mean, build_overview(lang)
-def build_overview(lang: str):
-    """
-    Build the Tag Overview content from CODES_BY_WC + labels.
-    """
-    lines = []
-    title = "Markayvirlit / Tag Overview" if lang == "fo" else "Tag Overview"
-    lines.append(f"### {title}")
-    lines.append("")
-    # Word class name mapping
-    for wc, codes in sorted(CODES_BY_WC.items(), key=lambda x: x[0]):
-        wc_name = label_for(lang, "word_class", wc, wc)
-        lines.append(f"**{wc} — {wc_name}**")
-        if codes:
-            lines.append(", ".join(codes))
-        else:
-            lines.append("_—_")
         lines.append("")
-    return "\n".join(lines)
-# ----------------------------
-# UI
-# ----------------------------
-with gr.Blocks(css=CSS, title="Marka") as demo:
-    state = gr.State([])  # stores last tagged rows
-    with gr.Row():
-        with gr.Column(scale=2):
-            inp = gr.Textbox(
-                label="",
-                placeholder="Skriv her...",
-                lines=6,
-                elem_id="input_box",
-            )
-        with gr.Column(scale=1, min_width=360):
             gr.Markdown(
-                "## Marka\n\n"
                 "Skriv ein setning í kassan og fá hann markaðan.\n\n"
-                f"Myndil / Model: [{MODEL_REPO}](https://huggingface.co/{MODEL_REPO})"
             )
-            btn = gr.Button("Marka / Tag", elem_id="tag_btn")
     # Hide results header + toggle until Tag
-    results_hdr = gr.Row(elem_id="results_hdr", visible=False)
     with results_hdr:
-        with gr.Column(scale=1):
-            results_title = gr.Markdown("### Úrslit / Results")
-        with gr.Column(scale=0, min_width=260, elem_id="lang_col"):
-            lang_radio = gr.Radio(
-                choices=[("Føroyskt","fo"), ("English","en")],
-                value="fo",
-                show_label=False,
-                interactive=True,
-                elem_id="lang_radio",
-            )
     out_df = gr.Dataframe(
-        value=pd.DataFrame(columns=["Orð", "Mark", "Útgreining"]),
-        interactive=False,
         visible=False,
     )
     expanded_acc = gr.Accordion("Útgreinað marking / Expanded tags", open=False, visible=False)
     with expanded_acc:
         out_mean_df = gr.Dataframe(
-            value=pd.DataFrame(columns=["tag", "analysis"]),
-            interactive=False,
         )
     overview_acc = gr.Accordion("Markayvirlit / Tag Overview", open=False, visible=True)
     with overview_acc:
-        overview_md = gr.Markdown(build_overview("fo"), elem_id="overview_md")
-    # ----------------------------
-    # Callbacks
-    # ----------------------------
-    def on_tag(sentence, lang_value):
         rows = run_model(sentence)
-        df_main, df_mean, _ = render(rows, lang_value)
         return (
             rows,
             gr.update(value=df_main, visible=True),
             gr.update(value=df_mean),
-            gr.update(value=build_overview(lang_value)),
-            gr.update(visible=True),  # expanded_acc
-            gr.update(visible=True),  # results_hdr
         )
-    def on_lang(rows, lang_value):
-        # Allow switching the overview even before anything is tagged.
-        if not rows:
-            return (
-                gr.update(),
-                gr.update(),
-                gr.update(value=build_overview(lang_value)),
-            )
-        df_main, df_mean, _ = render(rows, lang_value)
         return (
             gr.update(value=df_main),
             gr.update(value=df_mean),
-            gr.update(value=build_overview(lang_value)),
         )
-    # Wiring
     btn.click(
         on_tag,
-        inputs=[inp, lang_radio],
-        outputs=[state, out_df, out_mean_df, overview_md, expanded_acc, results_hdr],
     )
-    lang_radio.change(
-        on_lang,
-        inputs=[state, lang_radio],
-        outputs=[out_df, out_mean_df, overview_md],
         queue=False,
     )

+import os, re, string, json
+from collections import defaultdict
 import gradio as gr
+import torch
+import numpy as np
+import pandas as pd
+from transformers import AutoTokenizer, AutoModelForTokenClassification
 # ----------------------------
 # Config
 # ----------------------------
+MODEL_ID = "Setur/BRAGD"
+TAGS_FILEPATH = "Sosialurin-BRAGD_tags.csv"
+LABELS_FILEPATH = "tag_labels.json"
+HF_TOKEN = os.getenv("BRAGD")
+if not HF_TOKEN:
+    raise RuntimeError("Missing BRAGD token secret (Space → Settings → Secrets → BRAGD).")
+if not os.path.exists(LABELS_FILEPATH):
+    raise RuntimeError(f"Missing {LABELS_FILEPATH}. Add it to the Space repo root.")
+INTERVALS = (
+    (15, 29), (30, 33), (34, 36), (37, 41), (42, 43), (44, 45), (46, 50),
+    (51, 53), (54, 60), (61, 63), (64, 66), (67, 70), (71, 72)
+)
+GROUP_ORDER = ["subcategory","gender","number","case","article","proper","degree","declension","mood","voice","tense","person","definiteness"]
+HIDE_CODES = {"subcategory": {"B"}}  # Subcategory B to be removed
+UI = {
+    "fo": {"w":"Orð", "t":"Mark", "s":"Útgreining", "m":"Útgreinað marking"},
+    "en": {"w":"Word","t":"Tag", "s":"Analysis", "m":"Expanded tags"},
+}
+MODEL_LINK = "https://huggingface.co/Setur/BRAGD"
+CSS = """:root{
+  --primary-500:#89AFA9; --primary-600:#6F9992; --primary-700:#5B7F79;
+  --primary-100:#E1ECEA; --primary-200:#C6DAD6;
+  --page-bg:#f7f7f8;
 }
+/* Page background */
+html, body, .gradio-container{
   background: var(--page-bg) !important;
 }
+body, .gradio-container, .prose, .markdown, textarea, input, select, button, table{
+  font-family:-apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Arial, "Noto Sans", sans-serif !important;
+}
+a{ color:var(--primary-700)!important; }
+/* Primary button (Marka/Tag) */
+.gr-button-primary, button.primary, .primary{
+  background:var(--primary-500)!important;
+  border-color:var(--primary-600)!important;
+  color:#0b1b19!important;
 }
+.gr-button-primary:hover, button.primary:hover, .primary:hover{ background:var(--primary-600)!important; }
+.gr-button-primary{ padding:0.35rem 0.85rem!important; font-size:0.95rem!important; }
+/* --- Keep the textbox exactly as-is: wrapper blends with page, textarea stays white --- */
+#input_col, #input_col *{
+  background: transparent !important;
+}
+#input_col .gr-block, #input_col .gr-panel, #input_col .gr-box, #input_col .gr-group, #input_col .gr-form{
+  background: transparent !important;
+  box-shadow:none !important;
+  border:0 !important;
+}
+#input_box, #input_box > div, #input_box .wrap, #input_box .container{
+  background: transparent !important;
+  box-shadow:none !important;
+  border:0 !important;
+}
 #input_box textarea{
+  background:#ffffff !important;
+}
+/* Dataframe columns: keep Orð + Mark single-line */
+.gr-dataframe table td:nth-child(1), .gr-dataframe table th:nth-child(1){
+  white-space: nowrap !important; width: 18% !important;
+}
+.gr-dataframe table td:nth-child(2), .gr-dataframe table th:nth-child(2){
+  white-space: nowrap !important; width: 18% !important;
+  font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace !important;
+}
+.gr-dataframe table td:nth-child(3), .gr-dataframe table th:nth-child(3){
+  white-space: normal !important; width: 64% !important;
+}
+/* Selected = match Marka/Tag exactly */
+/* Hover = subtle */
+/* Keep selected button color on hover; only lighten UNSELECTED on hover */
+/* Push language buttons fully to the right */
+#results_hdr > .gr-markdown{
+  flex:1 1 auto !important;
+}
+/* Results header row: two-column layout, title left, toggle hard-right */
 #results_hdr{
+  display:grid !important;
+  grid-template-columns: 1fr auto !important;
+  align-items:center !important;
+  gap:12px !important;
+  padding:0 !important;
+  margin:0 !important;
+  background:transparent !important;
+  box-shadow:none !important;
+  border:0 !important;
+}
+#results_hdr > .gr-column:first-child{ justify-self:start !important; }
+#results_hdr > .gr-column:last-child{ justify-self:end !important; }
+/* Language toggle (gr.Radio): style the LABEL as the button (robust across Gradio DOM variants) */
+.lang_toggle{
+  background: transparent !important;
+  justify-self:end !important;
+}
+.lang_toggle fieldset{
+  border:0!important;
+  padding:0!important;
+  margin:0!important;
+  background:transparent!important;
+}
+.lang_toggle .wrap{
+  display:flex!important;
+  gap:10px!important;
+  background:transparent!important;
+  padding:0!important;
+  margin:0!important;
+}
+.lang_toggle input{
+  display:none!important;
+}
+/* Kill any default Gradio "pill" styling inside */
+.lang_toggle label *{
+  background:transparent!important;
+  box-shadow:none!important;
+  border:0!important;
+}
+/* The actual button */
+.lang_toggle label{
+  display:inline-flex !important;
+  align-items:center !important;
+  justify-content:center !important;
+  cursor:pointer !important;
+  user-select:none !important;
+  padding:0.35rem 0.85rem !important;
+  font-size:0.95rem !important;
+  border-radius:10px !important;
+  border:1px solid var(--primary-600) !important;
+  background: var(--primary-200) !important;  /* inactive: lighter than #89AFA9 */
+  color:#0b1b19 !important;                  /* black-ish */
+}
+/* Active/selected */
+.lang_toggle label:has(input:checked){
+  background: #89AFA9 !important;
+  border-color: var(--primary-600) !important;
+  color:#0b1b19 !important;
+}
+/* Hover: show #89AFA9 (inactive becomes active color on hover) */
+.lang_toggle label:hover{
+  background:#89AFA9 !important;
+  border-color: var(--primary-600) !important;
+  color:#0b1b19 !important;
 }
+/* Remove Gradio's default label styling completely */
+.lang_toggle label{
+  background:transparent!important;
+  border:0!important;
+  padding:0!important;
+  margin:0!important;
+  box-shadow:none!important;
+}
+/* Single visible button layer */
+.lang_toggle label span{
+  all: unset;
+  display:inline-block;
   cursor:pointer;
+  user-select:none;
+  padding:0.35rem 0.85rem;
+  font-size:0.95rem;
+  border-radius:10px;
   border:1px solid var(--primary-600);
+  background: transparent; /* same as page */
   color:#0b1b19;
+  box-shadow:none!important;
+}
+/* Selected state (robust selectors) */
+.lang_toggle input:checked ~ span,
+.lang_toggle label:has(input:checked) span{
+  background:var(--primary-500)!important;
+  border-color:var(--primary-600)!important;
+  color:#0b1b19!important;
+}
+/* Hover: only unselected gets light background */
+.lang_toggle label:hover input:not(:checked) ~ span,
+.lang_toggle label:hover:not(:has(input:checked)) span{
+  background:var(--primary-200)!important;
+}
+/* --- Language buttons (robust: 4 real buttons, show/hide to indicate active) --- */
+#results_hdr{
+  display:grid !important;
+  grid-template-columns: 1fr auto !important;
+  align-items:center !important;
+  gap:12px !important;
+  padding:0 !important;
   margin:0 !important;
+  background:transparent !important;
+  box-shadow:none !important;
+  border:0 !important;
 }
+#lang_buttons{
+  display:flex !important;
+  gap:10px !important;
+  justify-content:flex-end !important;
+  align-items:center !important;
+  flex-wrap:nowrap !important;
+}
+#lang_buttons .gr-button, #lang_buttons button{
+  padding:0.35rem 0.85rem !important;
+  font-size:0.95rem !important;
+  border-radius:10px !important;
 }
+/* Inactive: lighter than #89AFA9, black text */
+#lang_fo_off, #lang_en_off{
+  background:var(--primary-200) !important;
+  border-color:var(--primary-600) !important;
+  color:#0b1b19 !important;
+}
+/* Hover inactive -> active color (#89AFA9) */
+#lang_fo_off:hover, #lang_en_off:hover{
+  background:var(--primary-500) !important;
+  border-color:var(--primary-600) !important;
+  color:#0b1b19 !important;
+}
+/* Active: ensure black text */
+#lang_fo_on, #lang_en_on{
+  color:#0b1b19 !important;
 }
+/* Keep header transparent, but DON'T nuke button backgrounds */
+#results_hdr, #results_hdr > div{
+  background:transparent !important;
+  box-shadow:none !important;
+  border:0 !important;
+}
+/* Prevent Gradio from stacking/stretching language buttons */
+#lang_buttons .gr-button, #lang_buttons button{
+  width:auto !important;
+  min-width:120px !important;
+  flex:0 0 auto !important;
+}
+/* Language button colors */
+#lang_buttons .gr-button-primary, #lang_buttons button.primary{
+  background:#89AFA9 !important;
+  border-color:#6F9992 !important;
+  color:#0b1b19 !important;
+}
+#lang_buttons .gr-button-secondary, #lang_buttons button.secondary{
+  background:#C6DAD6 !important;  /* light green */
+  border-color:#6F9992 !important;
+  color:#0b1b19 !important;
+}
+#lang_buttons .gr-button-secondary:hover, #lang_buttons button.secondary:hover{
+  background:#89AFA9 !important;
+  border-color:#6F9992 !important;
+  color:#0b1b19 !important;
+}
+"""
 # ----------------------------
+# Tokenization
 # ----------------------------
+def simp_tok(sentence: str):
+    return re.findall(r"\w+|[" + re.escape(string.punctuation) + "]", sentence)
+# ----------------------------
+# CSV mapping
+# ----------------------------
+def load_tag_mappings(path: str):
+    df = pd.read_csv(path)
+    feature_cols = list(df.columns[1:])
+    tag_to_features = {row["Original Tag"]: row[1:].values.astype(int) for _, row in df.iterrows()}
+    features_to_tag = {tuple(row[1:].values.astype(int)): row["Original Tag"] for _, row in df.iterrows()}
+    return tag_to_features, features_to_tag, len(feature_cols), feature_cols
+def group_from_col(col: str):
+    if col == "Article": return ("article","A")
+    if col.startswith("No-Article "): return ("article", col.split()[-1])
+    if col == "Proper Noun": return ("proper","P")
+    if col.startswith("Not-Proper-Noun "): return ("proper", col.split()[-1])
+    prefixes = [
+        ("Word Class ","word_class"),
+        ("Subcategory ","subcategory"), ("No-Subcategory ","subcategory"),
+        ("Gender ","gender"), ("No-Gender ","gender"),
+        ("Number ","number"), ("No-Number ","number"),
+        ("Case ","case"), ("No-Case ","case"),
+        ("Degree ","degree"), ("No-Degree ","degree"),
+        ("Declension ","declension"), ("No-Declension ","declension"),
+        ("Mood ","mood"),
+        ("Voice ","voice"), ("No-Voice ","voice"),
+        ("Tense ","tense"), ("No-Tense ","tense"),
+        ("Person ","person"), ("No-Person ","person"),
+        ("Definite ","definiteness"), ("Indefinite ","definiteness"),
+    ]
+    for p,g in prefixes:
+        if col.startswith(p):
+            return (g, col.split()[-1])
+    return (None,None)
+def process_tag_features(tag_to_features: dict, intervals):
+    arrs = [np.array(tpl) for tpl in set(tuple(a) for a in tag_to_features.values())]
+    wt_masks = {wt:[a for a in arrs if a[wt]==1] for wt in range(15)}
     out = {}
+    for wt,labels in wt_masks.items():
+        if not labels:
+            out[wt]=[]
+            continue
+        sum_labels = np.sum(np.array(labels), axis=0)
+        out[wt] = [iv for iv in intervals if np.sum(sum_labels[iv[0]:iv[1]+1]) != 0]
     return out
+def predict_vectors(logits, attention_mask, begin_tokens, dict_intervals, vec_len):
+    softmax = torch.nn.Softmax(dim=0)
+    vectors = []
+    for idx in range(len(logits)):
+        if attention_mask[idx].item()!=1 or begin_tokens[idx]!=1:
+            continue
+        pred = logits[idx]
+        vec = torch.zeros(vec_len, device=logits.device)
+        wt = torch.argmax(softmax(pred[0:15])).item()
+        vec[wt]=1
+        for (a,b) in dict_intervals.get(wt, []):
+            seg = pred[a:b+1]
+            k = torch.argmax(softmax(seg)).item()
+            vec[a+k]=1
+        vectors.append(vec)
+    return vectors
 # ----------------------------
+# Load labels
 # ----------------------------
+with open(LABELS_FILEPATH, "r", encoding="utf-8") as f:
+    LABELS = json.load(f)
+def label_for(lang: str, group: str, wc: str, code: str) -> str:
+    lang = "fo" if lang=="fo" else "en"
+    by_wc = LABELS.get(lang, {}).get("by_word_class", {})
+    glob = LABELS.get(lang, {}).get("global", {})
+    if wc and wc in by_wc and code in by_wc[wc].get(group, {}):
+        return by_wc[wc][group][code]
+    return glob.get(group, {}).get(code, "")
+def clean_label(s: str) -> str:
+    s = (s or "").strip()
+    s = re.sub(r"\s+", " ", s)
+    return s.strip(" -;,:").strip()
 # ----------------------------
+# Load model + mapping
 # ----------------------------
+tag_to_features, features_to_tag, VEC_LEN, FEATURE_COLS = load_tag_mappings(TAGS_FILEPATH)
+tokenizer = AutoTokenizer.from_pretrained(MODEL_ID, token=HF_TOKEN)
+model = AutoModelForTokenClassification.from_pretrained(MODEL_ID, token=HF_TOKEN)
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+model.to(device); model.eval()
+if hasattr(model, "config") and hasattr(model.config, "num_labels") and model.config.num_labels != VEC_LEN:
+    raise RuntimeError(f"Label size mismatch: model={model.config.num_labels}, csv={VEC_LEN}. Wrong CSV?")
+DICT_INTERVALS = process_tag_features(tag_to_features, INTERVALS)
+GROUPS = defaultdict(list)
+for i,col in enumerate(FEATURE_COLS):
+    g,code = group_from_col(col)
+    if g and code not in HIDE_CODES.get(g, set()):
+        GROUPS[g].append((i, code, col))
+def vector_to_tag(vec: torch.Tensor) -> str:
+    return features_to_tag.get(tuple(vec.int().tolist()), "Unknown Tag")
+def wc_code(vec: torch.Tensor) -> str:
+    for idx,code,_ in GROUPS["word_class"]:
+        if int(vec[idx].item())==1:
+            return code
+    return ""
+def group_code(vec: torch.Tensor, group: str) -> str:
+    hidden = HIDE_CODES.get(group, set())
+    for idx,code,_ in GROUPS.get(group, []):
+        if code in hidden:
+            continue
+        if int(vec[idx].item())==1:
+            return code
+    return ""
+HIDE_IN_ANALYSIS = {("D","subcategory","G"), ("D","subcategory","N")}
+VOICE_ANALYSIS = {
+    "fo": {"A": "gerðsøgn", "M": "miðalsøgn", "v": "orð luttøkuháttur"},
+    "en": {"A": "active voice", "M": "middle voice", "v": "supine form"},
+}
+def analysis_text(vec: torch.Tensor, lang: str) -> str:
+    lang = "fo" if lang=="fo" else "en"
+    tag = vector_to_tag(vec)
+    wc = wc_code(vec)
+    if tag == "DGd":
+        return "fyriseting" if lang=="fo" else "preposition"
+    mood = group_code(vec, "mood")
+    if mood == "U":
+        sup = label_for(lang, "mood", wc, "U") or ("luttøkuháttur" if lang=="fo" else "supine")
+        vcode = group_code(vec, "voice") or "v"
+        vlabel = VOICE_ANALYSIS[lang].get(vcode, VOICE_ANALYSIS[lang]["v"])
+        return f"{clean_label(sup)}, {clean_label(vlabel)}"
+    parts = []
+    if wc in {"P","C"}:
+        subc = group_code(vec, "subcategory")
+        subl = clean_label(label_for(lang, "subcategory", wc, subc) or "")
+        if subl:
+            parts.append(subl)
+    else:
+        wcl = clean_label(label_for(lang, "word_class", wc, wc) or wc)
+        if wcl:
+            parts.append(wcl)
+    for g in GROUP_ORDER:
+        c = group_code(vec, g)
+        if not c:
+            continue
+        if wc in {"P","C"} and g == "subcategory":
+            continue
+        if (wc, g, c) in HIDE_IN_ANALYSIS:
+            continue
+        lbl = clean_label(label_for(lang, g, wc, c) or label_for(lang, g, "", c) or "")
+        if lbl and lbl not in parts:
+            parts.append(lbl)
+    return ", ".join(parts)
+def expanded_text(vec: torch.Tensor, lang: str) -> str:
+    lang = "fo" if lang=="fo" else "en"
+    wc = wc_code(vec)
+    parts = []
+    wc_lbl = label_for(lang, "word_class", wc, wc)
+    parts.append(f"{wc} – {wc_lbl}" if wc_lbl else wc)
+    for g in GROUP_ORDER:
+        c = group_code(vec, g)
+        if not c:
+            continue
+        lbl = label_for(lang, g, wc, c) or label_for(lang, g, "", c)
+        parts.append(f"{c} – {lbl}" if lbl else c)
+    return "; ".join([p for p in parts if p])
+def compute_codes_by_wc():
+    codes = defaultdict(lambda: defaultdict(set))
+    for arr in tag_to_features.values():
+        arr = np.array(arr)
+        wc = None
+        for idx,code,_ in GROUPS["word_class"]:
+            if arr[idx]==1:
+                wc = code
+                break
+        if not wc:
+            continue
+        for g in GROUP_ORDER:
+            hidden = HIDE_CODES.get(g, set())
+            for idx,code,_ in GROUPS.get(g, []):
+                if code in hidden:
+                    continue
+                if arr[idx]==1:
+                    codes[wc][g].add(code)
+    return codes
+CODES_BY_WC = compute_codes_by_wc()
+def build_overview(lang: str) -> str:
+    lang = "fo" if lang=="fo" else "en"
+    title = "### Markayvirlit" if lang=="fo" else "### Tag Overview"
+    lines = [title, ""]
+    for wc in sorted(CODES_BY_WC.keys()):
+        wcl = label_for(lang, "word_class", wc, wc) or ""
+        lines.append(f"#### {wc} — {wcl}" if wcl else f"#### {wc}")
+        for g in GROUP_ORDER:
+            cs = sorted(CODES_BY_WC[wc].get(g, set()))
+            if not cs:
+                continue
+            group_name = {
+                "fo": {"subcategory":"Undirflokkur","gender":"Kyn","number":"Tal","case":"Fall","article":"Bundni/óbundni",
+                       "proper":"Sernavn / felagsnavn","degree":"Stig","declension":"Bending","mood":"Háttur","voice":"Søgn",
+                       "tense":"Tíð","person":"Persónur","definiteness":"Bundni/óbundni"},
+                "en": {"subcategory":"Subcategory","gender":"Gender","number":"Number","case":"Case","article":"Definiteness",
+                       "proper":"Proper/common noun","degree":"Degree","declension":"Declension","mood":"Mood","voice":"Voice",
+                       "tense":"Tense","person":"Person","definiteness":"Definiteness"},
+            }[lang].get(g, g)
+            lines.append(f"**{group_name}**")
+            for c in cs:
+                lbl = label_for(lang, g, wc, c) or label_for(lang, g, "", c)
+                lines.append(f"- `{c}` — {lbl}" if lbl else f"- `{c}`")
+            lines.append("")
         lines.append("")
+    return "\n".join(lines).strip()
+def run_model(sentence: str):
+    s = (sentence or "").strip()
+    if not s:
+        return []
+    tokens = simp_tok(s)
+    if not tokens:
+        return []
+    enc = tokenizer(tokens, is_split_into_words=True, add_special_tokens=True, max_length=128,
+                    padding="max_length", truncation=True, return_attention_mask=True, return_tensors="pt")
+    input_ids = enc["input_ids"].to(device)
+    attention_mask = enc["attention_mask"].to(device)
+    word_ids = enc.word_ids(batch_index=0)
+    begin, last = [], None
+    for wid in word_ids:
+        if wid is None:
+            begin.append(0)
+        elif wid != last:
+            begin.append(1)
+        else:
+            begin.append(0)
+        last = wid
+    with torch.no_grad():
+        logits = model(input_ids=input_ids, attention_mask=attention_mask).logits[0]
+    vectors = predict_vectors(logits, attention_mask[0], begin, DICT_INTERVALS, VEC_LEN)
+    rows, vec_i, seen = [], 0, set()
+    for i,wid in enumerate(word_ids):
+        if wid is None or begin[i]!=1 or wid in seen:
+            continue
+        seen.add(wid)
+        word = tokens[wid] if wid < len(tokens) else "<UNK>"
+        vec = vectors[vec_i] if vec_i < len(vectors) else torch.zeros(VEC_LEN, device=device)
+        rows.append({"word": word, "vec": vec.int().tolist()})
+        vec_i += 1
+    return rows
+def render(rows_state, lang: str):
+    lang = "fo" if lang=="fo" else "en"
+    df_cols = [UI[lang]["w"], UI[lang]["t"], UI[lang]["s"]]
+    dfm_cols = [UI[lang]["w"], UI[lang]["t"], UI[lang]["m"]]
+    if not rows_state:
+        return (pd.DataFrame(columns=df_cols), pd.DataFrame(columns=dfm_cols), build_overview(lang))
+    out_main, out_mean = [], []
+    for r in rows_state:
+        vec = torch.tensor(r["vec"])
+        tag = vector_to_tag(vec)
+        out_main.append([r["word"], tag, analysis_text(vec, lang)])
+        out_mean.append([r["word"], tag, expanded_text(vec, lang)])
+    return (pd.DataFrame(out_main, columns=df_cols), pd.DataFrame(out_mean, columns=dfm_cols), build_overview(lang))
+theme = gr.themes.Soft()
+with gr.Blocks(theme=theme, css=CSS, title="Marka") as demo:
+    with gr.Row(equal_height=True):
+        with gr.Column(scale=2, elem_id="input_col"):
+            inp = gr.Textbox(lines=6, placeholder="Skriva her ... / Type here ...", show_label=False, elem_id="input_box")
+        with gr.Column(scale=1, min_width=320):
             gr.Markdown(
+                "## Marka\n"
                 "Skriv ein setning í kassan og fá hann markaðan.\n\n"
+                f"Myndil / Model: [{MODEL_ID}]({MODEL_LINK})"
             )
+            btn = gr.Button("Marka / Tag", variant="primary")
+    state = gr.State([])
+    lang_state = gr.State("fo")
     # Hide results header + toggle until Tag
+    results_hdr = gr.Row(elem_id="results_hdr", visible=True)
     with results_hdr:
+        results_title = gr.Markdown("### Úrslit / Results")
+        with gr.Row(elem_id="lang_buttons"):
+            btn_lang_fo_on  = gr.Button("Føroyskt", variant="primary",   elem_id="lang_fo_on",  visible=True)
+            btn_lang_fo_off = gr.Button("Føroyskt", variant="secondary", elem_id="lang_fo_off", visible=False)
+            btn_lang_en_on  = gr.Button("English",  variant="primary",   elem_id="lang_en_on",  visible=False)
+            btn_lang_en_off = gr.Button("English",  variant="secondary", elem_id="lang_en_off", visible=True)
     out_df = gr.Dataframe(
+        value=pd.DataFrame(columns=[UI["fo"]["w"], UI["fo"]["t"], UI["fo"]["s"]]),
+        wrap=True, interactive=False, show_label=False,
+        row_count=(0, "fixed"), col_count=(3, "fixed"),
         visible=False,
     )
     expanded_acc = gr.Accordion("Útgreinað marking / Expanded tags", open=False, visible=False)
     with expanded_acc:
         out_mean_df = gr.Dataframe(
+            value=pd.DataFrame(columns=[UI["fo"]["w"], UI["fo"]["t"], UI["fo"]["m"]]),
+            wrap=True, interactive=False, show_label=False,
+            row_count=(0, "fixed"), col_count=(3, "fixed"),
         )
     overview_acc = gr.Accordion("Markayvirlit / Tag Overview", open=False, visible=True)
     with overview_acc:
+        overview_md = gr.Markdown(build_overview("fo"))
+    def on_tag(sentence, lang_current):
         rows = run_model(sentence)
+        df_main, df_mean, overview = render(rows, lang_current)
+        show_fo = (lang_current == "fo")
+        show_en = (lang_current == "en")
         return (
             rows,
             gr.update(value=df_main, visible=True),
             gr.update(value=df_mean),
+            gr.update(value=overview),
+            gr.update(visible=True),   # expanded_acc
+            # results_hdr is always visible now
+            gr.update(visible=show_fo),   # fo_on
+            gr.update(visible=not show_fo), # fo_off
+            gr.update(visible=show_en),   # en_on
+            gr.update(visible=not show_en), # en_off
+            lang_current,
         )
+    def on_set_lang(rows, lang_value):
+        df_main, df_mean, overview = render(rows, lang_value)
+        show_fo = (lang_value == "fo")
+        show_en = (lang_value == "en")
         return (
+            lang_value,
             gr.update(value=df_main),
             gr.update(value=df_mean),
+            gr.update(value=overview),
+            gr.update(visible=show_fo),
+            gr.update(visible=not show_fo),
+            gr.update(visible=show_en),
+            gr.update(visible=not show_en),
         )
+    def on_set_fo(rows):
+        return on_set_lang(rows, "fo")
+    def on_set_en(rows):
+        return on_set_lang(rows, "en")
     btn.click(
         on_tag,
+        inputs=[inp, lang_state],
+        outputs=[state, out_df, out_mean_df, overview_md, expanded_acc,
+                 btn_lang_fo_on, btn_lang_fo_off, btn_lang_en_on, btn_lang_en_off, lang_state],
+        queue=False,
     )
+    # Language switch (does NOT rerun the model; just re-renders existing rows)
+    btn_lang_fo_on.click(
+        on_set_fo,
+        inputs=[state],
+        outputs=[lang_state, out_df, out_mean_df, overview_md,
+                 btn_lang_fo_on, btn_lang_fo_off, btn_lang_en_on, btn_lang_en_off],
+        queue=False,
+    )
+    btn_lang_fo_off.click(
+        on_set_fo,
+        inputs=[state],
+        outputs=[lang_state, out_df, out_mean_df, overview_md,
+                 btn_lang_fo_on, btn_lang_fo_off, btn_lang_en_on, btn_lang_en_off],
+        queue=False,
+    )
+    btn_lang_en_on.click(
+        on_set_en,
+        inputs=[state],
+        outputs=[lang_state, out_df, out_mean_df, overview_md,
+                 btn_lang_fo_on, btn_lang_fo_off, btn_lang_en_on, btn_lang_en_off],
+        queue=False,
+    )
+    btn_lang_en_off.click(
+        on_set_en,
+        inputs=[state],
+        outputs=[lang_state, out_df, out_mean_df, overview_md,
+                 btn_lang_fo_on, btn_lang_fo_off, btn_lang_en_on, btn_lang_en_off],
         queue=False,
     )